From phk at FreeBSD.org Thu Mar 1 08:09:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 1 Mar 2018 08:09:11 +0000 (UTC) Subject: [master] 23cfcb6 Exercise the raw H2 VSL records in at least one test Message-ID: <20180301080911.32A2FC0788@lists.varnish-cache.org> commit 23cfcb61a47f8c25e5940be91460317cad0668aa Author: Poul-Henning Kamp Date: Thu Mar 1 08:08:28 2018 +0000 Exercise the raw H2 VSL records in at least one test diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 3be9e17..6062a3a 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -17,7 +17,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set debug +syncvsl" -varnish v1 -cliok "param.set feature -http2" +varnish v1 -cliok "param.set vsl_mask +H2RxHdr,+H2RxBody,+H2TxHdr,+H2TxBody" client c1 { txpri From nils.goroll at uplex.de Thu Mar 1 10:20:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 1 Mar 2018 10:20:11 +0000 (UTC) Subject: [master] 2e0f09f fix std.ip documentation Message-ID: <20180301102011.08E4865C3F@lists.varnish-cache.org> commit 2e0f09fc5d2887e0508f383d52cdc43db62c3c99 Author: Nils Goroll Date: Tue Feb 27 22:06:24 2018 +0100 fix std.ip documentation diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 67d41f1..62579b5 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -172,7 +172,7 @@ Description the system library function getaddrinfo(3). If conversion fails, *fallback* will be returned. - If *resolve* is true, getaddrinfo() is called using *AI_NUMERICHOST* + If *resolve* is false, getaddrinfo() is called using *AI_NUMERICHOST* to avoid network lookups. This makes "pure" IP strings cheaper to convert. From phk at FreeBSD.org Thu Mar 1 11:20:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 1 Mar 2018 11:20:09 +0000 (UTC) Subject: [master] 98388b6 Rewrite to split lex and parsing into separate steps Message-ID: <20180301112009.46BFD95FA4@lists.varnish-cache.org> commit 98388b66951d600e0496fadc7f9ca1a64d8e80e6 Author: Poul-Henning Kamp Date: Thu Mar 1 11:19:08 2018 +0000 Rewrite to split lex and parsing into separate steps diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index fca5b03..e995ab3 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -169,20 +169,15 @@ def lwrap(s, width=64): return ll -def quote(s): - return s.replace("\"", "\\\"") - +####################################################################### -def indent(p, n): - n = len(p.expandtabs()) + n - p = "\t" * int(n / 8) - p += " " * int(n % 8) - return p -####################################################################### +inputline = None def err(str, warn=True): + if inputline is not None: + print("While parsing line:\n\t", inputline) if opts.strict or not warn: print("ERROR: " + str, file=sys.stderr) exit(1) @@ -203,13 +198,20 @@ enum_values = {} class ctype(object): - def __init__(self, vt, ct): - self.vt = vt - self.ct = ct + def __init__(self, wl): self.nm = None self.defval = None self.spec = None + self.vt = wl.pop(0) + self.ct = ctypes.get(self.vt) + if self.ct is None: + err("Expected type got '%s'" % self.vt, warn=False) + if len(wl) > 0 and wl[0] == "{": + if self.vt != "ENUM": + err("Only ENUMs take {...} specs", warn=False) + self.add_spec(wl) + def __str__(self): s = "<" + self.vt if self.nm is not None: @@ -220,6 +222,32 @@ class ctype(object): s += " SPEC=" + str(self.spec) return s + ">" + def set_defval(self, x): + if self.vt == "ENUM": + if x[0] == '"' and x[-1] == '"': + x = x[1:-1] + elif x[0] == "'" and x[-1] == "'": + x = x[1:-1] + self.defval = x + + def add_spec(self, wl): + assert self.vt == "ENUM" + assert wl.pop(0) == "{" + self.spec = [] + while True: + x = wl.pop(0) + if x[0] == '"' and x[-1] == '"': + x = x[1:-1] + elif x[0] == "'" and x[-1] == "'": + x = x[1:-1] + assert len(x) > 0 + self.spec.append(x) + enum_values[x] = True + w = wl.pop(0) + if w == "}": + break + assert w == "," + def vcl(self): if self.vt == "STRING_LIST": return "STRING" @@ -238,118 +266,102 @@ class ctype(object): jl[-1].pop(-1) -def vtype(txt): - j = len(txt) - for i in (',', ' ', '\n', '\t'): - x = txt.find(i) - if x > 0: - j = min(j, x) - t = txt[:j] - r = txt[j:].lstrip() - if t not in ctypes: - err("Did not recognize type <%s>" % txt) - ct = ctype(t, ctypes[t]) - if t != "ENUM": - return ct, r - assert r[0] == '{' - e = r[1:].split('}', 1) - r = e[1].lstrip() - e = e[0].split(',') - ct.spec = [] - for i in e: - j = i.strip() - enum_values[j] = True - ct.spec.append(j) - return ct, r - - -def arg(txt): - a, s = vtype(txt) - if len(s) == 0 or s[0] == ',': - return a, s - - i = s.find('=') - j = s.find(',') - if j < 0: - j = len(s) - if j < i: - i = -1 - if i < 0: - i = s.find(',') - if i < 0: - i = len(s) - a.nm = s[:i].rstrip() - s = s[i:] - return a, s - - a.nm = s[:i].rstrip() - s = s[i + 1:].lstrip() - if s[0] == '"' or s[0] == "'": - m = re.match("(['\"]).*?(\\1)", s) - if not m: - err("Unbalanced quote") - a.defval = s[:m.end()] - if a.vt == "ENUM": - a.defval = a.defval[1:-1] - s = s[m.end():] - else: - i = s.find(',') - if i < 0: - i = len(s) - a.defval = s[:i].rstrip() - s = s[i:] - if a.vt == "ENUM" and a.defval not in a.spec: - err("ENUM default value <%s> not valid" % a.defval, warn=False) +def lex(l): + wl = [] + s = 0 + for i in range(len(l)): + c = l[i] - return a, s + if s == 0 and re.match('[0-9a-zA-Z_.-]', c): + wl.append(c) + s = 3 + continue + if s == 3: + if re.match('[0-9a-zA-Z_.-]', c): + wl[-1] += c + continue + s = 0 + + if s == 0 and c in (' ', '\t', '\n', '\r'): + continue + + if s == 0 and c in ('(', '{', '}', ')', ',', '='): + wl.append(c) + elif s == 0 and c in ('"', "'"): + sep = c + s = 1 + wl.append(c) + elif s == 1: + if c == '\\': + s = 2 + else: + wl[-1] += c + if c == sep: + s = 0 + elif s == 2: + wl[-1] += c + s = 1 + else: + err("Syntax error at char", i, "'%s'" % c, warn=False) -def nmlegal(nm): - return re.match('^[a-zA-Z0-9_]+$', nm) + if s != 0: + err("Syntax error at char", i, "'%s'" % c, warn=False) + return wl -# XXX cant have ( or ) in an argument default value class prototype(object): def __init__(self, st, retval=True, prefix=""): + global inputline self.st = st self.obj = None - ll = st.line[1] + inputline = st.line[1] + wl = lex(st.line[1]) if retval: - self.retval, s = vtype(ll) - else: - self.retval = None - s = ll - i = s.find("(") - assert i > 0 + self.retval = ctype(wl) + + self.bname = wl.pop(0) + if not re.match("^[a-zA-Z.][a-zA-Z0-9_]*$", self.bname): + err("%s(): Illegal name\n" % self.nname, warn=False) + self.prefix = prefix - self.bname = s[:i].strip() self.name = self.prefix + self.bname self.vcc = st.vcc - if not nmlegal(self.cname()): - err("%s(): Illegal name\n" % self.name, warn=False) - s = s[i:].strip() - assert s[0] == "(" - assert s[-1] == ")" - s = s[1:-1].lstrip() - self.args = [] + if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', self.cname()): + err("%s(): Illegal C-name\n" % self.cname(), warn=False) + + x = wl.pop(0) + if x != "(": + err("Syntax error: Expected '(', got '%s'" % x, warn=False) + + x = wl.pop(-1) + if x != ")": + err("Syntax error: Expected ')', got '%s'" % x, warn=False) + names = {} - while len(s) > 0: - a, s = arg(s) - if a.nm is not None: - if not nmlegal(a.nm): - err("%s(): illegal argument name '%s'\n" - % (self.name, a.nm), warn=False) - if a.nm in names: - err("%s(): duplicate argument name '%s'\n" - % (self.name, a.nm), warn=False) - names[a.nm] = True - self.args.append(a) - s = s.lstrip() - if len(s) == 0: + self.args = [] + while len(wl) > 0: + t = ctype(wl) + self.args.append(t) + if not len(wl): + break + x = wl.pop(0) + if x == ",": + continue + if x in names: + err("%s(): Duplicate argument name" % x, warn=False) + names[x] = True + t.nm = x + if not len(wl): + break + if wl[0] == "=": + wl.pop(0) + t.set_defval(wl.pop(0)) + if not len(wl): break - assert s[0] == ',' - s = s[1:].lstrip() + assert wl.pop(0) == "," + inputline = None def cname(self, pfx=False): r = self.name.replace(".", "_") @@ -429,7 +441,7 @@ class prototype(object): def json(self, jl, cfunc): ll = [] self.retval.json(ll) - ll.append(cfunc) + ll.append('Vmod_%s_Func.%s' % (self.vcc.modname, cfunc)) for i in self.args: i.json(ll) jl.append(ll) @@ -613,18 +625,14 @@ class s_function(stanza): fo.write("\t" + self.proto.cname(pfx=True) + ",\n") def json(self, jl): - jl.append([ - "$FUNC", - "%s" % self.proto.name, - ]) - self.proto.json(jl[-1], 'Vmod_%s_Func.%s' % - (self.vcc.modname, self.proto.cname())) + jl.append(["$FUNC", "%s" % self.proto.name]) + self.proto.json(jl[-1], self.proto.cname()) class s_object(stanza): def parse(self): self.proto = prototype(self, retval=False) - self.proto.retval = vtype('VOID')[0] + self.proto.retval = ctype(['VOID']) self.proto.obj = "x" + self.proto.name self.init = copy.copy(self.proto) @@ -687,24 +695,22 @@ class s_object(stanza): def json(self, jl): ll = [ - "$OBJ", - self.proto.name, - "struct %s%s_%s" % - (self.vcc.sympfx, self.vcc.modname, self.proto.name), + "$OBJ", + self.proto.name, + "struct %s%s_%s" % + (self.vcc.sympfx, self.vcc.modname, self.proto.name), ] l2 = ["$INIT"] ll.append(l2) - self.init.json(l2, - 'Vmod_%s_Func.%s' % (self.vcc.modname, self.init.name)) + self.init.json(l2, self.init.name) l2 = ["$FINI"] ll.append(l2) - self.fini.json(l2, - 'Vmod_%s_Func.%s' % (self.vcc.modname, self.fini.name)) + self.fini.json(l2, self.fini.name) for i in self.methods: - i.json(ll) + i.json(ll) jl.append(ll) @@ -731,13 +737,8 @@ class s_method(stanza): fo.write('\t' + self.proto.cname(pfx=True) + ",\n") def json(self, jl): - jl.append([ - "$METHOD", - self.proto.name[len(self.pfx)+1:] - ]) - self.proto.json(jl[-1], - 'Vmod_%s_Func.%s' % - (self.vcc.modname, self.proto.cname())) + jl.append(["$METHOD", self.proto.name[len(self.pfx)+1:]]) + self.proto.json(jl[-1], self.proto.cname()) ####################################################################### @@ -864,26 +865,26 @@ class vcc(object): def json(self, fo): jl = [["$VMOD", "1.0"]] for j in self.contents: - j.json(jl) + j.json(jl) bz = bytearray(json.dumps(jl, separators=(",", ":")), - encoding = "ascii") + b"\0" + encoding="ascii") + b"\0" fo.write("\nstatic const char Vmod_Json[%d] = {\n" % len(bz)) t = "\t" for i in bz: - t += "%d," % i - if len(t) >= 69: - fo.write(t + "\n") - t = "\t" + t += "%d," % i + if len(t) >= 69: + fo.write(t + "\n") + t = "\t" if len(t) > 1: - fo.write(t[:-1]) + fo.write(t[:-1]) fo.write("\n};\n\n") for i in json.dumps(jl, indent=2, separators=(',', ': ')).split("\n"): - j = "// " + i - if len(j) > 72: - fo.write(j[:72] + "[...]\n") - else: - fo.write(j + "\n") + j = "// " + i + if len(j) > 72: + fo.write(j[:72] + "[...]\n") + else: + fo.write(j + "\n") fo.write("\n") def api(self, fo): From phk at FreeBSD.org Fri Mar 2 14:16:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 2 Mar 2018 14:16:08 +0000 (UTC) Subject: [master] e446c07 More restructuring Message-ID: <20180302141608.7C844BA259@lists.varnish-cache.org> commit e446c076a8ccbc8db7832add1776748b7b6fc5b0 Author: Poul-Henning Kamp Date: Fri Mar 2 10:59:34 2018 +0000 More restructuring diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e995ab3..aee9756 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -45,8 +45,6 @@ import random import copy import json -strict_abi = True - AMBOILERPLATE = ''' # Boilerplate generated by vmodtool.py - changes will be overwritten @@ -166,9 +164,18 @@ def lwrap(s, width=64): p = " " if len(s) > 0: ll.append(p + s) - return ll + return "\n".join(ll) + "\n" +def fmt_cstruct(fo, mn, x): + """ + Align fields in C struct + """ + a = "\ttd_" + mn + "_" + x + while len(a.expandtabs()) < 40: + a += "\t" + fo.write("%s*%s;\n" % (a, x)) + ####################################################################### @@ -184,21 +191,11 @@ def err(str, warn=True): else: print("WARNING: " + str, file=sys.stderr) - -def fmt_cstruct(fo, mn, x): - a = "\ttd_" + mn + "_" + x - while len(a.expandtabs()) < 40: - a += "\t" - fo.write("%s*%s;\n" % (a, x)) - ####################################################################### -enum_values = {} - - class ctype(object): - def __init__(self, wl): + def __init__(self, wl, enums): self.nm = None self.defval = None self.spec = None @@ -210,7 +207,7 @@ class ctype(object): if len(wl) > 0 and wl[0] == "{": if self.vt != "ENUM": err("Only ENUMs take {...} specs", warn=False) - self.add_spec(wl) + self.add_spec(wl, enums) def __str__(self): s = "<" + self.vt @@ -222,15 +219,7 @@ class ctype(object): s += " SPEC=" + str(self.spec) return s + ">" - def set_defval(self, x): - if self.vt == "ENUM": - if x[0] == '"' and x[-1] == '"': - x = x[1:-1] - elif x[0] == "'" and x[-1] == "'": - x = x[1:-1] - self.defval = x - - def add_spec(self, wl): + def add_spec(self, wl, enums): assert self.vt == "ENUM" assert wl.pop(0) == "{" self.spec = [] @@ -242,7 +231,7 @@ class ctype(object): x = x[1:-1] assert len(x) > 0 self.spec.append(x) - enum_values[x] = True + enums[x] = True w = wl.pop(0) if w == "}": break @@ -261,10 +250,48 @@ class ctype(object): return self.vt def json(self, jl): - jl.append([self.vt, self.nm, self.defval, self.spec]) + jl.append([self.vt]) while jl[-1][-1] is None: jl[-1].pop(-1) +####################################################################### + + +class arg(ctype): + def __init__(self, wl, argnames, enums, end): + super(arg, self).__init__(wl, enums) + + if wl[0] == end: + return + + x = wl.pop(0) + if x in argnames: + err("Duplicate argument name '%s'" % x, warn=False) + argnames[x] = True + self.nm = x + + if wl[0] == end: + return + + x = wl.pop(0) + if x != "=": + err("Expected '=' got '%s'" % x, warn=False) + + x = wl.pop(0) + if self.vt == "ENUM": + if x[0] == '"' and x[-1] == '"': + x = x[1:-1] + elif x[0] == "'" and x[-1] == "'": + x = x[1:-1] + self.defval = x + + def json(self, jl): + jl.append([self.vt, self.nm, self.defval, self.spec]) + while jl[-1][-1] is None: + jl[-1].pop(-1) + +####################################################################### + def lex(l): wl = [] @@ -309,65 +336,49 @@ def lex(l): err("Syntax error at char", i, "'%s'" % c, warn=False) return wl +####################################################################### + class prototype(object): def __init__(self, st, retval=True, prefix=""): - global inputline self.st = st self.obj = None - inputline = st.line[1] + self.args = [] wl = lex(st.line[1]) if retval: - self.retval = ctype(wl) + self.retval = ctype(wl, st.vcc.enums) + else: + self.retval = ctype(['VOID'], st.vcc.enums) self.bname = wl.pop(0) if not re.match("^[a-zA-Z.][a-zA-Z0-9_]*$", self.bname): err("%s(): Illegal name\n" % self.nname, warn=False) - self.prefix = prefix - self.name = self.prefix + self.bname - self.vcc = st.vcc + self.name = prefix + self.bname if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', self.cname()): err("%s(): Illegal C-name\n" % self.cname(), warn=False) - x = wl.pop(0) - if x != "(": - err("Syntax error: Expected '(', got '%s'" % x, warn=False) + if len(wl) == 2 and wl[0] == '(' and wl[1] == ')': + return - x = wl.pop(-1) - if x != ")": - err("Syntax error: Expected ')', got '%s'" % x, warn=False) + if wl[0] != "(": + err("Syntax error: Expected '(', got '%s'" % wl[0], warn=False) + wl[0] = ',' + + if wl[-1] != ")": + err("Syntax error: Expected ')', got '%s'" % wl[-1], warn=False) + wl[-1] = ',' names = {} - self.args = [] while len(wl) > 0: - t = ctype(wl) - self.args.append(t) - if not len(wl): - break x = wl.pop(0) - if x == ",": - continue - if x in names: - err("%s(): Duplicate argument name" % x, warn=False) - names[x] = True - t.nm = x - if not len(wl): - break - if wl[0] == "=": - wl.pop(0) - t.set_defval(wl.pop(0)) - if not len(wl): + if x != ',': + err("Expected ',' found '%s'" % x, warn=False) + if len(wl) == 0: break - assert wl.pop(0) == "," - inputline = None - - def cname(self, pfx=False): - r = self.name.replace(".", "_") - if pfx: - return self.vcc.sympfx + r - return r + t = arg(wl, names, st.vcc.enums, ',') + self.args.append(t) def vcl_proto(self, short, pfx=""): if type(self.st) == s_method: @@ -420,28 +431,39 @@ class prototype(object): fo.write(self.vcl_proto(True, pfx=" ") + "\n") fo.write(" \n") - def c_ret(self): - return self.retval.ct + def cname(self, pfx=False): + r = self.name.replace(".", "_") + if pfx: + return self.st.vcc.sympfx + r + return r - def c_args(self, a=[]): - ll = list(a) + def proto(self, args, name): + s = self.retval.ct + " " + name + '(' + ll = args for i in self.args: ll.append(i.ct) - return ", ".join(ll) - - def c_fn(self, args=[], h=False): - s = fn = '' - if not h: - s += 'typedef ' - fn += 'td_' + self.vcc.modname + '_' - fn += self.cname(pfx=h) - s += '%s %s(%s);' % (self.c_ret(), fn, self.c_args(args)) - return "\n".join(lwrap(s)) + "\n" + s += ", ".join(ll) + return s + ');' + + def typedef(self, args): + tn = 'td_' + self.st.vcc.modname + '_' + self.cname() + return "typedef " + self.proto(args, name=tn) + + def cstuff(self, args, where): + if where == 'h': + s = self.proto(args, self.cname(True)) + elif where == 'c': + s = self.typedef(args) + elif where == 'o': + s = self.typedef(args) + else: + assert False + return lwrap(s) def json(self, jl, cfunc): ll = [] self.retval.json(ll) - ll.append('Vmod_%s_Func.%s' % (self.vcc.modname, cfunc)) + ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc)) for i in self.args: i.json(ll) jl.append(ll) @@ -492,13 +514,10 @@ class stanza(object): if self.proto is not None: self.proto.synopsis(fo, man) - def hfile(self, fo): + def cstuff(self, fo, where): return - def cstruct(self, fo): - return - - def cstruct_init(self, fo): + def cstruct(self, fo, define): return def json(self, jl): @@ -566,11 +585,10 @@ class s_module(stanza): class s_abi(stanza): def parse(self): - global strict_abi if self.line[1] not in ('strict', 'vrt'): err("Valid ABI types are 'strict' or 'vrt', got '%s'\n" % self.line[1]) - strict_abi = self.line[1] == 'strict' + self.vcc.strict_abi = self.line[1] == 'strict' self.vcc.contents.append(self) @@ -590,14 +608,15 @@ class s_event(stanza): err("Not emitting .RST for $Event %s\n" % self.event_func) - def hfile(self, fo): - fo.write("vmod_event_f %s;\n" % self.event_func) - - def cstruct(self, fo): - fo.write("\tvmod_event_f\t\t\t*_event;\n") + def cstuff(self, fo, where): + if where == 'h': + fo.write("vmod_event_f %s;\n" % self.event_func) - def cstruct_init(self, fo): - fo.write("\t%s,\n" % self.event_func) + def cstruct(self, fo, define): + if define: + fo.write("\tvmod_event_f\t\t\t*_event;\n") + else: + fo.write("\t%s,\n" % self.event_func) def json(self, jl): jl.append([ @@ -612,17 +631,15 @@ class s_function(stanza): self.rstlbl = "func_" + self.proto.name self.vcc.contents.append(self) - def hfile(self, fo): - fo.write(self.proto.c_fn(['VRT_CTX'], True)) - - def cfile(self, fo): - fo.write(self.proto.c_fn(['VRT_CTX'])) + def cstuff(self, fo, where): + if where in ('h', 'c'): + fo.write(self.proto.cstuff(['VRT_CTX'], where)) - def cstruct(self, fo): - fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) - - def cstruct_init(self, fo): - fo.write("\t" + self.proto.cname(pfx=True) + ",\n") + def cstruct(self, fo, define): + if define: + fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) + else: + fo.write("\t" + self.proto.cname(pfx=True) + ",\n") def json(self, jl): jl.append(["$FUNC", "%s" % self.proto.name]) @@ -632,7 +649,6 @@ class s_function(stanza): class s_object(stanza): def parse(self): self.proto = prototype(self, retval=False) - self.proto.retval = ctype(['VOID']) self.proto.obj = "x" + self.proto.name self.init = copy.copy(self.proto) @@ -662,35 +678,27 @@ class s_object(stanza): for i in self.methods: i.proto.synopsis(fo, man) - def chfile(self, fo, h): + def cstuff(self, fo, w): sn = self.vcc.sympfx + self.vcc.modname + "_" + self.proto.name fo.write("struct %s;\n" % sn) - fo.write(self.init.c_fn( - ['VRT_CTX', 'struct %s **' % sn, 'const char *'], h)) - fo.write(self.fini.c_fn(['struct %s **' % sn], h)) + fo.write(self.init.cstuff( + ['VRT_CTX', 'struct %s **' % sn, 'const char *'], w)) + fo.write(self.fini.cstuff(['struct %s **' % sn], w)) for i in self.methods: - fo.write(i.proto.c_fn(['VRT_CTX', 'struct %s *' % sn], h)) + fo.write(i.proto.cstuff(['VRT_CTX', 'struct %s *' % sn], w)) fo.write("\n") - def hfile(self, fo): - self.chfile(fo, True) - - def cfile(self, fo): - self.chfile(fo, False) - - def cstruct(self, fo): - fmt_cstruct(fo, self.vcc.modname, self.init.name) - fmt_cstruct(fo, self.vcc.modname, self.fini.name) - for i in self.methods: - i.cstruct(fo) - - def cstruct_init(self, fo): - p = "\t" + self.vcc.sympfx - fo.write(p + self.init.name + ",\n") - fo.write(p + self.fini.name + ",\n") + def cstruct(self, fo, define): + if define: + fmt_cstruct(fo, self.vcc.modname, self.init.name) + fmt_cstruct(fo, self.vcc.modname, self.fini.name) + else: + p = "\t" + self.vcc.sympfx + fo.write(p + self.init.name + ",\n") + fo.write(p + self.fini.name + ",\n") for i in self.methods: - i.cstruct_init(fo) + i.cstruct(fo, define) fo.write("\n") def json(self, jl): @@ -719,6 +727,8 @@ class s_object(stanza): for i in self.methods: i.dump() +####################################################################### + class s_method(stanza): def parse(self): @@ -730,11 +740,11 @@ class s_method(stanza): self.rstlbl = "func_" + self.proto.name p.methods.append(self) - def cstruct(self, fo): - fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) - - def cstruct_init(self, fo): - fo.write('\t' + self.proto.cname(pfx=True) + ",\n") + def cstruct(self, fo, define): + if define: + fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) + else: + fo.write('\t' + self.proto.cname(pfx=True) + ",\n") def json(self, jl): jl.append(["$METHOD", self.proto.name[len(self.pfx)+1:]]) @@ -763,6 +773,8 @@ class vcc(object): self.contents = [] self.commit_files = [] self.copyright = "" + self.enums = {} + self.strict_abi = True def openfile(self, fn): self.commit_files.append(fn) @@ -773,6 +785,7 @@ class vcc(object): os.rename(i + ".tmp", i) def parse(self): + global inputline a = "\n" + open(self.inputfile, "r").read() s = a.split("\n$") self.copyright = s.pop(0).strip() @@ -783,11 +796,13 @@ class vcc(object): i += 1 else: i = len(ss) + inputline = ss[:i] c = ss[:i].split() m = dispatch.get(c[0]) if m is None: err("Unknown stanze $%s" % ss[:i]) m([c[0], " ".join(c[1:])], ss[i:].split('\n'), self) + inputline = None def rst_copyright(self, fo): write_rst_hdr(fo, "COPYRIGHT", "=") @@ -818,8 +833,7 @@ class vcc(object): fo.close() def amboilerplate(self): - fn = "automake_boilerplate.am" - fo = self.openfile(fn) + fo = self.openfile("automake_boilerplate.am") fo.write(AMBOILERPLATE.replace("XXX", self.modname)) fo.close() @@ -835,30 +849,28 @@ class vcc(object): fo.write("#endif\n") fo.write("\n") - for j in sorted(enum_values): + for j in sorted(self.enums): fo.write("extern VCL_ENUM %senum_%s;\n" % (self.sympfx, j)) fo.write("\n") for j in self.contents: - j.hfile(fo) + j.cstuff(fo, 'h') fo.close() def cstruct(self, fo, csn): - fo.write("\n%s {\n" % csn) for j in self.contents: - j.cstruct(fo) - fo.write("\n") - for j in sorted(enum_values): + j.cstruct(fo, True) + for j in sorted(self.enums): fo.write("\tVCL_ENUM\t\t\t*enum_%s;\n" % j) fo.write("};\n") def cstruct_init(self, fo, csn): fo.write("\nstatic const %s Vmod_Func = {\n" % csn) for j in self.contents: - j.cstruct_init(fo) + j.cstruct(fo, False) fo.write("\n") - for j in sorted(enum_values): + for j in sorted(self.enums): fo.write("\t&%senum_%s,\n" % (self.sympfx, j)) fo.write("};\n") @@ -887,14 +899,13 @@ class vcc(object): fo.write(j + "\n") fo.write("\n") - def api(self, fo): + def vmod_data(self, fo): + vmd = "Vmod_%s_Data" % self.modname for i in (714, 759, 765): - fo.write("\n/*lint -esym(%d, Vmod_%s_Data) */\n" % - (i, self.modname)) - fo.write("\nextern const struct vmod_data Vmod_%s_Data;\n" % - (self.modname)) - fo.write("\nconst struct vmod_data Vmod_%s_Data = {\n" % self.modname) - if strict_abi: + fo.write("\n/*lint -esym(%d, %s) */\n" % (i, vmd)) + fo.write("\nextern const struct vmod_data %s;\n" % vmd) + fo.write("\nconst struct vmod_data %s = {\n" % vmd) + if self.strict_abi: fo.write("\t.vrt_major =\t0,\n") fo.write("\t.vrt_minor =\t0,\n") else: @@ -917,59 +928,55 @@ class vcc(object): fo.write("};\n") def cfile(self): - fn = self.pfx + ".c" - fo = self.openfile(fn) - write_c_file_warning(fo) + fno = self.pfx + ".c" + fo = self.openfile(fno) + fnx = fno + ".tmp2" + fx = open(fnx, "w") - fn2 = fn + ".tmp2" + write_c_file_warning(fo) fo.write('#include "config.h"\n') fo.write('#include \n') for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: fo.write('#include "%s.h"\n' % i) - fo.write("\n") - for j in sorted(enum_values): + for j in sorted(self.enums): fo.write('VCL_ENUM %senum_%s = "%s";\n' % (self.sympfx, j, j)) fo.write("\n") - fx = open(fn2, "w") - for i in self.contents: if type(i) == s_object: - i.cfile(fo) - i.cfile(fx) + i.cstuff(fo, 'c') + i.cstuff(fx, 'c') fx.write("/* Functions */\n") for i in self.contents: if type(i) == s_function: - i.cfile(fo) - i.cfile(fx) + i.cstuff(fo, 'c') + i.cstuff(fx, 'c') csn = "Vmod_%s_Func" % self.modname + scsn = "struct " + csn - self.cstruct(fo, "struct " + csn) - - self.cstruct(fx, "struct " + csn) + self.cstruct(fo, scsn) + self.cstruct(fx, scsn) - fo.write("\n/*lint -esym(754, Vmod_" + self.modname + "_Func::*) */\n") - self.cstruct_init(fo, "struct " + csn) + fo.write("\n/*lint -esym(754, " + csn + "::*) */\n") + self.cstruct_init(fo, scsn) fx.close() fo.write("\nstatic const char Vmod_Proto[] =\n") - fi = open(fn2) - for i in fi: + for i in open(fnx): fo.write('\t"%s\\n"\n' % i.rstrip()) - fi.close() fo.write('\t"static struct %s %s;";\n' % (csn, csn)) - os.remove(fn2) + os.remove(fnx) self.json(fo) - self.api(fo) + self.vmod_data(fo) fo.close() From phk at FreeBSD.org Fri Mar 2 14:16:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 2 Mar 2018 14:16:08 +0000 (UTC) Subject: [master] f279df5 Add support for optional arguments to VMOD functions. Message-ID: <20180302141608.949CBBA25C@lists.varnish-cache.org> commit f279df5536424d97a24ee1c9839909a90184c430 Author: Poul-Henning Kamp Date: Fri Mar 2 14:14:09 2018 +0000 Add support for optional arguments to VMOD functions. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 2191d4b..ddb1376 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -406,6 +406,8 @@ struct func_arg { const char *name; const char *val; struct expr *result; + int avail; + int optional; VTAILQ_ENTRY(func_arg) list; }; @@ -449,6 +451,7 @@ vcc_do_arg(struct vcc *tl, struct func_arg *fa) assert(e2->fmt == fa->type); fa->result = e2; } + fa->avail = 1; } static void @@ -462,6 +465,9 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, VTAILQ_HEAD(,func_arg) head; struct token *t1; const struct vjsn_val *vv, *vvp; + const char *sa; + char ssa[64]; + char ssa2[64]; CAST_OBJ_NOTNULL(vv, priv, VJSN_VAL_MAGIC); assert(vv->type == VJSN_ARRAY); @@ -471,6 +477,15 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, vv = VTAILQ_NEXT(vv, list); cfunc = vv->value; vv = VTAILQ_NEXT(vv, list); + sa = vv->value; + if (*sa == '\0') { + sa = NULL; + } else { + bprintf(ssa, "args_%u", tl->unique++); + VSB_printf(tl->curproc->prologue, " %s %s;\n", sa, ssa); + sa = ssa; + } + vv = VTAILQ_NEXT(vv, list); SkipToken(tl, '('); if (extra == NULL) extra = ""; @@ -504,6 +519,10 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, } } } + if (sa != NULL && vvp != NULL && vvp->type == VJSN_TRUE) { + fa->optional = 1; + vvp = VTAILQ_NEXT(vvp, list); + } AZ(vvp); } @@ -551,23 +570,37 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, SkipToken(tl, ','); } - e1 = vcc_mk_expr(rfmt, "%s(ctx%s\v+", cfunc, extra); + if (sa != NULL) + e1 = vcc_mk_expr(rfmt, "%s(ctx%s,\v+(\n", cfunc, extra); + else + e1 = vcc_mk_expr(rfmt, "%s(ctx%s\v+", cfunc, extra); VTAILQ_FOREACH_SAFE(fa, &head, list, fa2) { + if (fa->optional) + VSB_printf(tl->curproc->prologue, + " %s.valid_%s = %d;\n", sa, fa->name, fa->avail); if (fa->result == NULL && fa->type == ENUM && fa->val != NULL) vcc_do_enum(tl, fa, strlen(fa->val), fa->val); if (fa->result == NULL && fa->val != NULL) fa->result = vcc_mk_expr(fa->type, "%s", fa->val); - if (fa->result != NULL) + if (fa->result != NULL && sa != NULL) { + bprintf(ssa2, "\v1%s.%s = \v2,\n", sa, fa->name); + e1 = vcc_expr_edit(tl, e1->fmt, ssa2, e1, fa->result); + } else if (fa->result != NULL) { e1 = vcc_expr_edit(tl, e1->fmt, "\v1,\n\v2", e1, fa->result); - else { + } else if (!fa->optional) { VSB_printf(tl->sb, "Argument '%s' missing\n", fa->name); vcc_ErrWhere(tl, tl->t); } free(fa); } - *e = vcc_expr_edit(tl, e1->fmt, "\v1\n)\v-", e1, NULL); + if (sa != NULL) { + bprintf(ssa2, "\v1&%s\v-\n))", sa); + *e = vcc_expr_edit(tl, e1->fmt, ssa2, e1, NULL); + } else { + *e = vcc_expr_edit(tl, e1->fmt, "\v1\n)\v-", e1, NULL); + } SkipToken(tl, ')'); } diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index aee9756..853abb7 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -199,6 +199,7 @@ class ctype(object): self.nm = None self.defval = None self.spec = None + self.opt = False self.vt = wl.pop(0) self.ct = ctypes.get(self.vt) @@ -287,6 +288,8 @@ class arg(ctype): def json(self, jl): jl.append([self.vt, self.nm, self.defval, self.spec]) + if self.opt: + jl[-1].append(True) while jl[-1][-1] is None: jl[-1].pop(-1) @@ -313,7 +316,7 @@ def lex(l): if s == 0 and c in (' ', '\t', '\n', '\r'): continue - if s == 0 and c in ('(', '{', '}', ')', ',', '='): + if s == 0 and c in ('[', '(', '{', '}', ')', ']', ',', '='): wl.append(c) elif s == 0 and c in ('"', "'"): sep = c @@ -330,10 +333,10 @@ def lex(l): wl[-1] += c s = 1 else: - err("Syntax error at char", i, "'%s'" % c, warn=False) + err("Syntax error at char %d '%s'" % (i, c), warn=False) if s != 0: - err("Syntax error at char", i, "'%s'" % c, warn=False) + err("Syntax error at char %d '%s'" % (i, c), warn=False) return wl ####################################################################### @@ -344,6 +347,7 @@ class prototype(object): self.st = st self.obj = None self.args = [] + self.argstruct = False wl = lex(st.line[1]) if retval: @@ -371,13 +375,30 @@ class prototype(object): wl[-1] = ',' names = {} + n = 0 while len(wl) > 0: + n += 1 x = wl.pop(0) if x != ',': err("Expected ',' found '%s'" % x, warn=False) if len(wl) == 0: break - t = arg(wl, names, st.vcc.enums, ',') + if wl[0] == '[': + wl.pop(0) + t = arg(wl, names, st.vcc.enums, ']') + if t.nm is None: + err("Optional arguments must have names", warn=False) + t.opt = True + x = wl.pop(0) + if x != ']': + err("Expected ']' found '%s'" % x, warn=False) + self.argstruct = True + else: + t = arg(wl, names, st.vcc.enums, ',') + if t.nm is None: + t.nm2 = "arg%d" % n + else: + t.nm2 = t.nm self.args.append(t) def vcl_proto(self, short, pfx=""): @@ -406,6 +427,8 @@ class prototype(object): t += " " + i.nm if i.defval is not None: t += "=" + i.defval + if i.opt: + t = "[" + t + "]" ll.append(t) t = ",@".join(ll) if len(s + t) > 68 and not short: @@ -440,8 +463,11 @@ class prototype(object): def proto(self, args, name): s = self.retval.ct + " " + name + '(' ll = args - for i in self.args: - ll.append(i.ct) + if self.argstruct: + ll.append(self.argstructname() + "*") + else: + for i in self.args: + ll.append(i.ct) s += ", ".join(ll) return s + ');' @@ -449,21 +475,49 @@ class prototype(object): tn = 'td_' + self.st.vcc.modname + '_' + self.cname() return "typedef " + self.proto(args, name=tn) + def argstructname(self): + return "struct %s_arg" % self.cname(True) + + def argstructure(self): + s = "\n" + self.argstructname() + " {\n" + for i in self.args: + if i.opt: + assert i.nm is not None + s += "\tchar\t\t\tvalid_%s;\n" % i.nm + for i in self.args: + s += "\t" + i.ct + if len(i.ct) < 8: + s += "\t" + if len(i.ct) < 16: + s += "\t" + s += "\t" + i.nm2 + ";\n" + s += "};\n" + return s + def cstuff(self, args, where): + s = "" if where == 'h': - s = self.proto(args, self.cname(True)) + if self.argstruct: + s += self.argstructure() + s += lwrap(self.proto(args, self.cname(True))) elif where == 'c': - s = self.typedef(args) + s += lwrap(self.typedef(args)) elif where == 'o': - s = self.typedef(args) + if self.argstruct: + s += self.argstructure() + s += lwrap(self.typedef(args)) else: assert False - return lwrap(s) + return s def json(self, jl, cfunc): ll = [] self.retval.json(ll) ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc)) + if self.argstruct: + ll.append(self.argstructname()) + else: + ll.append("") for i in self.args: i.json(ll) jl.append(ll) @@ -632,8 +686,7 @@ class s_function(stanza): self.vcc.contents.append(self) def cstuff(self, fo, where): - if where in ('h', 'c'): - fo.write(self.proto.cstuff(['VRT_CTX'], where)) + fo.write(self.proto.cstuff(['VRT_CTX'], where)) def cstruct(self, fo, define): if define: @@ -948,13 +1001,13 @@ class vcc(object): for i in self.contents: if type(i) == s_object: i.cstuff(fo, 'c') - i.cstuff(fx, 'c') + i.cstuff(fx, 'o') fx.write("/* Functions */\n") for i in self.contents: if type(i) == s_function: i.cstuff(fo, 'c') - i.cstuff(fx, 'c') + i.cstuff(fx, 'o') csn = "Vmod_%s_Func" % self.modname scsn = "struct " + csn From phk at FreeBSD.org Fri Mar 2 14:16:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 2 Mar 2018 14:16:08 +0000 (UTC) Subject: [master] 6515f17 Add an optional argument to the debug.argtest function Message-ID: <20180302141608.AC14FBA260@lists.varnish-cache.org> commit 6515f17ed61ee979a170612a3467d87fedd757b1 Author: Poul-Henning Kamp Date: Fri Mar 2 14:14:47 2018 +0000 Add an optional argument to the debug.argtest function diff --git a/bin/varnishtest/tests/m00019.vtc b/bin/varnishtest/tests/m00019.vtc index c8251c9..b2c9d66 100644 --- a/bin/varnishtest/tests/m00019.vtc +++ b/bin/varnishtest/tests/m00019.vtc @@ -24,6 +24,7 @@ varnish v1 -vcl+backend { set resp.http.foo4 = debug.argtest("1", 2.4, three="3d", four=-1); set resp.http.foo5 = debug.argtest("1", 2.5); set resp.http.foo6 = debug.argtest("1", four=6); + set resp.http.foo7 = debug.argtest("1", opt="7"); set resp.http.obj0 = obj0.string() + ", " + obj0.number(); set resp.http.obj1 = obj1.string() + ", " + obj1.number(); @@ -38,12 +39,13 @@ client c1 { txreq rxresp expect resp.bodylen == "6" - expect resp.http.foo1 == "1 2.1 3a , 4" - expect resp.http.foo2 == "1 -2.2 3b , 4" - expect resp.http.foo3 == "1 2.3 3c , 4" - expect resp.http.foo4 == "1 2.4 3d , -1" - expect resp.http.foo5 == "1 2.5 3 , 4" - expect resp.http.foo6 == "1 2 3 , 6" + expect resp.http.foo1 == "1 2.1 3a , 4 0 " + expect resp.http.foo2 == "1 -2.2 3b , 4 0 " + expect resp.http.foo3 == "1 2.3 3c , 4 0 " + expect resp.http.foo4 == "1 2.4 3d , -1 0 " + expect resp.http.foo5 == "1 2.5 3 , 4 0 " + expect resp.http.foo6 == "1 2 3 , 6 0 " + expect resp.http.foo7 == "1 2 3 , 4 1 7" expect resp.http.obj0 == "default, one" expect resp.http.obj1 == "only_argument, one" diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index ba5fa94..c27e866 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -117,7 +117,9 @@ Encrypt the HTTP header with quad-ROT13 encryption, $Function STRING argtest( STRING one, REAL two =2, STRING three= "3", - STRING comma="," , INT four = 4 ) + STRING comma=",", INT four = 4, + [ STRING opt] +) $Function INT vre_limit() diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 579d384..d833fb2 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -151,12 +151,14 @@ xyzzy_rot52(VRT_CTX, VCL_HTTP hp) } VCL_STRING v_matchproto_(td_debug_argtest) -xyzzy_argtest(VRT_CTX, VCL_STRING one, VCL_REAL two, VCL_STRING three, - VCL_STRING comma, VCL_INT four) +xyzzy_argtest(VRT_CTX, struct xyzzy_argtest_arg *arg) { char buf[100]; - bprintf(buf, "%s %g %s %s %ld", one, two, three, comma, four); + AN(arg); + bprintf(buf, "%s %g %s %s %ld %d %s", + arg->one, arg->two, arg->three, arg->comma, arg->four, + arg->valid_opt, arg->valid_opt ? arg->opt : ""); return (WS_Copy(ctx->ws, buf, -1)); } From dridi.boukelmoune at gmail.com Mon Mar 5 08:21:09 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Mar 2018 08:21:09 +0000 (UTC) Subject: [master] 0f7baca Disable deprecated persistent storage by default Message-ID: <20180305082109.8BA44B2E66@lists.varnish-cache.org> commit 0f7baca494d06c07f9fc34e9594b36e8748f15b2 Author: Dridi Boukelmoune Date: Mon Mar 5 09:17:33 2018 +0100 Disable deprecated persistent storage by default diff --git a/autogen.des b/autogen.des index 6b75a76..0cc52dc 100755 --- a/autogen.des +++ b/autogen.des @@ -36,4 +36,5 @@ $SRCDIR/configure \ --enable-developer-warnings \ --enable-debugging-symbols \ --enable-dependency-tracking \ + --with-persistent-storage \ "$@" diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 66907db..a135f80 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -91,15 +91,11 @@ varnishd_SOURCES = \ mgt/mgt_vcl.c \ proxy/cache_proxy_proto.c \ storage/mgt_stevedore.c \ - storage/mgt_storage_persistent.c \ storage/stevedore.c \ storage/stevedore_utils.c \ storage/storage_file.c \ storage/storage_lru.c \ storage/storage_malloc.c \ - storage/storage_persistent.c \ - storage/storage_persistent_silo.c \ - storage/storage_persistent_subr.c \ storage/storage_simple.c \ storage/storage_umem.c \ waiter/cache_waiter.c \ @@ -109,6 +105,14 @@ varnishd_SOURCES = \ waiter/cache_waiter_ports.c \ waiter/mgt_waiter.c +if WITH_PERSISTENT_STORAGE +varnishd_SOURCES += \ + storage/mgt_storage_persistent.c \ + storage/storage_persistent.c \ + storage/storage_persistent_silo.c \ + storage/storage_persistent_subr.c +endif + nodist_varnishd_SOURCES = \ builtin_vcl.c diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index d1ab577..e2381a4 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -360,9 +360,11 @@ child_main(int sigmagic, size_t altstksz) CLI_AddFuncs(debug_cmds); +#if WITH_PERSISTENT_STORAGE /* Wait for persistent storage to load if asked to */ if (FEATURE(FEATURE_WAIT_SILO)) SMP_Ready(); +#endif CLI_Run(); diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index 8ebfca2..ba7e5b1 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -92,6 +92,7 @@ static struct cli_proto cli_stv[] = { /*-------------------------------------------------------------------- */ +#ifdef WITH_PERSISTENT_STORAGE static void v_matchproto_(storage_init_f) smp_fake_init(struct stevedore *parent, int ac, char * const *av) { @@ -111,6 +112,7 @@ static const struct stevedore smp_fake_stevedore = { .name = "deprecated_persistent", .init = smp_fake_init, }; +#endif /*-------------------------------------------------------------------- * Parse a stevedore argument on the form: @@ -120,8 +122,10 @@ static const struct stevedore smp_fake_stevedore = { static const struct choice STV_choice[] = { { "file", &smf_stevedore }, { "malloc", &sma_stevedore }, +#ifdef WITH_PERSISTENT_STORAGE { "deprecated_persistent", &smp_stevedore }, { "persistent", &smp_fake_stevedore }, +#endif #if defined(HAVE_LIBUMEM) { "umem", &smu_stevedore }, { "default", &smu_stevedore }, diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index e4d0aa6..37565c7 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -23,10 +23,6 @@ shell "varnishd -x vsl > ${tmpdir}/_.vsl" shell "varnishd -x cli > ${tmpdir}/_.cli" shell "varnishd -x builtin > ${tmpdir}/_.builtin" -shell -err -expect {-spersistent has been deprecated} { - varnishd -spersistent -f '' -} - shell -err -expect {-C needs either -b or -f } { varnishd -C } @@ -50,3 +46,10 @@ shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d " shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F " shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0 } +# This check is kept last because it may be skipped + +feature persistent_storage + +shell -err -expect {-spersistent has been deprecated} { + varnishd -spersistent -f '' +} diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index a2cdc2c..e3d2302 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -1,5 +1,7 @@ varnishtest "Test Basic persistence" +feature persistent_storage + server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc index 44d6c26..77a393b 100644 --- a/bin/varnishtest/tests/p00002.vtc +++ b/bin/varnishtest/tests/p00002.vtc @@ -1,5 +1,7 @@ varnishtest "Ban a persistent object" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per[12]" server s1 { diff --git a/bin/varnishtest/tests/p00003.vtc b/bin/varnishtest/tests/p00003.vtc index 1367fe8..9504393 100644 --- a/bin/varnishtest/tests/p00003.vtc +++ b/bin/varnishtest/tests/p00003.vtc @@ -1,5 +1,7 @@ varnishtest "Ban a persistent object" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per" server s1 { diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index 4e7326f..986f7ec 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -1,5 +1,7 @@ varnishtest "Check object references" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per" server s1 { diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc index 7d46d40..353c930 100644 --- a/bin/varnishtest/tests/p00005.vtc +++ b/bin/varnishtest/tests/p00005.vtc @@ -1,5 +1,7 @@ varnishtest "Check expiry of non-instantiated object" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per" server s1 { diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc index 38e2993..dfec357 100644 --- a/bin/varnishtest/tests/p00006.vtc +++ b/bin/varnishtest/tests/p00006.vtc @@ -1,5 +1,7 @@ varnishtest "Check that Vary headers are stored" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per" server s1 { diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc index ddae80d..08c5c27 100644 --- a/bin/varnishtest/tests/p00007.vtc +++ b/bin/varnishtest/tests/p00007.vtc @@ -1,5 +1,7 @@ varnishtest "test reload of object spanning incomplete segment" +feature persistent_storage + barrier b1 cond 2 barrier b2 cond 2 diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc index c3973b4..cde88be 100644 --- a/bin/varnishtest/tests/p00008.vtc +++ b/bin/varnishtest/tests/p00008.vtc @@ -1,5 +1,7 @@ varnishtest "Ban list sync across silos" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per[12]" # Silo 1 & 2 diff --git a/bin/varnishtest/tests/p00009.vtc b/bin/varnishtest/tests/p00009.vtc index 184beca..6a53adf 100644 --- a/bin/varnishtest/tests/p00009.vtc +++ b/bin/varnishtest/tests/p00009.vtc @@ -1,5 +1,7 @@ varnishtest "Check that reloaded bans with completed flag are really completed on restart" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per[12]" server s1 { diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc index 1add7cd..794a994 100644 --- a/bin/varnishtest/tests/r00915.vtc +++ b/bin/varnishtest/tests/r00915.vtc @@ -1,5 +1,7 @@ varnishtest "error object allocation with persistent" +feature persistent_storage + server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index 7f96219..84f5eea 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -1,5 +1,7 @@ varnishtest "Test address remapping" +feature persistent_storage + feature disable_aslr # VM-remapping is too random on OSX diff --git a/bin/varnishtest/tests/r01225.vtc b/bin/varnishtest/tests/r01225.vtc index 2cbf448..607f786 100644 --- a/bin/varnishtest/tests/r01225.vtc +++ b/bin/varnishtest/tests/r01225.vtc @@ -1,5 +1,7 @@ varnishtest "Test bans_req counter on persistent reload - #1225" +feature persistent_storage + shell "rm -f ${tmpdir}/_.per" server s1 { diff --git a/bin/varnishtest/tests/r01266.vtc b/bin/varnishtest/tests/r01266.vtc index 3d20685..f93f7a0 100644 --- a/bin/varnishtest/tests/r01266.vtc +++ b/bin/varnishtest/tests/r01266.vtc @@ -1,5 +1,7 @@ varnishtest "#1266 - Check persisted truncated completed bans" +feature persistent_storage + # Test that bans which has been completed, truncated and persisted works shell "rm -f ${tmpdir}/_.per1" diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 4b90c2f..26a2002 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -714,13 +714,22 @@ cmd_delay(CMD_ARGS) * Do not fail the test if a string of the form ${...} is not * recognized as a macro. * term - * Support for ADM3A terminal + * Support for ADM3A terminal + * + * persistent_storage + * Varnish was built with the deprecated persistent storage. * * Be careful with ignore_unknown_macro, because it may cause a test with a * misspelled macro to fail silently. You should only need it if you must * run a test with strings of the form "${...}". */ +#if WITH_PERSISTENT_STORAGE +static const unsigned with_persistent_storage = 1; +#else +static const unsigned with_persistent_storage = 0; +#endif + static int test_term(struct vtclog *vl) { @@ -787,6 +796,7 @@ cmd_feature(CMD_ARGS) FEATURE("user_vcache", getpwnam("vcache") != NULL); FEATURE("group_varnish", getgrnam("varnish") != NULL); FEATURE("term", test_term(vl)); + FEATURE("persistent_storage", with_persistent_storage); if (!strcmp(*av, "disable_aslr")) { good = 1; diff --git a/configure.ac b/configure.ac index 42ed74a..cf14412 100644 --- a/configure.ac +++ b/configure.ac @@ -372,6 +372,21 @@ else ac_cv_func_port_create=no fi +# --with-persistent-storage +AC_ARG_WITH(persistent-storage, + AS_HELP_STRING([--with-persistent-storage], + [use deprecated persistent storage (default is NO)]), + [], + [with_persistent_storage=no]) + +if test "$with_persistent_storage" = yes; then + AC_DEFINE([WITH_PERSISTENT_STORAGE], [1], + [Define to 1 to build the deprecated peristent storage.]) +fi + +AM_CONDITIONAL([WITH_PERSISTENT_STORAGE], + [test "$with_persistent_storage" = yes]) + AM_MISSING_HAS_RUN AC_CHECK_DECL([SO_ACCEPTFILTER], diff --git a/doc/sphinx/phk/persistent.rst b/doc/sphinx/phk/persistent.rst index 6587e67..2e0c40f 100644 --- a/doc/sphinx/phk/persistent.rst +++ b/doc/sphinx/phk/persistent.rst @@ -5,7 +5,8 @@ A persistent message ==================== This message is about -spersistent and why you should not use it, -even though it is still present in Varnish 4.x. +even though it is still present in Varnish 4.x. Starting with Varnish +6 it is only present when explicitly enabled at compile time. TL;DR: ------ From phk at FreeBSD.org Mon Mar 5 10:09:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 10:09:08 +0000 (UTC) Subject: [master] 2d0843e Rename SLT_F_BINARY to the more precise SLT_F_UNSAFE Message-ID: <20180305100908.4A13EB4ACA@lists.varnish-cache.org> commit 2d0843eac8d77cfb3369591a61234d9c3ed6881e Author: Poul-Henning Kamp Date: Mon Mar 5 08:45:43 2018 +0000 Rename SLT_F_BINARY to the more precise SLT_F_UNSAFE diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 4b75923..c80c135 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -47,7 +47,7 @@ #define NODEF_NOTICE \ "NB: This log record is masked by default.\n\n" -SLTM(Debug, SLT_F_BINARY, "Debug messages", +SLTM(Debug, SLT_F_UNSAFE, "Debug messages", "Debug messages can normally be ignored, but are sometimes" " helpful during trouble-shooting. Most debug messages must" " be explicitly enabled with parameters.\n\n" @@ -126,7 +126,7 @@ SLTM(BackendClose, 0, "Backend connection closed", "\n" ) -SLTM(HttpGarbage, SLT_F_BINARY, "Unparseable HTTP request", +SLTM(HttpGarbage, SLT_F_UNSAFE, "Unparseable HTTP request", "Logs the content of unparseable HTTP requests.\n\n" ) @@ -363,7 +363,7 @@ SLTM(ESI_xmlerror, 0, "ESI parser error or warning message", " The log record describes the problem encountered." ) -SLTM(Hash, SLT_F_BINARY, "Value added to hash", +SLTM(Hash, SLT_F_UNSAFE, "Value added to hash", "This value was added to the object lookup hash.\n\n" NODEF_NOTICE ) diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h index 2209ac3..b2d7f1c 100644 --- a/include/vapi/vsl.h +++ b/include/vapi/vsl.h @@ -163,7 +163,7 @@ extern const unsigned VSL_tagflags[SLT__MAX]; * Tag flags array. * Use these macros with VSL_tagflags (included from vsl_int.h): * - * VSL_tagflags[tag] & SLT_F_BINARY + * VSL_tagflags[tag] & SLT_F_UNSAFE * Non-zero if the payload with this tag may include * non-printable characters * diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index bfc24e5..57986cd 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -97,6 +97,6 @@ enum VSL_tag_e { /* VSL tag flags */ #define SLT_F_UNUSED (1 << 0) -#define SLT_F_BINARY (1 << 1) +#define SLT_F_UNSAFE (1 << 1) #endif /* VAPI_VSL_INT_H_INCLUDED */ diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index b4b7752..af23e92 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -244,7 +244,7 @@ VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) 'b' : '-'; data = VSL_CDATA(c->rec.ptr); - if (VSL_tagflags[tag] & SLT_F_BINARY) { + if (VSL_tagflags[tag] & SLT_F_UNSAFE) { VSL_PRINT(fo, "%10u %-14s %c \"", vxid, VSL_tags[tag], type); while (len-- > 0) { if (len == 0 && tag == SLT_Debug && *data == '\0') @@ -279,7 +279,7 @@ VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) len = VSL_LEN(c->rec.ptr); data = VSL_CDATA(c->rec.ptr); - if (VSL_tagflags[tag] & SLT_F_BINARY) { + if (VSL_tagflags[tag] & SLT_F_UNSAFE) { VSL_PRINT(fo, "%-14s \"", VSL_tags[tag]); while (len-- > 0) { if (len == 0 && tag == SLT_Debug && *data == '\0') From phk at FreeBSD.org Mon Mar 5 10:09:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 10:09:08 +0000 (UTC) Subject: [master] 1de0618 Flexelinting Message-ID: <20180305100908.5E131B4ACD@lists.varnish-cache.org> commit 1de06184ddb0508dfead90ada358a0dbd7407cfa Author: Poul-Henning Kamp Date: Mon Mar 5 09:38:08 2018 +0000 Flexelinting diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 183be32..ae1ceec 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -245,6 +245,8 @@ int STV_BanInfoDrop(const uint8_t *ban, unsigned len); int STV_BanInfoNew(const uint8_t *ban, unsigned len); void STV_BanExport(const uint8_t *banlist, unsigned len); +#if WITH_PERSISTENT_STORAGE /* storage_persistent.c */ void SMP_Ready(void); +#endif From phk at FreeBSD.org Mon Mar 5 10:09:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 10:09:08 +0000 (UTC) Subject: [master] 49d52eb Introduce a new SLT_F_BINARY flag which hexdumps the VSL record. Message-ID: <20180305100908.78C3FB4AD1@lists.varnish-cache.org> commit 49d52ebdfd8cdf96817cc2758155ff4adc989bfb Author: Poul-Henning Kamp Date: Mon Mar 5 09:40:32 2018 +0000 Introduce a new SLT_F_BINARY flag which hexdumps the VSL record. Set this on the H2 'raw' records. Move the hexdumping to the API side. diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 754c8f6..04e425c 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -392,37 +392,33 @@ VSLb_ts(struct vsl_log *vsl, const char *event, double first, double *pprev, void VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr) { + unsigned mlen; char *p; - const uint8_t *pp = ptr; - int suff = 0; - size_t tl, ll; - assert(len >= 0); - AN(pp); + vsl_sanity(vsl); + AN(ptr); if (vsl_tag_is_masked(tag)) return; - vsl_sanity(vsl); - tl = len * 2 + 1; - if (tl > cache_param->vsl_reclen) { - len = (cache_param->vsl_reclen - 2) / 2; - tl = len * 2 + 2; - suff = 1; - } - if (VSL_END(vsl->wlp, tl) >= vsl->wle) + mlen = cache_param->vsl_reclen; + + /* Truncate */ + if (len > mlen) + len = mlen; + + assert(vsl->wlp < vsl->wle); + + /* Flush if necessary */ + if (VSL_END(vsl->wlp, len) >= vsl->wle) VSL_Flush(vsl, 1); - assert(VSL_END(vsl->wlp, tl) < vsl->wle); + assert(VSL_END(vsl->wlp, len) < vsl->wle); p = VSL_DATA(vsl->wlp); - for (ll = 0; ll < len; ll++) { - assert(snprintf(p, 3, "%02x", *pp) == 2); - pp++; - p += 2; - } - if (suff) - *p++ = '-'; - *p = '\0'; - vsl->wlp = vsl_hdr(tag, vsl->wlp, tl, vsl->wid); + memcpy(p, ptr, len); + vsl->wlp = vsl_hdr(tag, vsl->wlp, len, vsl->wid); assert(vsl->wlp < vsl->wle); vsl->wlr++; + + if (DO_DEBUG(DBG_SYNCVSL)) + VSL_Flush(vsl, 0); } /*-------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 6062a3a..c2a1b0a 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -19,6 +19,8 @@ varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "param.set vsl_mask +H2RxHdr,+H2RxBody,+H2TxHdr,+H2TxBody" +process p1 {exec varnishlog -n ${v1_name} -g raw -w ${tmpdir}/vlog -A} -start + client c1 { txpri expect_close @@ -56,3 +58,8 @@ varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 varnish v1 -expect MEMPOOL.sess1.live == 0 + +process p1 -stop +# shell {cat ${tmpdir}/vlog} +shell -match {1001 H2TxHdr c \[000006040000000000\]} \ + {cat ${tmpdir}/vlog} diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 1ba5e7c..12e4244 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -214,6 +214,7 @@ varnishlog_thread(void *priv) unsigned len; const char *tagname, *data; int type, i, opt; + struct vsb *vsb = NULL; CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC); @@ -258,8 +259,19 @@ varnishlog_thread(void *priv) 'b' : '-'; data = VSL_CDATA(c->rec.ptr); v->vsl_tag_count[tag]++; - vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", - vxid, tagname, type, (int)len, data); + if (VSL_tagflags[tag] & SLT_F_BINARY) { + if (vsb == NULL) + vsb = VSB_new_auto(); + VSB_clear(vsb); + VSB_quote(vsb, data, len, VSB_QUOTE_HEX); + AZ(VSB_finish(vsb)); + /* +2 to skip "0x" */ + vtc_log(v->vl, 4, "vsl| %10u %-15s %c [%s]", + vxid, tagname, type, VSB_data(vsb) + 2); + } else { + vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", + vxid, tagname, type, (int)len, data); + } } if (i == 0) { /* Nothing to do but wait */ @@ -282,6 +294,8 @@ varnishlog_thread(void *priv) if (c) VSL_DeleteCursor(c); VSL_Delete(vsl); + if (vsb != NULL) + VSB_destroy(&vsb); return (NULL); } diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index c80c135..728011b 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -545,19 +545,19 @@ SLTM(BackendStart, 0, "Backend request start", "\n" ) -SLTM(H2RxHdr, 0, "Received HTTP2 frame header", +SLTM(H2RxHdr, SLT_F_BINARY, "Received HTTP2 frame header", "Binary data" ) -SLTM(H2RxBody, 0, "Received HTTP2 frame body", +SLTM(H2RxBody, SLT_F_BINARY, "Received HTTP2 frame body", "Binary data" ) -SLTM(H2TxHdr, 0, "Transmitted HTTP2 frame header", +SLTM(H2TxHdr, SLT_F_BINARY, "Transmitted HTTP2 frame header", "Binary data" ) -SLTM(H2TxBody, 0, "Transmitted HTTP2 frame body", +SLTM(H2TxBody, SLT_F_BINARY, "Transmitted HTTP2 frame body", "Binary data" ) diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index 57986cd..589a966 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -98,5 +98,6 @@ enum VSL_tag_e { /* VSL tag flags */ #define SLT_F_UNUSED (1 << 0) #define SLT_F_UNSAFE (1 << 1) +#define SLT_F_BINARY (1 << 2) #endif /* VAPI_VSL_INT_H_INCLUDED */ diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index af23e92..970c175 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -223,8 +223,39 @@ static const char * const VSL_transactions[VSL_t__MAX] = { return (-5); \ } while (0) -int -VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) +static int +vsl_print_unsafe(FILE *fo, unsigned len, const char *data) +{ + + VSL_PRINT(fo, "\""); + while (len-- > 0) { + if (*data >= ' ' && *data <= '~') + VSL_PRINT(fo, "%c", *data); + else + VSL_PRINT(fo, "%%%02x", (unsigned char)*data); + data++; + } + VSL_PRINT(fo, "\"\n"); + return (0); +} + + +static int +vsl_print_binary(FILE *fo, unsigned len, const char *data) +{ + + VSL_PRINT(fo, "["); + while (len-- > 0) { + VSL_PRINT(fo, "%02x", (unsigned char)*data); + data++; + } + VSL_PRINT(fo, "]\n"); + return (0); +} + +static int +vsl_print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo, + int terse) { enum VSL_tag_e tag; uint32_t vxid; @@ -244,57 +275,34 @@ VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) 'b' : '-'; data = VSL_CDATA(c->rec.ptr); - if (VSL_tagflags[tag] & SLT_F_UNSAFE) { - VSL_PRINT(fo, "%10u %-14s %c \"", vxid, VSL_tags[tag], type); - while (len-- > 0) { - if (len == 0 && tag == SLT_Debug && *data == '\0') - break; - if (*data >= ' ' && *data <= '~') - VSL_PRINT(fo, "%c", *data); - else - VSL_PRINT(fo, "%%%02x", (unsigned char)*data); - data++; - } - VSL_PRINT(fo, "\"\n"); - } else - VSL_PRINT(fo, "%10u %-14s %c %.*s\n", vxid, VSL_tags[tag], - type, (int)len, data); + if (!terse) + VSL_PRINT(fo, "%10u ", vxid); + VSL_PRINT(fo, "%-14s ", VSL_tags[tag]); + if (!terse) + VSL_PRINT(fo, "%c ", type); + + if (VSL_tagflags[tag] & SLT_F_UNSAFE) + (void)vsl_print_unsafe(fo, len, data); + else if (VSL_tagflags[tag] & SLT_F_BINARY) + (void)vsl_print_binary(fo, len, data); + else + VSL_PRINT(fo, "%.*s\n", (int)len, data); return (0); } int -VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) +VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) { - enum VSL_tag_e tag; - unsigned len; - const char *data; - CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); - if (c == NULL || c->rec.ptr == NULL) - return (0); - if (fo == NULL) - fo = stdout; - tag = VSL_TAG(c->rec.ptr); - len = VSL_LEN(c->rec.ptr); - data = VSL_CDATA(c->rec.ptr); + return (vsl_print(vsl, c, fo, 0)); +} - if (VSL_tagflags[tag] & SLT_F_UNSAFE) { - VSL_PRINT(fo, "%-14s \"", VSL_tags[tag]); - while (len-- > 0) { - if (len == 0 && tag == SLT_Debug && *data == '\0') - break; - if (*data >= ' ' && *data <= '~') - VSL_PRINT(fo, "%c", *data); - else - VSL_PRINT(fo, "%%%02x", (unsigned char)*data); - data++; - } - VSL_PRINT(fo, "\"\n"); - } else - VSL_PRINT(fo, "%-14s %.*s\n", VSL_tags[tag], (int)len, data); +int +VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo) +{ - return (0); + return (vsl_print(vsl, c, fo, 1)); } int From phk at FreeBSD.org Mon Mar 5 10:09:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 10:09:08 +0000 (UTC) Subject: [master] 63807dd Make the utils ignore binary and unsafe VSL records. Message-ID: <20180305100908.9760EB4AD5@lists.varnish-cache.org> commit 63807dd22725653e23c20071a9eb958642413f36 Author: Poul-Henning Kamp Date: Mon Mar 5 10:07:30 2018 +0000 Make the utils ignore binary and unsafe VSL records. This includes the Debug record, if that becomes a problem we will deal with it. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 3ba61db..fdc773a 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -258,6 +258,9 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], /* get the value we want and register if it's a hit */ tag = VSL_TAG(tr->c->rec.ptr); + if (VSL_tagflags[tag]) + continue; + switch (tag) { case SLT_Hit: hit = 1; @@ -549,6 +552,11 @@ main(int argc, char **argv) VUT_Error(vut, 1, "-P: '%s' is not a valid tag name", optarg); + if (VSL_tagflags[match_tag]) + VUT_Error(vut, 1, + "-P: '%s' is an unsafe or binary record", + optarg); + cli_p.name = "custom"; cli_p.tag = match_tag; profile = NULL; diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 6870ab3..5573407 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -570,6 +570,9 @@ addf_vsl(enum VSL_tag_e tag, long i, const char *prefix) ALLOC_OBJ(w, VSL_WATCH_MAGIC); AN(w); + if (VSL_tagflags[tag]) + VUT_Error(vut, 1, "Tag %s can contain control characters", + VSL_tags[tag]); w->tag = tag; assert(i <= INT_MAX); w->idx = i; @@ -961,6 +964,9 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], skip = 0; while (skip == 0 && 1 == VSL_Next(t->c)) { tag = VSL_TAG(t->c->rec.ptr); + if (VSL_tagflags[tag]) + continue; + b = VSL_CDATA(t->c->rec.ptr); e = b + VSL_LEN(t->c->rec.ptr); while (e > b && e[-1] == '\0') diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index c687376..61f0e4d 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -131,9 +131,11 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], for (tr = pt[0]; tr != NULL; tr = *++pt) { while ((1 == VSL_Next(tr->c))) { + tag = VSL_TAG(tr->c->rec.ptr); + if (VSL_tagflags[tag]) + continue; if (!VSL_Match(vsl, tr->c)) continue; - tag = VSL_TAG(tr->c->rec.ptr); b = VSL_CDATA(tr->c->rec.ptr); e = b + VSL_LEN(tr->c->rec.ptr); u = 0; diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h index b2d7f1c..f91c661 100644 --- a/include/vapi/vsl.h +++ b/include/vapi/vsl.h @@ -163,6 +163,9 @@ extern const unsigned VSL_tagflags[SLT__MAX]; * Tag flags array. * Use these macros with VSL_tagflags (included from vsl_int.h): * + * VSL_tagflags[tag] & SLT_F_BINARY + * Non-zero if the payload is binary data + * * VSL_tagflags[tag] & SLT_F_UNSAFE * Non-zero if the payload with this tag may include * non-printable characters From phk at FreeBSD.org Mon Mar 5 11:34:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 11:34:07 +0000 (UTC) Subject: [master] 0d370c8 Give varnishlog a chance to start on slow systems Message-ID: <20180305113407.5B69061274@lists.varnish-cache.org> commit 0d370c83db6d1dbc671994aa2a6afd71315317c6 Author: Poul-Henning Kamp Date: Mon Mar 5 11:32:33 2018 +0000 Give varnishlog a chance to start on slow systems diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index c2a1b0a..c116466 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -21,6 +21,8 @@ varnish v1 -cliok "param.set vsl_mask +H2RxHdr,+H2RxBody,+H2TxHdr,+H2TxBody" process p1 {exec varnishlog -n ${v1_name} -g raw -w ${tmpdir}/vlog -A} -start +delay 1 + client c1 { txpri expect_close From phk at FreeBSD.org Mon Mar 5 11:38:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 11:38:07 +0000 (UTC) Subject: [master] 81f4633 Better yet, wait until varnishlog is started. Message-ID: <20180305113807.2BD31613D1@lists.varnish-cache.org> commit 81f4633e57cb45de9947ca51451758ad45a5f290 Author: Poul-Henning Kamp Date: Mon Mar 5 11:37:28 2018 +0000 Better yet, wait until varnishlog is started. diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index c116466..cb7dc75 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -21,7 +21,7 @@ varnish v1 -cliok "param.set vsl_mask +H2RxHdr,+H2RxBody,+H2TxHdr,+H2TxBody" process p1 {exec varnishlog -n ${v1_name} -g raw -w ${tmpdir}/vlog -A} -start -delay 1 +shell {while ! test -s ${tmpdir}/vlog ; do sleep 1 ; done} client c1 { txpri From dridi.boukelmoune at gmail.com Mon Mar 5 12:22:08 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Mar 2018 12:22:08 +0000 (UTC) Subject: [master] 0d1ab25 Add Obj* records to the default vsl_mask Message-ID: <20180305122208.3186A6216C@lists.varnish-cache.org> commit 0d1ab2597f2bd4ccf5d66ab0ddfc4132b9e594d5 Author: Dridi Boukelmoune Date: Tue Feb 27 20:54:38 2018 +0100 Add Obj* records to the default vsl_mask They add up to a considerable chunk of the VSL traffic, especially when cookies are involved, while providing little value when troubleshooting. In an effort to reduce VSL cycles on heavy workloads I found these to be good candidates for several reasons: - obj.* fields are immutable in VCL - after a hit, we initially get the same Resp* records in vcl_deliver - before vcl_backend_response returns, we can infer them from Beresp* The main difference would be ObjHeader:X-Varnish, but it would be the same as BerespHeader:X-Varnish, and be part of the RespHeader counterpart in the case of a hit. That, and cache-related headers adjusted to the current time. One case where they wouldn't appear is for example on a hit followed by a restart. I have yet to find a use case where I'd need to query them with varnishlog or some other VUT. diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index ad90a12..a6d7b92 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -131,6 +131,10 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg) (void)bit(mgt_param.vsl_mask, SLT_H2TxHdr, BSET); (void)bit(mgt_param.vsl_mask, SLT_H2RxBody, BSET); (void)bit(mgt_param.vsl_mask, SLT_H2RxHdr, BSET); + (void)bit(mgt_param.vsl_mask, SLT_ObjHeader, BSET); + (void)bit(mgt_param.vsl_mask, SLT_ObjProtocol, BSET); + (void)bit(mgt_param.vsl_mask, SLT_ObjReason, BSET); + (void)bit(mgt_param.vsl_mask, SLT_ObjStatus, BSET); } else { return (bit_tweak(vsb, mgt_param.vsl_mask, SLT__Reserved, arg, VSL_tags, From nils.goroll at uplex.de Mon Mar 5 12:51:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 12:51:08 +0000 (UTC) Subject: [master] 3c5ed76 Pass delivery abandoned does not qualify as an error Message-ID: <20180305125109.0151962A2D@lists.varnish-cache.org> commit 3c5ed76fea33c98dc5464c8e8ff902172bcd8dbb Author: Nils Goroll Date: Fri Oct 6 12:58:24 2017 +0200 Pass delivery abandoned does not qualify as an error ... so log it under the Debug tag. FetchErrors should be actual errors which can be addressed. In this case, nothing is wrong in any way, the fact that we abort a fetch if we don't need the body is a varnish internal optimization (which makes sense, but comes at the cost of closing a connection). Merges #2450 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 581a707..f64cfb4 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -449,8 +449,8 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) * objects to be created. */ AN(vfc->oc->flags & OC_F_PASS); - VSLb(wrk->vsl, SLT_FetchError, - "Pass delivery abandoned"); + VSLb(wrk->vsl, SLT_Debug, + "Fetch: Pass delivery abandoned"); bo->htc->doclose = SC_RX_BODY; break; } From nils.goroll at uplex.de Mon Mar 5 13:36:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 13:36:08 +0000 (UTC) Subject: [master] 373388b fix typo Message-ID: <20180305133608.B9BD163832@lists.varnish-cache.org> commit 373388bb5abe26941e34871cb0b20ad5cd04c9ab Author: Nils Goroll Date: Mon Mar 5 14:35:30 2018 +0100 fix typo diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 853abb7..e866c80 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -357,7 +357,7 @@ class prototype(object): self.bname = wl.pop(0) if not re.match("^[a-zA-Z.][a-zA-Z0-9_]*$", self.bname): - err("%s(): Illegal name\n" % self.nname, warn=False) + err("%s(): Illegal name\n" % self.bname, warn=False) self.name = prefix + self.bname if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', self.cname()): From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] 020c5a6 Split pushing of VCLs and starting the acceptor Message-ID: <20180305135107.37BBD63D34@lists.varnish-cache.org> commit 020c5a620d9dd8f8b82c5564dd8c8ee2d1dd8b19 Author: Martin Blix Grydeland Date: Tue Feb 27 15:46:17 2018 +0100 Split pushing of VCLs and starting the acceptor This is so that errors in either will be reported under the right heading. diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 81d11f5..7575de2 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -213,7 +213,7 @@ char *mgt_VccCompile(struct cli *, struct vclprog *, const char *vclname, void mgt_vcl_init(void); void mgt_vcl_startup(struct cli *, const char *vclsrc, const char *origin, const char *vclname, int Cflag); -int mgt_push_vcls_and_start(struct cli *, unsigned *status, char **p); +int mgt_push_vcls(struct cli *, unsigned *status, char **p); void mgt_vcl_export_labels(struct vcc *); int mgt_has_vcl(void); void mgt_vcl_depends(struct vclprog *vp1, const char *name); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 8b70f1a..0c8660c 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -395,15 +395,25 @@ mgt_launch_child(struct cli *cli) mgt_cli_start_child(child_cli_in, child_cli_out); child_pid = pid; - if (mgt_push_vcls_and_start(cli, &u, &p)) { + child_state = CH_RUNNING; + + if (mgt_push_vcls(cli, &u, &p)) { VCLI_SetResult(cli, u); MGT_Complain(C_ERR, "Child (%jd) Pushing vcls failed:\n%s", (intmax_t)child_pid, p); free(p); - child_state = CH_RUNNING; MCH_Stop_Child(); - } else - child_state = CH_RUNNING; + return; + } + + if (mgt_cli_askchild(&u, &p, "start\n")) { + VCLI_SetResult(cli, u); + MGT_Complain(C_ERR, "Child (%jd) Acceptor start failed:\n%s", + (intmax_t)child_pid, p); + free(p); + MCH_Stop_Child(); + return; + } } /*===================================================================== diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index ae31eed..5974b58 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -545,7 +545,7 @@ mgt_vcl_export_labels(struct vcc *vcc) /*--------------------------------------------------------------------*/ int -mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) +mgt_push_vcls(struct cli *cli, unsigned *status, char **p) { struct vclprog *vp; struct vcldep *vd; @@ -594,10 +594,6 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) return (1); free(*p); *p = NULL; - if (mgt_cli_askchild(status, p, "start\n")) - return (1); - free(*p); - *p = NULL; return (0); } From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] f80c0a1 Move the reopening of listen sockets down to after handling the child exit. Message-ID: <20180305135107.4E56D63D37@lists.varnish-cache.org> commit f80c0a1e22419a9b195b0af5f6c31576040fc758 Author: Martin Blix Grydeland Date: Wed Feb 28 11:30:02 2018 +0100 Move the reopening of listen sockets down to after handling the child exit. This makes sure that any panic message from the child is handled and shown before continuing (or possibly failing). diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 0c8660c..032b387 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -492,17 +492,6 @@ mgt_reap_child(void) fprintf(stderr, "WAIT 0x%jd\n", (intmax_t)r); assert(r == child_pid); - /* - * XXX exit mgr if we fail even with retries? - * number of retries? interval? - */ - for (i = 0; i < 3; i++) { - if (MAC_reopen_sockets() == 0) - break; - /* error already logged */ - (void)sleep(1); - } - VSB_printf(vsb, "Child (%jd) %s", (intmax_t)r, status ? "died" : "ended"); if (WIFEXITED(status) && WEXITSTATUS(status)) { @@ -549,6 +538,17 @@ mgt_reap_child(void) MGT_Complain(C_DEBUG, "Child cleanup complete"); + /* + * XXX exit mgr if we fail even with retries? + * number of retries? interval? + */ + for (i = 0; i < 3; i++) { + if (MAC_reopen_sockets() == 0) + break; + /* error already logged */ + (void)sleep(1); + } + if (child_state == CH_DIED && mgt_param.auto_restart) mgt_launch_child(NULL); else if (child_state == CH_DIED) From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] 6e8fad2 Exit the management process if we fail to reopen the listening sockets. Message-ID: <20180305135107.6B20F63D3B@lists.varnish-cache.org> commit 6e8fad2c06db868ec8cbe27d2447c6a3c19d3418 Author: Martin Blix Grydeland Date: Wed Feb 28 11:33:48 2018 +0100 Exit the management process if we fail to reopen the listening sockets. As the code was, we would run into an assert immediately upon starting the child when we handed it a socket==-1. Better to exit with an error message. To keep running with a degraded set might catch people by surprise, so the only option left is to exit the process. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 032b387..88acd0e 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -538,16 +538,20 @@ mgt_reap_child(void) MGT_Complain(C_DEBUG, "Child cleanup complete"); - /* - * XXX exit mgr if we fail even with retries? - * number of retries? interval? - */ + /* XXX number of retries? interval? */ for (i = 0; i < 3; i++) { if (MAC_reopen_sockets() == 0) break; /* error already logged */ (void)sleep(1); } + if (i == 3) { + /* We failed to reopen our listening sockets. No choice + * but to exit. */ + MGT_Complain(C_ERR, + "Could not reopen listening sockets. Exiting."); + exit(1); + } if (child_state == CH_DIED && mgt_param.auto_restart) mgt_launch_child(NULL); From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] 4a095b7 Move the cache acceptor socket preparation to the CLI call Message-ID: <20180305135107.8BEC963D3F@lists.varnish-cache.org> commit 4a095b751f656ef5c28b0fb348a2a3ad64cbd4e0 Author: Martin Blix Grydeland Date: Wed Feb 28 11:52:14 2018 +0100 Move the cache acceptor socket preparation to the CLI call This moves the first time initialization of the listening sockets to the CLI "start" command handler. This enables errors to be reported to the management process if we were unable to comply. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index c41194d..398998b 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -586,38 +586,8 @@ vca_acct(void *arg) (void)arg; t0 = VTIM_real(); - vca_periodic(t0); - (void)vca_tcp_opt_init(); - - AZ(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); - if (ls->sock == -2) - continue; // VCA_Shutdown - assert (ls->sock > 0); // We know where stdin is - if (cache_param->tcp_fastopen) { - int i; - i = VTCP_fastopen(ls->sock, cache_param->listen_depth); - if (i) - VSL(SLT_Error, ls->sock, - "Kernel TCP Fast Open: sock=%d, ret=%d %s", - ls->sock, i, strerror(errno)); - } - AZ(listen(ls->sock, cache_param->listen_depth)); - vca_tcp_opt_set(ls, 1); - if (cache_param->accept_filter) { - int i; - i = VTCP_filter_http(ls->sock); - if (i) - VSL(SLT_Error, ls->sock, - "Kernel filtering: sock=%d, ret=%d %s", - ls->sock, i, strerror(errno)); - } - } - AZ(pthread_mutex_unlock(&shut_mtx)); - need_test = 1; pool_accepting = 1; while (1) { @@ -642,11 +612,39 @@ vca_acct(void *arg) static void v_matchproto_(cli_func_t) ccf_start(struct cli *cli, const char * const *av, void *priv) { + struct listen_sock *ls; (void)cli; (void)av; (void)priv; + (void)vca_tcp_opt_init(); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); + assert (ls->sock > 0); // We know where stdin is + if (cache_param->tcp_fastopen) { + int i; + i = VTCP_fastopen(ls->sock, cache_param->listen_depth); + if (i) + VSL(SLT_Error, ls->sock, + "Kernel TCP Fast Open: sock=%d, ret=%d %s", + ls->sock, i, strerror(errno)); + } + AZ(listen(ls->sock, cache_param->listen_depth)); + vca_tcp_opt_set(ls, 1); + if (cache_param->accept_filter) { + int i; + i = VTCP_filter_http(ls->sock); + if (i) + VSL(SLT_Error, ls->sock, + "Kernel filtering: sock=%d, ret=%d %s", + ls->sock, i, strerror(errno)); + } + } + + need_test = 1; + AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); } From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] f9ea7e3 Fix the logging of warnings during applying tcp_fastopen and accept_filter params Message-ID: <20180305135107.A877263D43@lists.varnish-cache.org> commit f9ea7e34a1fba0621ffa4c0036dafb8282a118ec Author: Martin Blix Grydeland Date: Wed Feb 28 11:56:12 2018 +0100 Fix the logging of warnings during applying tcp_fastopen and accept_filter params These were logged with a vxid==socket, which is wrong (leftover from old logging). They are now logged with vxid==0, which is for non-transactional messages. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 398998b..a617581 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -627,7 +627,7 @@ ccf_start(struct cli *cli, const char * const *av, void *priv) int i; i = VTCP_fastopen(ls->sock, cache_param->listen_depth); if (i) - VSL(SLT_Error, ls->sock, + VSL(SLT_Error, 0, "Kernel TCP Fast Open: sock=%d, ret=%d %s", ls->sock, i, strerror(errno)); } @@ -637,7 +637,7 @@ ccf_start(struct cli *cli, const char * const *av, void *priv) int i; i = VTCP_filter_http(ls->sock); if (i) - VSL(SLT_Error, ls->sock, + VSL(SLT_Error, 0, "Kernel filtering: sock=%d, ret=%d %s", ls->sock, i, strerror(errno)); } From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] ab550ba Report errors on listen() to the management process. Message-ID: <20180305135107.C4CE963D47@lists.varnish-cache.org> commit ab550bae3bccda66786e596a0746907655eb28e6 Author: Martin Blix Grydeland Date: Wed Feb 28 11:58:40 2018 +0100 Report errors on listen() to the management process. If the listen() system call were to report problems, they are now reported back to the management process, failing the child startup (instead of asserting). This can happen on many widely used Linux kernels if the listen socket already has another process listening on that port with the SO_REUSEPORT flag set. Newer Linux kernels will report the problem at bind() time (causing error to be reported upon management process startup). Fixes: #2551 diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index a617581..6973d01 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -631,7 +631,12 @@ ccf_start(struct cli *cli, const char * const *av, void *priv) "Kernel TCP Fast Open: sock=%d, ret=%d %s", ls->sock, i, strerror(errno)); } - AZ(listen(ls->sock, cache_param->listen_depth)); + if (listen(ls->sock, cache_param->listen_depth)) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Listen failed on socket '%s': %s", + ls->endpoint, strerror(errno)); + return; + } vca_tcp_opt_set(ls, 1); if (cache_param->accept_filter) { int i; From martin at varnish-software.com Mon Mar 5 13:51:07 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 5 Mar 2018 13:51:07 +0000 (UTC) Subject: [master] ee1d34d Changelog entry for #2551 Message-ID: <20180305135107.DDD1663D4F@lists.varnish-cache.org> commit ee1d34dea8ecb5576d0f2740e807ad4d9e67b3c6 Author: Martin Blix Grydeland Date: Wed Feb 28 13:50:55 2018 +0100 Changelog entry for #2551 diff --git a/doc/changes.rst b/doc/changes.rst index 07791b5..7468107 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -134,6 +134,12 @@ Fixed bugs which may influence VCL behaviour Fixed bugs ---------- +* Problems during late socket initialization performed by the Varnish + child process can now be reported back to the management process with an + error message. (2551_) + +.. _2551: https://github.com/varnishcache/varnish-cache/issues/2551 + **TODO** ================================ From nils.goroll at uplex.de Mon Mar 5 14:52:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 14:52:09 +0000 (UTC) Subject: [master] a6da61e fix vsc level filtering Message-ID: <20180305145209.8BDAE650A3@lists.varnish-cache.org> commit a6da61ede92da732d6215058f5a49c1c1d51b761 Author: Nils Goroll Date: Mon Mar 5 15:46:16 2018 +0100 fix vsc level filtering By imposing a fixed ordering on our levels structs, we ensure that pointers to them are also ordered such that comparing two of them will compare as level ordering. Fixes #2587 diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 7df0858..0c243fe 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -96,14 +96,14 @@ struct vsc { * Build the static level, type and point descriptions */ -#define VSC_LEVEL_F(v,l,e,d) \ - static const struct VSC_level_desc level_##v = {#v, l, e, d}; +enum vsc_levels { +#define VSC_LEVEL_F(v,l,e,d) v, #include "tbl/vsc_levels.h" +}; -static const struct VSC_level_desc * const levels[] = { -#define VSC_LEVEL_F(v,l,e,d) &level_##v, +static const struct VSC_level_desc levels[] = { +#define VSC_LEVEL_F(v,l,e,d) [v] = {#v, l, e, d}, #include "tbl/vsc_levels.h" -#undef VSC_LEVEL_F }; static const ssize_t nlevels = sizeof(levels)/sizeof(*levels); @@ -266,11 +266,11 @@ vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg, assert(vt->type == VJSN_STRING); if (!strcmp(vt->value, "info")) { - point->point.level = &level_info; + point->point.level = &levels[info]; } else if (!strcmp(vt->value, "diag")) { - point->point.level = &level_diag; + point->point.level = &levels[diag]; } else if (!strcmp(vt->value, "debug")) { - point->point.level = &level_debug; + point->point.level = &levels[debug]; } else { WRONG("Illegal level"); } @@ -490,9 +490,9 @@ VSC_ChangeLevel(const struct VSC_level_desc *old, int chg) int i; if (old == NULL) - old = levels[0]; + old = &levels[0]; for (i = 0; i < nlevels; i++) - if (old == levels[i]) + if (old == &levels[i]) break; if (i == nlevels) i = 0; @@ -502,7 +502,7 @@ VSC_ChangeLevel(const struct VSC_level_desc *old, int chg) i = nlevels - 1; if (i < 0) i = 0; - return (levels[i]); + return (&levels[i]); } /*--------------------------------------------------------------------*/ From nils.goroll at uplex.de Mon Mar 5 16:51:06 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 16:51:06 +0000 (UTC) Subject: [master] baea879 gc unused variable b(ranch) Message-ID: <20180305165106.7600591409@lists.varnish-cache.org> commit baea879811cc4ee8e52385de5d85c8596ac7b529 Author: Nils Goroll Date: Mon Mar 5 17:21:58 2018 +0100 gc unused variable b(ranch) diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index d744cf3..efb178a 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -827,13 +827,7 @@ if os.path.isdir(os.path.join(srcroot, ".git")): " show -s --pretty=format:%H" ], shell=True, universal_newlines=True) v = v.strip() - b = subprocess.check_output([ - "git --git-dir=" + os.path.join(srcroot, ".git") + - " rev-parse --abbrev-ref HEAD" - ], shell=True, universal_newlines=True) - b = b.strip() else: - b = "NOGIT" v = "NOGIT" vcsfn = os.path.join(srcroot, "include", "vcs_version.h") From nils.goroll at uplex.de Mon Mar 5 16:51:06 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 16:51:06 +0000 (UTC) Subject: [master] d731f47 rewrite vmod_abi.h / vcs_version.h only if git hash changed Message-ID: <20180305165106.8F6B09140C@lists.varnish-cache.org> commit d731f4732f4b33784b7a92dddadf0ba1ae9d34da Author: Nils Goroll Date: Mon Mar 5 17:42:36 2018 +0100 rewrite vmod_abi.h / vcs_version.h only if git hash changed This got lost with a29fca70f7ccc75964bcfffb8c8ab1617fcf2bba diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index efb178a..dfe28cd 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -837,8 +837,11 @@ try: except IOError: i = "" -if i != "/* " + v + " */": +ident = "/* " + v + " */\n" + +if i != ident: fo = open(vcsfn, "w") + fo.write(ident) file_header(fo) fo.write('#define VCS_Version "%s"\n' % v) fo.close() From nils.goroll at uplex.de Mon Mar 5 17:38:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 17:38:09 +0000 (UTC) Subject: [master] 9305a2d pull out vcs_version.h / vmod_abi.h generation to own file again Message-ID: <20180305173809.13D0792299@lists.varnish-cache.org> commit 9305a2d4c4734eaf93f5ecec2a823f5d493d0be8 Author: Nils Goroll Date: Mon Mar 5 18:12:21 2018 +0100 pull out vcs_version.h / vmod_abi.h generation to own file again This basically reverts a29fca70f7ccc75964bcfffb8c8ab1617fcf2bba, except that we are using python instead of make-inlined shell code to do the work. Reason: vcs_version.h needs to be up-to-date under all circumstances, otherwise we will end up with wrong version information in binary builds, which would divert developer resources into unproductive confusion compensation. With an all-in-one generate.py, we basically have the choice between: - running it with each build, which breaks incremental builds as generate.py rewrites central includes like vcl.h - adding version information to all generate.py-build files or similar mechanisms to avoid re-writing them unless they have actually changed. - contradicting the argument given above I think that, unless there are strong reasons for a single generate.py, avoiding these issues by splitting functions is the best option. diff --git a/include/Makefile.am b/include/Makefile.am index 9ddb077..1b13126 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -101,7 +101,7 @@ nobase_noinst_HEADERS = \ vus.h ## keep in sync with lib/libvcc/Makefile.am -vmod_abi.h: \ +vcl.h: \ $(top_srcdir)/lib/libvcc/generate.py \ $(top_srcdir)/include/vdef.h \ $(top_srcdir)/include/vrt.h \ @@ -114,15 +114,31 @@ GEN_H = \ tbl/vrt_stv_var.h \ tbl/vcl_returns.h \ tbl/vcc_types.h \ - vrt_obj.h \ - vcl.h \ - vcs_version.h + vrt_obj.h + +$(GEN_H): vcl.h + +GENERATED_H = vcl.h $(GEN_H) -$(GEN_H): vmod_abi.h +## vcs_version.h / vmod_abi.h need to be up-to-date with every build +## except when building from a distribution -GENERATED_H = vmod_abi.h $(GEN_H) +vcs_version.h: + if test -d $(top_srcdir)/.git ; then \ + @PYTHON@ $(srcdir)/generate.py \ + $(top_srcdir) $(top_builddir) ; \ + fi -BUILT_SOURCES = $(GENERATED_H) +vmod_abi.h: vcs_version.h + +.PHONY: vcs_version.h + +## + +BUILT_SOURCES = \ + $(GENERATED_H) \ + vcs_version.h \ + vmod_abi.h MAINTAINERCLEANFILES = $(GENERATED_H) diff --git a/include/generate.py b/include/generate.py new file mode 100755 index 0000000..f6cd339 --- /dev/null +++ b/include/generate.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Copyright (c) 2006 Verdens Gang AS +# Copyright (c) 2006-2015 Varnish Software AS +# All rights reserved. +# +# Author: Poul-Henning Kamp +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# Generate vcs_version.h vmod_abi.h + +from __future__ import print_function + +import subprocess +import collections +import os +import sys + +srcroot = "../.." +buildroot = "../.." +if len(sys.argv) == 3: + srcroot = sys.argv[1] + buildroot = sys.argv[2] +elif len(sys.argv) != 1: + print("Two arguments or none") + exit(2) + +####################################################################### +def file_header(fo): + fo.write("""/* + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit and run lib/libvcc/generate.py instead. + */ + +""") + +####################################################################### + +if os.path.isdir(os.path.join(srcroot, ".git")): + v = subprocess.check_output([ + "git --git-dir=" + os.path.join(srcroot, ".git") + + " show -s --pretty=format:%H" + ], shell=True, universal_newlines=True) + v = v.strip() +else: + v = "NOGIT" + +vcsfn = os.path.join(srcroot, "include", "vcs_version.h") + +try: + i = open(vcsfn).readline() +except IOError: + i = "" + +ident = "/* " + v + " */\n" + +if i != ident: + fo = open(vcsfn, "w") + fo.write(ident) + file_header(fo) + fo.write('#define VCS_Version "%s"\n' % v) + fo.close() + + for i in open(os.path.join(buildroot, "Makefile")): + if i[:14] == "PACKAGE_STRING": + break + i = i.split("=")[1].strip() + + fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w") + file_header(fo) + fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v)) + fo.close() diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index dfe28cd..2d556a7 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -818,40 +818,3 @@ for i in stv_variables: fo.write("#undef VRTSTVVAR\n") lint_end(fo) fo.close() - -####################################################################### - -if os.path.isdir(os.path.join(srcroot, ".git")): - v = subprocess.check_output([ - "git --git-dir=" + os.path.join(srcroot, ".git") + - " show -s --pretty=format:%H" - ], shell=True, universal_newlines=True) - v = v.strip() -else: - v = "NOGIT" - -vcsfn = os.path.join(srcroot, "include", "vcs_version.h") - -try: - i = open(vcsfn).readline() -except IOError: - i = "" - -ident = "/* " + v + " */\n" - -if i != ident: - fo = open(vcsfn, "w") - fo.write(ident) - file_header(fo) - fo.write('#define VCS_Version "%s"\n' % v) - fo.close() - - for i in open(os.path.join(buildroot, "Makefile")): - if i[:14] == "PACKAGE_STRING": - break - i = i.split("=")[1].strip() - - fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w") - file_header(fo) - fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v)) - fo.close() From dridi at varni.sh Mon Mar 5 18:32:37 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 5 Mar 2018 19:32:37 +0100 Subject: [master] 9305a2d pull out vcs_version.h / vmod_abi.h generation to own file again In-Reply-To: <20180305173809.13D0792299@lists.varnish-cache.org> References: <20180305173809.13D0792299@lists.varnish-cache.org> Message-ID: On Mon, Mar 5, 2018 at 6:38 PM, Nils Goroll wrote: > > commit 9305a2d4c4734eaf93f5ecec2a823f5d493d0be8 > Author: Nils Goroll > Date: Mon Mar 5 18:12:21 2018 +0100 > > pull out vcs_version.h / vmod_abi.h generation to own file again > > This basically reverts a29fca70f7ccc75964bcfffb8c8ab1617fcf2bba, > except that we are using python instead of make-inlined shell code to > do the work. > > Reason: vcs_version.h needs to be up-to-date under all circumstances, > otherwise we will end up with wrong version information in binary > builds, which would divert developer resources into unproductive > confusion compensation. > > With an all-in-one generate.py, we basically have the choice between: > > - running it with each build, which breaks incremental builds as > generate.py rewrites central includes like vcl.h > > - adding version information to all generate.py-build files or similar > mechanisms to avoid re-writing them unless they have actually > changed. > > - contradicting the argument given above > > I think that, unless there are strong reasons for a single > generate.py, avoiding these issues by splitting functions is the best > option. I'm afraid this isn't the last patch for this: https://travis-ci.org/Dridi/libvmod-querystring/jobs/349421603 This is a cron job building vmod-querystring against current master once a week, and today it fails with: version.c:37:10: fatal error: 'vcs_version.h' file not found #include "vcs_version.h" ^~~~~~~~~~~~~~~ 1 error generated. FYI, this is a source tree downloaded from github (same as git-archive) and should result in NOGIT since this is not a dist archive. Dridi From nils.goroll at uplex.de Mon Mar 5 19:03:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 19:03:10 +0000 (UTC) Subject: [master] 15cc069 bring back beresp.storage_hint for vcl 4.0 Message-ID: <20180305190310.92D7E93BAE@lists.varnish-cache.org> commit 15cc069bdc4ec7ff1925848b8cd8bfb2e9edeece Author: Nils Goroll Date: Mon Mar 5 20:00:04 2018 +0100 bring back beresp.storage_hint for vcl 4.0 see doc/changes.rst for details Closes #2509 diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 1b071c9..d781ce7 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -365,6 +365,53 @@ VRT_l_beresp_storage(VRT_CTX, VCL_STEVEDORE stv) ctx->bo->storage = stv; } +/*-------------------------------------------------------------------- + * VCL <= 4.0 ONLY + */ + +#include "storage/storage.h" + +const char * +VRT_r_beresp_storage_hint(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + if (ctx->bo->storage == NULL) + return (NULL); + CHECK_OBJ_NOTNULL(ctx->bo->storage, STEVEDORE_MAGIC); + return (ctx->bo->storage->vclname); +} + +void +VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...) +{ + const char *p; + va_list ap; + uintptr_t sn; + VCL_STEVEDORE stv; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + + sn = WS_Snapshot(ctx->ws); + va_start(ap, str); + p = VRT_String(ctx->ws, NULL, str, ap); + va_end(ap); + + if (p == NULL) { + VSLb(ctx->vsl, SLT_LostHeader, "storage_hint"); + WS_MarkOverflow(ctx->ws); + WS_Reset(ctx->ws, sn); + return; + } + + stv = VRT_stevedore(p); + if (stv != NULL) + ctx->bo->storage = stv; + + WS_Reset(ctx->ws, sn); +} + /*--------------------------------------------------------------------*/ VCL_STEVEDORE diff --git a/bin/varnishtest/tests/c00078.vtc b/bin/varnishtest/tests/c00078.vtc index f8d234a..9f14a77 100644 --- a/bin/varnishtest/tests/c00078.vtc +++ b/bin/varnishtest/tests/c00078.vtc @@ -1,6 +1,6 @@ varnishtest "Stevedores RR, beresp.storage" -server s1 -repeat 6 { +server s1 -repeat 7 { rxreq txresp } -start @@ -17,8 +17,11 @@ varnish v1 \ set beresp.storage = storage.s1; } else if (bereq.url == "/6") { set beresp.storage = vtc.no_stevedore(); + } else if (bereq.url == "/deprecated") { + set beresp.storage_hint = "s1"; } set beresp.http.storage = beresp.storage; + set beresp.http.storage-hint = beresp.storage_hint; } } -start @@ -26,19 +29,38 @@ client c1 { txreq -url /1 rxresp expect resp.http.storage == "storage.s1" + expect resp.http.storage == resp.http.storage-hint txreq -url /2 rxresp expect resp.http.storage == "storage.s1" + expect resp.http.storage == resp.http.storage-hint txreq -url /3 rxresp expect resp.http.storage == "storage.s0" + expect resp.http.storage == resp.http.storage-hint txreq -url /4 rxresp expect resp.http.storage == "storage.s1" + expect resp.http.storage == resp.http.storage-hint txreq -url /5 rxresp expect resp.http.storage == "storage.s2" + expect resp.http.storage == resp.http.storage-hint txreq -url /6 rxresp expect resp.http.storage == + expect resp.http.storage == resp.http.storage-hint + txreq -url /deprecated + rxresp + expect resp.http.storage == "storage.s1" + expect resp.http.storage == resp.http.storage-hint } -run + +varnish v1 \ + -syntax 4.1 \ + -errvcl "Only available when VCL syntax <= 4.0" { + import vtc; + sub vcl_backend_response { + set beresp.storage_hint = "foo"; + } +} diff --git a/doc/changes.rst b/doc/changes.rst index 7468107..6648e52 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -41,7 +41,17 @@ VCL and bundled VMODs ``req.hash_always_miss`` are now accessible from all of the client side subs, not just ``vcl_recv{}`` -* Removed ``beresp.storage_hint`` (was deprecated since Varnish 5.1) +* Removed ``beresp.storage_hint`` for VCL 4.1 (was deprecated since + Varnish 5.1) + + For VCL 4.0, compatibility is preserved, but the implementation is + changed slightly: ``beresp.storage_hint`` is now referring to the + same internal data structure as ``beresp.storage``. + + In particular, it was previously possible to set + ``beresp.storage_hint`` to an invalid storage name and later + retrieve it back. Doing so will now yield the last successfully set + stevedore or the undefined (``NULL``) string. * workspace overflows in ``std.log()`` now trigger a VCL failure diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 3d17078..37e88c9 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -868,6 +868,21 @@ beresp.storage The storage backend to use to save this object. +beresp.storage_hint ``VCL <= 4.0`` + + Type: STRING + + Readable from: vcl_backend_response, vcl_backend_error + + Writable from: vcl_backend_response, vcl_backend_error + + + Deprecated since varnish 5.1 and discontinued since VCL + 4.1 (varnish 6.0). Use beresp.storage instead. + + Hint to Varnish that you want to save this object to a + particular storage backend. + obj ~~~ diff --git a/include/vrt.h b/include/vrt.h index 9d67405..603d24f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -61,8 +61,8 @@ * VRT_VSC_Overhead() added * struct director.event added * struct director.destroy added - * VRT_r_beresp_storage_hint() removed - under discussion #2509 - * VRT_l_beresp_storage_hint() removed - under discussion #2509 + * VRT_r_beresp_storage_hint() VCL <= 4.0 #2509 + * VRT_l_beresp_storage_hint() VCL <= 4.0 #2509 * VRT_blob() added * VCL_STRANDS added * 6.1 (2017-09-15 aka 5.2) From nils.goroll at uplex.de Mon Mar 5 19:09:05 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 19:09:05 +0000 (UTC) Subject: [master] b4d6b71 beautify make output Message-ID: <20180305190905.CDAD593D81@lists.varnish-cache.org> commit b4d6b7145717f63f405f84ea805a035452cfd624 Author: Nils Goroll Date: Mon Mar 5 20:08:55 2018 +0100 beautify make output diff --git a/include/Makefile.am b/include/Makefile.am index 1b13126..80f6569 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -124,7 +124,7 @@ GENERATED_H = vcl.h $(GEN_H) ## except when building from a distribution vcs_version.h: - if test -d $(top_srcdir)/.git ; then \ + $(AM_V_GEN) if test -d $(top_srcdir)/.git ; then \ @PYTHON@ $(srcdir)/generate.py \ $(top_srcdir) $(top_builddir) ; \ fi From nils.goroll at uplex.de Mon Mar 5 19:26:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 19:26:08 +0000 (UTC) Subject: [master] 4094cf5 retire shard .key() and .reconfigure() algorithm choice Message-ID: <20180305192609.03E7A94268@lists.varnish-cache.org> commit 4094cf5b6cf2718d137dd63ac57e2085e52de377 Author: Nils Goroll Date: Sun Dec 10 17:07:00 2017 +0100 retire shard .key() and .reconfigure() algorithm choice Ref: #2500 diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc index 4d49563..a8f33b1 100644 --- a/bin/varnishtest/tests/d00017.vtc +++ b/bin/varnishtest/tests/d00017.vtc @@ -25,6 +25,7 @@ server s3 { varnish v1 -vcl+backend { import std; import directors; + import blob; sub vcl_init { new vd = directors.shard(); @@ -44,8 +45,9 @@ varnish v1 -vcl+backend { } sub vcl_recv { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(req.url, CRC32)); + set req.backend_hint = vd.backend(by=BLOB, + key_blob=blob.decode(HEX, encoded= + regsub(req.url, "^/", ""))); return(pass); } @@ -142,15 +144,15 @@ logexpect l1 -v v1 -g raw -d 1 { } -start client c1 { - txreq -url /eishoSu2 + txreq -url /68b902f7 rxresp expect resp.body == "ech3Ooj" - txreq -url /Zainao9d + txreq -url /39dc4613 rxresp expect resp.body == "ieQu2qua" - txreq -url /Aunah3uo + txreq -url /c7793505 rxresp expect resp.body == "xiuFi3Pe" } -run diff --git a/bin/varnishtest/tests/d00019.vtc b/bin/varnishtest/tests/d00019.vtc index 5635a1c..9695457 100644 --- a/bin/varnishtest/tests/d00019.vtc +++ b/bin/varnishtest/tests/d00019.vtc @@ -1,4 +1,4 @@ -varnishtest "shard director SHA256 (default)" +varnishtest "shard director by req.url (default)" server s1 { rxreq diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc deleted file mode 100644 index 1fe91c7..0000000 --- a/bin/varnishtest/tests/d00020.vtc +++ /dev/null @@ -1,50 +0,0 @@ -varnishtest "shard director RS" - -server s1 { - rxreq - txresp -body "ech3Ooj" -} -start - -server s2 { - rxreq - txresp -body "ieQu2qua" -} -start - -server s3 { - rxreq - txresp -body "xiuFi3Pe" -} -start - -varnish v1 -vcl+backend { - import directors; - - sub vcl_init { - new vd = directors.shard(); - vd.add_backend(s1); - vd.add_backend(s2); - vd.add_backend(s3); - vd.reconfigure(replicas=25); - } - - sub vcl_recv { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(req.url, alg=RS)); - return(pass); - } - -} -start - - -client c1 { - txreq -url /we0eeTho - rxresp - expect resp.body == "ech3Ooj" - - txreq -url /mae8ooNu - rxresp - expect resp.body == "ieQu2qua" - - txreq -url /oob3dahS - rxresp - expect resp.body == "xiuFi3Pe" -} -run diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc deleted file mode 100644 index 627f717..0000000 --- a/bin/varnishtest/tests/d00021.vtc +++ /dev/null @@ -1,81 +0,0 @@ -varnishtest "shard director key function" - -server s1 { - rxreq - txresp -body "ech3Ooj" - rxreq - txresp -body "ech3Ooj" - rxreq - txresp -body "ech3Ooj" -} -start - -server s2 { - rxreq - txresp -body "ieQu2qua" -} -start - -server s3 { - rxreq - txresp -body "xiuFi3Pe" -} -start - -varnish v1 -vcl+backend { - import directors; - - sub vcl_init { - new vd = directors.shard(); - vd.add_backend(s1); - vd.add_backend(s2); - vd.add_backend(s3); - vd.reconfigure(25); - } - - sub recv_sub { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(req.http.X-Hash, RS)); - } - - sub vcl_recv { - if (req.url == "/1") { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(alg=CRC32, string="/eishoSu2")); - } else if (req.url == "/2") { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key("/eishoSu2")); - } else if (req.url == "/3") { - set req.http.X-Hash = "/oob3dahS"; - call recv_sub; - } else if (req.url == "/null_by_string") { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(req.http.NonExistent)); - } else if (req.url == "/null_by_string_hash") { - set req.backend_hint = vd.backend(by=KEY, - key=vd.key(req.http.NonExistent, SHA256)); - } - return(pass); - } - -} -start - - -client c1 { - txreq -url /1 - rxresp - expect resp.body == "ech3Ooj" - - txreq -url /2 - rxresp - expect resp.body == "ieQu2qua" - - txreq -url /3 - rxresp - expect resp.body == "xiuFi3Pe" - - txreq -url /null_by_string - rxresp - expect resp.body == "ech3Ooj" - - txreq -url /null_by_string_hash - rxresp - expect resp.body == "ech3Ooj" -} -run diff --git a/bin/varnishtest/tests/d00022.vtc b/bin/varnishtest/tests/d00022.vtc index 3d0e9e1..08130a1 100644 --- a/bin/varnishtest/tests/d00022.vtc +++ b/bin/varnishtest/tests/d00022.vtc @@ -40,7 +40,7 @@ varnish v1 -vcl+backend { sub vcl_recv { set req.backend_hint = vd.backend(by=KEY, - key=vd.key("/eishoSu2", CRC32), + key=1756955383, alt=req.restarts, healthy=ALL); diff --git a/bin/varnishtest/tests/d00023.vtc b/bin/varnishtest/tests/d00023.vtc index e427264..36ebe8e 100644 --- a/bin/varnishtest/tests/d00023.vtc +++ b/bin/varnishtest/tests/d00023.vtc @@ -1,8 +1,6 @@ varnishtest "shard director Unhealthy" server s1 { - rxreq - txresp -body "ech3Ooj" } -start server s2 { @@ -32,7 +30,7 @@ varnish v1 -vcl+backend { sub vcl_recv { set req.backend_hint = vd.backend(by=KEY, - key=vd.key("/eishoSu2", CRC32)); + key=1756955383); set req.http.healthy = std.healthy(req.backend_hint); return(pass); } diff --git a/bin/varnishtest/tests/d00024.vtc b/bin/varnishtest/tests/d00024.vtc index 57f76da..3d5606a 100644 --- a/bin/varnishtest/tests/d00024.vtc +++ b/bin/varnishtest/tests/d00024.vtc @@ -35,7 +35,7 @@ varnish v1 -vcl+backend { sub vcl_recv { set req.backend_hint = vd.backend(by=KEY, - key=vd.key(alg=CRC32, string="/eishoSu2")); + key=1756955383); return(pass); } } -start diff --git a/bin/varnishtest/tests/d00026.vtc b/bin/varnishtest/tests/d00026.vtc index 64d63bc..0cfd816 100644 --- a/bin/varnishtest/tests/d00026.vtc +++ b/bin/varnishtest/tests/d00026.vtc @@ -1,4 +1,4 @@ -varnishtest "shard director - same as v01000.vtc but setting backend in fetch" +varnishtest "shard director - same as d00017.vtc but setting backend in fetch" server s1 { rxreq @@ -17,6 +17,7 @@ server s3 { varnish v1 -vcl+backend { import directors; + import blob; sub vcl_init { new vd = directors.shard(); @@ -27,8 +28,9 @@ varnish v1 -vcl+backend { } sub vcl_backend_fetch { - set bereq.backend = vd.backend(by=KEY, - key=vd.key(bereq.url, CRC32)); + set bereq.backend = vd.backend(by=BLOB, + key_blob=blob.decode(HEX, encoded= + regsub(bereq.url, "^/", ""))); return(fetch); } @@ -36,15 +38,15 @@ varnish v1 -vcl+backend { client c1 { - txreq -url /eishoSu2 + txreq -url /68b902f7 rxresp expect resp.body == "ech3Ooj" - txreq -url /Zainao9d + txreq -url /39dc4613 rxresp expect resp.body == "ieQu2qua" - txreq -url /Aunah3uo + txreq -url /c7793505 rxresp expect resp.body == "xiuFi3Pe" } -run diff --git a/doc/changes.rst b/doc/changes.rst index 6648e52..e366f63 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -59,6 +59,51 @@ VCL and bundled VMODs * added ``return(restart)`` from ``vcl_recv{}`` +* The ``alg`` argument of the ``shard`` director ``.reconfigure()`` + method has been removed - the consistent hashing ring is now always + generated using the last 32 bits of a SHA256 hash of ``"ident%d"`` + as with ``alg=SHA256`` or the default. + + We believe that the other algorithms did not yield sufficiently + dispersed placement of backends on the consistent hashing ring and + thus retire this option without replacement. + + Users of ``.reconfigure(alg=CRC32)`` or ``.reconfigure(alg=RS)`` be + advised that when upgrading and removing the ``alg`` argument, + consistent hashing values for all backends will change once and only + once. + +* The ``alg`` argument of the ``shard`` director ``.key()`` method has + been removed - it now always hashes its arguments using SHA256 and + returns the last 32 bits for use as a shard key. + + Backwards compatibility is provided through `vmod blobdigest`_ with + the ``key_blob`` argument of the ``shard`` director ``.backend()`` + method: + + * for ``alg=CRC32``, replace:: + + .backend(by=KEY, key=.key(, CRC32)) + + with:: + + .backend(by=BLOB, key_blob=blobdigest.hash(ICRC32, + blob.decode(encoded=))) + + `Note:` The `vmod blobdigest`_ hash method corresponding to the + shard director CRC32 method is called **I**\ CRC32 + +.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst + + * for ``alg=RS``, replace:: + + .backend(by=KEY, key=.key(, RS)) + + with:: + + .backend(by=BLOB, key_blob=blobdigest.hash(RS, + blob.decode(encoded=))) + Logging / statistics -------------------- diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index ff0c244..d9fce2f 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -12,8 +12,6 @@ libvmod_directors_la_SOURCES = \ shard_cfg.h \ shard_dir.c \ shard_dir.h \ - shard_hash.c \ - shard_hash.h \ shard_parse_vcc_enums.h \ shard_parse_vcc_enums.c diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 620126f..de841ef 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -39,7 +39,6 @@ #include "shard_dir.h" #include "shard_cfg.h" -#include "shard_hash.h" /*lint -esym(749, shard_change_task_e::*) */ enum shard_change_task_e { @@ -232,11 +231,12 @@ circlepoint_compare(const struct shard_circlepoint *a, } static void -shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) +shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas) { int i, j; const char *ident; - int len; + const int len = 12; // log10(UINT32_MAX) + 2; + char s[len]; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); AZ(shardd->hashcircle); @@ -259,14 +259,10 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) assert(ident[0] != '\0'); - len = strlen(ident) + 12; // log10(UINT32_MAX) + 2; - - char s[len]; - for (j = 0; j < replicas; j++) { - assert(snprintf(s, len, "%s%d", ident, j) < len); + assert(snprintf(s, len, "%d", j) < len); shardd->hashcircle[i * replicas + j].point = - shard_hash_f[alg](s); + sharddir_sha256(ident, s, vrt_magic_string_end); shardd->hashcircle[i * replicas + j].host = i; } /* not used in current interface */ @@ -574,7 +570,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) + struct sharddir *shardd, VCL_INT replicas) { struct shard_change *change; @@ -607,7 +603,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, return 0; } - shardcfg_hashcircle(shardd, replicas, alg); + shardcfg_hashcircle(shardd, replicas); sharddir_unlock(shardd); return (1); } diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h index 8d1d88c..2ce1817 100644 --- a/lib/libvmod_directors/shard_cfg.h +++ b/lib/libvmod_directors/shard_cfg.h @@ -34,7 +34,7 @@ VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd); VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_INT replicas, enum alg_e alg_e); + struct sharddir *shardd, VCL_INT replicas); VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio); VCL_VOID shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 36c901b..ff5f9af 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -41,6 +41,8 @@ #include "vbm.h" #include "vrnd.h" +#include "vsha256.h" +#include "vend.h" #include "shard_dir.h" @@ -87,6 +89,47 @@ sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...) va_end(ap); } +uint32_t +sharddir_sha256v(const char *s, va_list ap) +{ + struct VSHA256Context sha256; + union { + unsigned char digest[32]; + uint32_t uint32_digest[8]; + } sha256_digest; + uint32_t r; + const char *p; + + VSHA256_Init(&sha256); + p = s; + while (p != vrt_magic_string_end) { + if (p != NULL && *p != '\0') + VSHA256_Update(&sha256, p, strlen(p)); + p = va_arg(ap, const char *); + } + VSHA256_Final(sha256_digest.digest, &sha256); + + /* + * use low 32 bits only + * XXX: Are these the best bits to pick? + */ + vle32enc(&r, sha256_digest.uint32_digest[7]); + return (r); +} + +uint32_t +sharddir_sha256(const char *s, ...) +{ + va_list ap; + uint32_t r; + + va_start(ap, s); + r = sharddir_sha256v(s, ap); + va_end(ap); + + return (r); +} + static int shard_lookup(const struct sharddir *shardd, const uint32_t key) { diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 35a947c..c893c3d 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -103,6 +103,8 @@ sharddir_backend_ident(const struct sharddir *shardd, int host) void sharddir_debug(struct sharddir *shardd, const uint32_t flags); void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...); +uint32_t sharddir_sha256v(const char *s, va_list ap); +uint32_t sharddir_sha256(const char *s, ...); void sharddir_new(struct sharddir **sharddp, const char *vcl_name); void sharddir_delete(struct sharddir **sharddp); void sharddir_wrlock(struct sharddir *shardd); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c deleted file mode 100644 index 9d0dc94..0000000 --- a/lib/libvmod_directors/shard_hash.c +++ /dev/null @@ -1,110 +0,0 @@ -/*- - * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Authors: Nils Goroll - * Geoffrey Simmons - * Julian Wiesener - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include - -#include "cache/cache.h" - -#include "vsha256.h" -#include "vend.h" - -#include "shard_parse_vcc_enums.h" -#include "shard_hash.h" - -/* - * XXX use the crc32 from libvgz, but declare it here to avoid an include - * dependency nightmare (at least for now) - */ - -unsigned long crc32(unsigned long, const unsigned char *buf, unsigned len); - -static uint32_t v_matchproto_(hash_func) -shard_hash_crc32(VCL_STRING s) -{ - uint32_t crc; - crc = crc32(~0U, (const unsigned char *)s, strlen(s)); - crc ^= ~0U; - return (crc); -} - -static uint32_t v_matchproto_(hash_func) -shard_hash_sha256(VCL_STRING s) -{ - struct VSHA256Context sha256; - union { - unsigned char digest[32]; - uint32_t uint32_digest[8]; - } sha256_digest; - uint32_t r; - - VSHA256_Init(&sha256); - VSHA256_Update(&sha256, s, strlen(s)); - VSHA256_Final(sha256_digest.digest, &sha256); - - /* - * use low 32 bits only - * XXX: Are these the best bits to pick? - */ - vle32enc(&r, sha256_digest.uint32_digest[7]); - return (r); -} - -static uint32_t v_matchproto_(hash_func) -shard_hash_rs(VCL_STRING s) -{ - uint32_t res = 0; - /* hash function from Robert Sedgwicks 'Algorithms in C' book */ - const uint32_t b = 378551; - uint32_t a = 63689; - - while (*s) { - res = res * a + (*s++); - a *= b; - } - - return (res); -} - -static uint32_t v_matchproto_(hash_func) -_shard_hash_invalid(VCL_STRING s) -{ - (void)s; - WRONG("invalid hash fp _ALG_E_ENVALID"); - NEEDLESS(return(0)); -} - -const hash_func shard_hash_f[_ALG_E_MAX] = { - [_ALG_E_INVALID] = _shard_hash_invalid, - [CRC32] = shard_hash_crc32, - [SHA256] = shard_hash_sha256, - [RS] = shard_hash_rs -}; diff --git a/lib/libvmod_directors/shard_hash.h b/lib/libvmod_directors/shard_hash.h deleted file mode 100644 index 838032a..0000000 --- a/lib/libvmod_directors/shard_hash.h +++ /dev/null @@ -1,30 +0,0 @@ -/*- - * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Author: Julian Wiesener - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -typedef uint32_t (*hash_func)(VCL_STRING); -extern const hash_func shard_hash_f[_ALG_E_MAX]; diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 869f62e..4737284 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -307,10 +307,11 @@ Method `````` When ``.reconfigure()`` is called, a consistent hashing circular data -structure gets built from hash values of "ident%d" (default ident -being the backend name) for each backend and for a running number from -1 to n (n is the number of `replicas`). Hashing creates the seemingly -random order for placement of backends on the consistent hashing ring. +structure gets built from the last 32 bits of SHA256 hash values of +``\ `` (default `ident` being the backend name) for each +backend and for a running number `n` from 1 to `replicas`. Hashing +creates the seemingly random order for placement of backends on the +consistent hashing ring. When ``.backend()`` is called, a load balancing key gets generated unless provided. The smallest hash value in the circle is looked up @@ -388,19 +389,24 @@ Remove all backends from the director. NOTE: Backend changes need to be finalized with `shard.reconfigure()` and are only supported on one shard director at a time. -$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67, - ENUM { CRC32, SHA256, RS } alg="SHA256") +$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67) Reconfigure the consistent hashing ring to reflect backend changes. This method must be called at least once before the director can be used. -$Method INT .key(STRING string, ENUM { CRC32, SHA256, RS } alg="SHA256") +$Method INT .key(STRING_LIST) -Utility method to generate a sharding key for use with the -``shard.backend()`` method by hashing `string` with hash algorithm -`alg`. +Convenience method to generate a sharding key for use with the `key` +argument to the ``shard.backend()`` method by hashing the given string +with SHA256. + +To generate sharding keys using other hashes, use a custom vmod like +`vmod blobdigest`_ with the `key_blob` argument of the +``shard.backend()`` method. + +.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst $Method BACKEND .backend( ENUM {HASH, URL, KEY, BLOB} by="HASH", diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index e5ddb58..6a8660a 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -40,7 +40,6 @@ #include "vcc_if.h" #include "shard_dir.h" #include "shard_cfg.h" -#include "shard_hash.h" struct vmod_directors_shard { unsigned magic; @@ -56,7 +55,7 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, VCL_INT t1; uint32_t t2a, t2b; - /* see vmod_key comment */ + /* we put our uint32 key in a VCL_INT container */ assert(sizeof(VCL_INT) >= sizeof(uint32_t)); t2a = UINT32_MAX; t1 = (VCL_INT)t2a; @@ -84,22 +83,20 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp) FREE_OBJ(vshard); } -/* - * our key is a uint32_t, but VCL_INT is a (signed) long. We cast back and - * forth, asserting in vmod_shard__init() that VCL_INT is a large enough - * container - */ VCL_INT v_matchproto_(td_directors_shard_key) - vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard, - VCL_STRING s, VCL_ENUM alg_s) +vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard, const char *s, ...) { - enum alg_e alg = parse_alg_e(alg_s); - hash_func hash_fp = shard_hash_f[alg]; + va_list ap; + uint32_t r; (void)ctx; - (void)vshard;; + (void)vshard; + + va_start(ap, s); + r = sharddir_sha256v(s, ap); + va_end(ap); - return (VCL_INT)hash_fp(s ? s : ""); + return ((VCL_INT)r); } VCL_VOID v_matchproto_(td_directors_set_warmup) @@ -169,11 +166,9 @@ vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard, VCL_BOOL v_matchproto_(td_directors_shard_reconfigure) vmod_shard_reconfigure(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_priv *priv, VCL_INT replicas, VCL_ENUM alg_s) + struct vmod_priv *priv, VCL_INT replicas) { - enum alg_e alg = parse_alg_e(alg_s); - - return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas, alg); + return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas); } static inline uint32_t @@ -198,7 +193,8 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) AN(ctx->http_bereq); AN(http = ctx->http_bereq); } - return (shard_hash_f[SHA256](http->hd[HTTP_HDR_URL].b)); + return (sharddir_sha256(http->hd[HTTP_HDR_URL].b, + vrt_magic_string_end)); case BY_KEY: return ((uint32_t)key_int); case BY_BLOB: From nils.goroll at uplex.de Mon Mar 5 19:26:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 19:26:09 +0000 (UTC) Subject: [master] 2676c2e shard director vcc enum overhaul Message-ID: <20180305192609.278419426B@lists.varnish-cache.org> commit 2676c2e3793fe6fdb691295890b2400391564ed0 Author: Nils Goroll Date: Mon Dec 11 17:52:33 2017 +0100 shard director vcc enum overhaul diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index d9fce2f..1275938 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -12,8 +12,8 @@ libvmod_directors_la_SOURCES = \ shard_cfg.h \ shard_dir.c \ shard_dir.h \ - shard_parse_vcc_enums.h \ - shard_parse_vcc_enums.c + tbl_by.h \ + tbl_healthy.h # Use vmodtool.py generated automake boilerplate include $(srcdir)/automake_boilerplate.am diff --git a/lib/libvmod_directors/flint.lnt b/lib/libvmod_directors/flint.lnt index e69de29..1b6f088 100644 --- a/lib/libvmod_directors/flint.lnt +++ b/lib/libvmod_directors/flint.lnt @@ -0,0 +1 @@ +-efile(451, "tbl_*.h") // No include guard diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index c893c3d..4df4151 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -27,7 +27,19 @@ * SUCH DAMAGE. */ -#include "shard_parse_vcc_enums.h" +enum by_e { + _BY_E_INVALID = 0, +#define VMODENUM(x) BY_ ## x, +#include "tbl_by.h" + _BY_E_MAX +}; + +enum healthy_e { + _HEALTHY_E_INVALID = 0, +#define VMODENUM(x) x, +#include "tbl_healthy.h" + _HEALTHY_E_MAX +}; struct vbitmap; diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c deleted file mode 100644 index 7061ced..0000000 --- a/lib/libvmod_directors/shard_parse_vcc_enums.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * for the time being, this code is auto-generated outside the varnishd source - * tree, see - * https://code.uplex.de/uplex-varnish/libvmod-vslp/blob/shard/src/gen_enum_parse.pl - * - * TODO: integrate in vmodtool.py or replace with something else - */ - -/*lint -e801 */ - -#include "shard_parse_vcc_enums.h" -#define term(c) ((c) == '\0') - - - -enum alg_e parse_alg_e (const char *m) { - enum alg_e r; - - switch (m[0]) { - case 'C': goto _0C; // CRC32 - case 'R': goto _0R; // RS - case 'S': goto _0S; // SHA256 - default: goto invalid; - } - _0C: - //CRC32 - if ((m[1] == 'R') && (m[2] == 'C') && (m[3] == '3') && (m[4] == '2') && (term(m[5]))) { - r = CRC32; - goto ok; - } - goto invalid; - _0R: - //RS - if ((m[1] == 'S') && (term(m[2]))) { - r = RS; - goto ok; - } - goto invalid; - _0S: - //SHA256 - if ((m[1] == 'H') && (m[2] == 'A') && (m[3] == '2') && (m[4] == '5') && (m[5] == '6') && (term(m[6]))) { - r = SHA256; - goto ok; - } - goto invalid; - ok: - return r; - invalid: - return _ALG_E_INVALID; -} - - -enum by_e parse_by_e (const char *m) { - enum by_e r; - - switch (m[0]) { - case 'B': goto _0B; // BLOB - case 'H': goto _0H; // HASH - case 'K': goto _0K; // KEY - case 'U': goto _0U; // URL - default: goto invalid; - } - _0B: - //BLOB - if ((m[1] == 'L') && (m[2] == 'O') && (m[3] == 'B') && (term(m[4]))) { - r = BY_BLOB; - goto ok; - } - goto invalid; - _0H: - //HASH - if ((m[1] == 'A') && (m[2] == 'S') && (m[3] == 'H') && (term(m[4]))) { - r = BY_HASH; - goto ok; - } - goto invalid; - _0K: - //KEY - if ((m[1] == 'E') && (m[2] == 'Y') && (term(m[3]))) { - r = BY_KEY; - goto ok; - } - goto invalid; - _0U: - //URL - if ((m[1] == 'R') && (m[2] == 'L') && (term(m[3]))) { - r = BY_URL; - goto ok; - } - goto invalid; - ok: - return r; - invalid: - return _BY_E_INVALID; -} - - -enum healthy_e parse_healthy_e (const char *m) { - enum healthy_e r; - - switch (m[0]) { - case 'A': goto _0A; // ALL - case 'C': goto _0C; // CHOSEN - case 'I': goto _0I; // IGNORE - default: goto invalid; - } - _0A: - //ALL - if ((m[1] == 'L') && (m[2] == 'L') && (term(m[3]))) { - r = ALL; - goto ok; - } - goto invalid; - _0C: - //CHOSEN - if ((m[1] == 'H') && (m[2] == 'O') && (m[3] == 'S') && (m[4] == 'E') && (m[5] == 'N') && (term(m[6]))) { - r = CHOSEN; - goto ok; - } - goto invalid; - _0I: - //IGNORE - if ((m[1] == 'G') && (m[2] == 'N') && (m[3] == 'O') && (m[4] == 'R') && (m[5] == 'E') && (term(m[6]))) { - r = IGNORE; - goto ok; - } - goto invalid; - ok: - return r; - invalid: - return _HEALTHY_E_INVALID; -} diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.h b/lib/libvmod_directors/shard_parse_vcc_enums.h deleted file mode 100644 index afa0c47..0000000 --- a/lib/libvmod_directors/shard_parse_vcc_enums.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * for the time being, this code is auto-generated outside the varnishd source - * tree, see - * https://code.uplex.de/uplex-varnish/libvmod-vslp/blob/shard/src/gen_enum_parse.pl - * - * TODO: integrate in vmodtool.py or replace with something else - */ - -enum alg_e { - _ALG_E_INVALID = 0, - CRC32, - SHA256, - RS, - _ALG_E_MAX -}; - - -enum alg_e parse_alg_e (const char *); - -/*lint -esym(769, by_e::_BY_E_MAX) */ - -enum by_e { - _BY_E_INVALID = 0, - BY_HASH, - BY_URL, - BY_KEY, - BY_BLOB, - _BY_E_MAX -}; - -enum by_e parse_by_e (const char *); - -/*lint -esym(769, healthy_e::_HEALTHY_E_MAX) */ - -enum healthy_e { - _HEALTHY_E_INVALID = 0, - CHOSEN, - IGNORE, - ALL, - _HEALTHY_E_MAX -}; - - -enum healthy_e parse_healthy_e (const char *); - diff --git a/lib/libvmod_directors/tbl_by.h b/lib/libvmod_directors/tbl_by.h new file mode 100644 index 0000000..97cb828 --- /dev/null +++ b/lib/libvmod_directors/tbl_by.h @@ -0,0 +1,5 @@ +VMODENUM(HASH) +VMODENUM(URL) +VMODENUM(KEY) +VMODENUM(BLOB) +#undef VMODENUM diff --git a/lib/libvmod_directors/tbl_healthy.h b/lib/libvmod_directors/tbl_healthy.h new file mode 100644 index 0000000..7be0e44 --- /dev/null +++ b/lib/libvmod_directors/tbl_healthy.h @@ -0,0 +1,4 @@ +VMODENUM(CHOSEN) +VMODENUM(IGNORE) +VMODENUM(ALL) +#undef VMODENUM diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 4737284..ef2fd75 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -409,13 +409,13 @@ To generate sharding keys using other hashes, use a custom vmod like .. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst $Method BACKEND .backend( - ENUM {HASH, URL, KEY, BLOB} by="HASH", + ENUM {HASH, URL, KEY, BLOB} by=HASH, INT key=0, BLOB key_blob=0, INT alt=0, REAL warmup=-1, BOOL rampup=1, - ENUM {CHOSEN, IGNORE, ALL} healthy="CHOSEN") + ENUM {CHOSEN, IGNORE, ALL} healthy=CHOSEN) Lookup a backend on the consistent hashing ring. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 6a8660a..655e2ad 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -218,6 +218,22 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) } } +static enum by_e +parse_by_e(VCL_ENUM e) +{ +#define VMODENUM(n) if (e == vmod_enum_ ## n) return(BY_ ## n); +#include "tbl_by.h" + WRONG("illegal by enum"); +} + +static enum healthy_e +parse_healthy_e(VCL_ENUM e) +{ +#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#include "tbl_healthy.h" + WRONG("illegal healthy enum"); +} + VCL_BACKEND v_matchproto_(td_directors_shard_backend) vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, VCL_ENUM by_s, VCL_INT key_int, VCL_BLOB key_blob, VCL_INT alt, From nils.goroll at uplex.de Mon Mar 5 19:26:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 19:26:09 +0000 (UTC) Subject: [master] bc795d1 polishing thanks to flexelint Message-ID: <20180305192609.47AB09426F@lists.varnish-cache.org> commit bc795d16c589679af5e2ef93a91a1443a1b59111 Author: Nils Goroll Date: Wed Dec 20 20:30:10 2017 +0100 polishing thanks to flexelint diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index de841ef..adec623 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -67,9 +67,9 @@ struct shard_change { struct backend_reconfig { struct sharddir * const shardd; - int hint; // on number of backends after reconfig - int hole_n; // number of holes in backends array - int hole_i; // index hint on first hole + unsigned hint; // on number of backends after reconfig + unsigned hole_n; // number of holes in backends array + unsigned hole_i; // index hint on first hole }; /* @@ -350,7 +350,7 @@ static const struct shard_backend * shardcfg_backend_lookup(const struct backend_reconfig *re, const struct shard_backend *b) { - int i, max = re->shardd->n_backend + re->hole_n; + unsigned i, max = re->shardd->n_backend + re->hole_n; const struct shard_backend *bb = re->shardd->backend; for (i = 0; i < max; i++) @@ -363,7 +363,7 @@ shardcfg_backend_lookup(const struct backend_reconfig *re, static void shardcfg_backend_expand(const struct backend_reconfig *re) { - int min = re->hint; + unsigned min = re->hint; CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC); @@ -389,7 +389,7 @@ static void shardcfg_backend_add(struct backend_reconfig *re, const struct shard_backend *b) { - int i; + unsigned i; struct shard_backend *bb = re->shardd->backend; if (re->hole_n == 0) { @@ -428,7 +428,7 @@ static void shardcfg_backend_del(struct backend_reconfig *re, const struct shard_backend *spec) { - int i, max = re->shardd->n_backend + re->hole_n; + unsigned i, max = re->shardd->n_backend + re->hole_n; struct shard_backend * const bb = re->shardd->backend; for (i = 0; i < max; i++) { @@ -448,7 +448,7 @@ shardcfg_backend_del(struct backend_reconfig *re, static void shardcfg_backend_finalize(struct backend_reconfig *re) { - int i; + unsigned i; struct shard_backend * const bb = re->shardd->backend; while (re->hole_n > 0) { From phk at FreeBSD.org Mon Mar 5 20:41:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 20:41:10 +0000 (UTC) Subject: [master] 3c5fc32 Keep the symbol table sorted, also with versioned symbols. Message-ID: <20180305204110.48D9E958D5@lists.varnish-cache.org> commit 3c5fc3266c52acdde016788b422bd51b7f5a8541 Author: Poul-Henning Kamp Date: Mon Mar 5 20:34:41 2018 +0000 Keep the symbol table sorted, also with versioned symbols. Add assert to make sure it stays that way. diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 823a53f..4d088ec 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -130,8 +130,6 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent, assert(l > 0); VTAILQ_FOREACH(sym, &parent->children, list) { - if (sym->lorev > vhi || sym->hirev < vlo) - continue; i = strncasecmp(sym->name, b, l); if (i < 0) continue; @@ -142,6 +140,8 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent, } if (l > sym->nlen) continue; + if (sym->lorev > vhi || sym->hirev < vlo) + continue; if (q < e) break; if ((kind == SYM_NONE && kind == sym->kind)) @@ -288,9 +288,12 @@ static void vcc_walksymbols(struct vcc *tl, const struct symbol *root, symwalk_f *func, vcc_kind_t kind) { - struct symbol *sym; + struct symbol *sym, *sym2 = NULL; VTAILQ_FOREACH(sym, &root->children, list) { + if (sym2 != NULL) + assert(strcasecmp(sym->name, sym2->name) >= 0); + sym2 = sym; if (kind == SYM_NONE || kind == sym->kind) func(tl, sym); ERRCHK(tl); From phk at FreeBSD.org Mon Mar 5 20:41:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 20:41:10 +0000 (UTC) Subject: [master] de0ad93 Version the reserved word symbols to VCL41+ Message-ID: <20180305204110.5CE70958D8@lists.varnish-cache.org> commit de0ad935fe7d103c6d66778cb93dad9de1537d8f Author: Poul-Henning Kamp Date: Mon Mar 5 20:35:30 2018 +0000 Version the reserved word symbols to VCL41+ Spotted by: Dridi diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index eb82b52..a81950a 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -347,14 +347,16 @@ typedef void parse_f(struct vcc *tl); static struct toplev { const char *name; parse_f *func; + unsigned vcllo; + unsigned vclhi; } toplev[] = { - { "acl", vcc_ParseAcl }, - { "sub", vcc_ParseFunction }, - { "backend", vcc_ParseBackend }, - { "probe", vcc_ParseProbe }, - { "import", vcc_ParseImport }, - { "vcl", vcc_ParseVcl }, - { "default", NULL }, + { "acl", vcc_ParseAcl, VCL_41, VCL_HIGH }, + { "sub", vcc_ParseFunction, VCL_41, VCL_HIGH }, + { "backend", vcc_ParseBackend, VCL_41, VCL_HIGH }, + { "probe", vcc_ParseProbe, VCL_41, VCL_HIGH }, + { "import", vcc_ParseImport, VCL_41, VCL_HIGH }, + { "vcl", vcc_ParseVcl, VCL_41, VCL_HIGH }, + { "default", NULL, VCL_41, VCL_HIGH }, { NULL, NULL } }; @@ -432,5 +434,5 @@ vcc_Parse_Init(struct vcc *tl) struct toplev *tp; for (tp = toplev; tp->name != NULL; tp++) - AN(VCC_MkSym(tl, tp->name, SYM_NONE, VCL_LOW, VCL_HIGH)); + AN(VCC_MkSym(tl, tp->name, SYM_NONE, tp->vcllo, tp->vclhi)); } From phk at FreeBSD.org Mon Mar 5 20:41:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 20:41:10 +0000 (UTC) Subject: [master] f28cafe Keep the storage.*.* symbols around for vcl4.1 Message-ID: <20180305204110.7585C958DB@lists.varnish-cache.org> commit f28cafe8eea6861fb4827d9a7f76917620695712 Author: Poul-Henning Kamp Date: Mon Mar 5 20:38:52 2018 +0000 Keep the storage.*.* symbols around for vcl4.1 Properly implementing vmod_storage requires the VMOD to sanction symbols at VCC compile time, and I'd rather not face that right now. Fixes #2596 diff --git a/bin/varnishtest/tests/c00078.vtc b/bin/varnishtest/tests/c00078.vtc index 9f14a77..898d27e 100644 --- a/bin/varnishtest/tests/c00078.vtc +++ b/bin/varnishtest/tests/c00078.vtc @@ -9,7 +9,7 @@ varnish v1 \ -arg "-ss1=default,1m" \ -arg "-ss2=default,1m" \ -arg "-ss0=default,1m" \ - -syntax 4.0 \ + -syntax 4.1 \ -vcl+backend { import vtc; sub vcl_backend_response { diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c index a979708..aa83a32 100644 --- a/lib/libvcc/vcc_storage.c +++ b/lib/libvcc/vcc_storage.c @@ -84,7 +84,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name) CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC); bprintf(buf, "storage.%s", stv_name); - sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_40); + sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_41); AN(sym); sym->type = STEVEDORE; sym->eval = vcc_Eval_Var; @@ -94,7 +94,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name) for (sv = stvars; sv->name != NULL; sv++) { bprintf(buf, "storage.%s.%s", stv_name, sv->name); - sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_40); + sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_41); AN(sym); sym->type = sv->type; sym->eval = vcc_Eval_Var; From phk at FreeBSD.org Mon Mar 5 21:00:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Mar 2018 21:00:10 +0000 (UTC) Subject: [master] 2ecd469 Go back to vcl4.0, Nils and I crossed commits. Message-ID: <20180305210010.41CA395EA8@lists.varnish-cache.org> commit 2ecd469c8ceb9cdecb0b80e796fd445fb7773c22 Author: Poul-Henning Kamp Date: Mon Mar 5 20:59:32 2018 +0000 Go back to vcl4.0, Nils and I crossed commits. diff --git a/bin/varnishtest/tests/c00078.vtc b/bin/varnishtest/tests/c00078.vtc index 898d27e..9f14a77 100644 --- a/bin/varnishtest/tests/c00078.vtc +++ b/bin/varnishtest/tests/c00078.vtc @@ -9,7 +9,7 @@ varnish v1 \ -arg "-ss1=default,1m" \ -arg "-ss2=default,1m" \ -arg "-ss0=default,1m" \ - -syntax 4.1 \ + -syntax 4.0 \ -vcl+backend { import vtc; sub vcl_backend_response { From nils.goroll at uplex.de Mon Mar 5 21:47:06 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 21:47:06 +0000 (UTC) Subject: [master] 9533146 leave a note on a shard director requirement Message-ID: <20180305214706.AA0C496C14@lists.varnish-cache.org> commit 9533146fc4106729262fdd388167b6e0b6662bde Author: Nils Goroll Date: Mon Mar 5 21:42:43 2018 +0100 leave a note on a shard director requirement diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 8d1d380..7db5a4b 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -39,6 +39,7 @@ typedef unsigned vdi_healthy_f(const struct director *, const struct busyobj *, double *changed); +/* XXX need a VRT_CTX argument */ typedef const struct director *vdi_resolve_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_gethdrs_f(const struct director *, struct worker *, From nils.goroll at uplex.de Mon Mar 5 21:47:06 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Mar 2018 21:47:06 +0000 (UTC) Subject: [master] 9b8f6e2 shard director: LAZY mode (vdi resolve function), parameter objects Message-ID: <20180305214706.BC7AE96C16@lists.varnish-cache.org> commit 9b8f6e21809bf867eb5bd792331484d0e70eb367 Author: Nils Goroll Date: Mon Mar 5 22:30:16 2018 +0100 shard director: LAZY mode (vdi resolve function), parameter objects We introduce a shard_param object to hold the shard director lookup parameters which until now could only be passed to the .backend() method. By associating a parameter object with a shard director, we enable LAZY lookups as with the other directors. Parameter objects are defined with VCL scope (normal vmod objects), but can be overridden per backend request using a task priv. We use the same concept to carry shard.backend() parameters to vdi resolve for LAZY mode: They get saved in a per-director task scope parameter object. Each object points to another object providing defaults for values which are not defined. Actual resolution of the various parameter objects does not happen before they are used to allow changing them independently (ie, shard .backend() parameters have precedence over an associated parameter object, which by itself can be overridden). Overview of parameter objects (pointers are alternatives) shard() director shard_param() object default praram ---------------------------------> vmod static VCL obj / -> .param -+---------> VCL obj / _ .default -------- /| / ^ / | / / .default / -------------> TASK priv / / / .default ----------------------------- TASK priv diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc new file mode 100644 index 0000000..358a623 --- /dev/null +++ b/bin/varnishtest/tests/d00020.vtc @@ -0,0 +1,412 @@ +varnishtest "shard director parameters" + +server s1 -repeat 20 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import directors; + import blob; + + sub vcl_init { + new shard = directors.shard(); + + new p_def = directors.shard_param(); + + new p_hash = directors.shard_param(); + p_hash.set(by=HASH, alt=1); + + new p_url = directors.shard_param(); + p_url.set(by=URL, warmup=0.5); + + new p_key = directors.shard_param(); + p_key.set(by=KEY, key=5, rampup=false); + + new p_blob = directors.shard_param(); + p_blob.set(by=BLOB, healthy=IGNORE, + key_blob=blob.decode(HEX, encoded="ffffffff00")); + } + + sub vcl_synth { + if (req.url == "/def") { + set resp.http.sha256 = shard.key(req.url); + set resp.http.by = p_def.get_by(); + set resp.http.key = p_def.get_key(); + set resp.http.alt = p_def.get_alt(); + set resp.http.warmup = p_def.get_warmup(); + set resp.http.rampup = p_def.get_rampup(); + set resp.http.healthy = p_def.get_healthy(); + } else + if (req.url == "/hash") { + set resp.http.sha256 = shard.key(req.url); + set resp.http.by = p_hash.get_by(); + set resp.http.key = p_hash.get_key(); + set resp.http.alt = p_hash.get_alt(); + set resp.http.warmup = p_hash.get_warmup(); + set resp.http.rampup = p_hash.get_rampup(); + set resp.http.healthy = p_hash.get_healthy(); + } else + if (req.url == "/url") { + set resp.http.sha256 = shard.key(req.url); + set resp.http.by = p_url.get_by(); + set resp.http.key = p_url.get_key(); + set resp.http.alt = p_url.get_alt(); + set resp.http.warmup = p_url.get_warmup(); + set resp.http.rampup = p_url.get_rampup(); + set resp.http.healthy = p_url.get_healthy(); + } else + if (req.url == "/key") { + set resp.http.by = p_key.get_by(); + set resp.http.key = p_key.get_key(); + set resp.http.alt = p_key.get_alt(); + set resp.http.warmup = p_key.get_warmup(); + set resp.http.rampup = p_key.get_rampup(); + set resp.http.healthy = p_key.get_healthy(); + } else + if (req.url == "/blob") { + set resp.http.by = p_blob.get_by(); + set resp.http.key = p_blob.get_key(); + set resp.http.alt = p_blob.get_alt(); + set resp.http.warmup = p_blob.get_warmup(); + set resp.http.rampup = p_blob.get_rampup(); + set resp.http.healthy = p_blob.get_healthy(); + } + } + + sub vcl_backend_response { + # overriding things + if (bereq.url ~ "^/b/c/hash/") { + set beresp.http.override = bereq.url; + p_def.set(by=HASH, alt=7); + p_hash.set(by=HASH, alt=8); + p_url.set(by=HASH, alt=9); + p_key.set(by=HASH, alt=10); + p_blob.set(by=HASH, alt=11); + } + if (bereq.url ~ "^/b/c/url/") { + set beresp.http.override = bereq.url; + p_def.set(by=URL, warmup=0.7); + p_hash.set(by=URL, warmup=0.8); + p_url.set(by=URL, rampup=false); + p_key.set(by=URL, healthy=ALL); + p_blob.set(by=URL, warmup=0.9); + } + if (bereq.url ~ "^/b/c/key/") { + set beresp.http.override = bereq.url; + p_def.set(by=KEY, key=7); + p_hash.set(by=KEY, key=8); + p_url.set(by=KEY, key=9); + p_key.set(by=KEY, key=10); + p_blob.set(by=KEY, key=11); + } + if (bereq.url ~ "/hash|/def") { + set beresp.http.hash = blob.encode(HEX, blob=bereq.hash); + } + if (bereq.url ~ "/url") { + set beresp.http.sha256 = shard.key(bereq.url); + } + if (bereq.url ~ "/def$") { + set beresp.http.by = p_def.get_by(); + set beresp.http.key = p_def.get_key(); + set beresp.http.alt = p_def.get_alt(); + set beresp.http.warmup = p_def.get_warmup(); + set beresp.http.rampup = p_def.get_rampup(); + set beresp.http.healthy = p_def.get_healthy(); + } else + if (bereq.url ~ "/hash$") { + set beresp.http.by = p_hash.get_by(); + set beresp.http.key = p_hash.get_key(); + set beresp.http.alt = p_hash.get_alt(); + set beresp.http.warmup = p_hash.get_warmup(); + set beresp.http.rampup = p_hash.get_rampup(); + set beresp.http.healthy = p_hash.get_healthy(); + } else + if (bereq.url ~ "/url$") { + set beresp.http.by = p_url.get_by(); + set beresp.http.key = p_url.get_key(); + set beresp.http.alt = p_url.get_alt(); + set beresp.http.warmup = p_url.get_warmup(); + set beresp.http.rampup = p_url.get_rampup(); + set beresp.http.healthy = p_url.get_healthy(); + } else + if (bereq.url ~ "/key$") { + set beresp.http.by = p_key.get_by(); + set beresp.http.key = p_key.get_key(); + set beresp.http.alt = p_key.get_alt(); + set beresp.http.warmup = p_key.get_warmup(); + set beresp.http.rampup = p_key.get_rampup(); + set beresp.http.healthy = p_key.get_healthy(); + } else + if (bereq.url ~ "/blob$") { + set beresp.http.by = p_blob.get_by(); + set beresp.http.key = p_blob.get_key(); + set beresp.http.alt = p_blob.get_alt(); + set beresp.http.warmup = p_blob.get_warmup(); + set beresp.http.rampup = p_blob.get_rampup(); + set beresp.http.healthy = p_blob.get_healthy(); + } + } + + sub vcl_recv { + if (req.url ~ "^/b/") { + return (pass); + } + return (synth(200)); + } +} -start + + +client c1 { + txreq -url /def + rxresp + expect resp.http.sha256 == 2002449278 + expect resp.http.by == "HASH" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /hash + rxresp + expect resp.http.sha256 == 2439792896 + expect resp.http.by == "HASH" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 1 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + # for client side, HASH = URL + txreq -url /url + rxresp + expect resp.http.sha256 == 3281611639 + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.500" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /key + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 5 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "false" + expect resp.http.healthy == "CHOSEN" + + txreq -url /blob + rxresp + expect resp.http.by == "BLOB" + expect resp.http.key == 4294967295 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "IGNORE" +} -run + +client c2 { + txreq -url /b/def + rxresp + expect resp.http.hash == "93d1c4ad76396c91dd97fa310f7f26445332662c89393dbeeb77fe49f9111ee4" + expect resp.http.by == "HASH" + # == 0x93d1c4ad + expect resp.http.key == 2479998125 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/hash + rxresp + expect resp.http.hash == "e47da20ea4db49d4f22acdadc69f02f445002be520a2865cd3351272add62540" + expect resp.http.by == "HASH" + # == 0xe47da20e + expect resp.http.key == 3833438734 + expect resp.http.alt == 1 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/url + rxresp + expect resp.http.sha256 == 108501858 + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.500" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/key + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 5 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "false" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/blob + rxresp + expect resp.http.by == "BLOB" + expect resp.http.key == 4294967295 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "IGNORE" +} -run + +client c3 { + txreq -url /b/c/hash/def + rxresp + expect resp.http.hash == "df9a465f8a0455c334b24c1638d3adda0f6e64fbe759029ab83602e3b9138884" + expect resp.http.by == "HASH" + # == 0xdf9a465f + expect resp.http.key == 3751429727 + expect resp.http.alt == 7 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/hash/hash + rxresp + expect resp.http.hash == "0eb35bc1fab5aad5902fd1bac86540bd13d43aa31c6c46f54e776b43392e66e6" + expect resp.http.by == "HASH" + # == 0x0eb35bc1 + expect resp.http.key == 246635457 + expect resp.http.alt == 8 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/hash/url + rxresp + expect resp.http.hash == "1eb67b701ea07151cac5bea1f11b6267b9de15a3ff83cec995590480cbc2c750" + expect resp.http.by == "HASH" + # == 0x1eb67b70 + expect resp.http.key == 515275632 + expect resp.http.alt == 9 + expect resp.http.warmup == "0.500" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/hash/key + rxresp + expect resp.http.hash == "a11b617e21aa7db22b6205d7612002e595b1b00d8c11602017f65456a1be3a35" + expect resp.http.by == "HASH" + # == 0xa11b617e + expect resp.http.key == 2702926206 + expect resp.http.alt == 10 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "false" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/hash/blob + rxresp + expect resp.http.hash == "d7eecc0ac83e1727332dcd8c7c8ae9f3114123abb2bf7e3fb15ecea8c84bb239" + expect resp.http.by == "HASH" + # == 0xd7eecc0a + expect resp.http.key == 3622751242 + expect resp.http.alt == 11 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "IGNORE" +} -run + +client c3 { + txreq -url /b/c/url/def + rxresp + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.700" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/url/hash + rxresp + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 1 + expect resp.http.warmup == "0.800" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/url/url + rxresp + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.500" + expect resp.http.rampup == "false" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/url/key + rxresp + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "false" + expect resp.http.healthy == "ALL" + + txreq -url /b/c/url/blob + rxresp + expect resp.http.by == "URL" + expect resp.http.key == resp.http.sha256 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.900" + expect resp.http.rampup == "true" + expect resp.http.healthy == "IGNORE" +} -run + +client c4 { + txreq -url /b/c/key/def + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 7 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/key/hash + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 8 + expect resp.http.alt == 1 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/key/url + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 9 + expect resp.http.alt == 0 + expect resp.http.warmup == "0.500" + expect resp.http.rampup == "true" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/key/key + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 10 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "false" + expect resp.http.healthy == "CHOSEN" + + txreq -url /b/c/key/blob + rxresp + expect resp.http.by == "KEY" + expect resp.http.key == 11 + expect resp.http.alt == 0 + expect resp.http.warmup == "-1.000" + expect resp.http.rampup == "true" + expect resp.http.healthy == "IGNORE" +} -run diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc new file mode 100644 index 0000000..c29e03f --- /dev/null +++ b/bin/varnishtest/tests/d00021.vtc @@ -0,0 +1,79 @@ +varnishtest "shard director LAZY" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + if (!vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (!vd.add_backend(s2)) { + std.log("add s2 failed"); + } + if (!vd.add_backend(s3)) { + std.log("add s3 failed"); + } + if (!vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + } + + sub vcl_recv { + return(pass); + } + + sub vcl_backend_fetch { + if (bereq.url == "/1") { + set bereq.backend = + vd.backend(resolve=LAZY, by=KEY, key=1); + } + if (bereq.url == "/2") { + set bereq.backend = + vd.backend(resolve=LAZY, by=KEY, key=2147483647); + } + if (bereq.url == "/3") { + set bereq.backend = + vd.backend(resolve=LAZY, by=KEY, key=4294967295); + } + } + + sub vcl_backend_response { + set beresp.http.healthy = std.healthy( + vd.backend(resolve=LAZY, by=KEY, key=1)); + } +} -start + + +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ech3Ooj" + expect resp.http.healthy == "true" + + txreq -url /2 + rxresp + expect resp.body == "ieQu2qua" + expect resp.http.healthy == "true" + + txreq -url /3 + rxresp + expect resp.body == "xiuFi3Pe" + expect resp.http.healthy == "true" +} -run diff --git a/bin/varnishtest/tests/d00029.vtc b/bin/varnishtest/tests/d00029.vtc new file mode 100644 index 0000000..93a1517 --- /dev/null +++ b/bin/varnishtest/tests/d00029.vtc @@ -0,0 +1,88 @@ +varnishtest "shard director LAZY - d18.vtc" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + if (!vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (!vd.add_backend(s2)) { + std.log("add s2 failed"); + } + if (!vd.add_backend(s3)) { + std.log("add s3 failed"); + } + if (!vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + + vd.debug(1); + + new p = directors.shard_param(); + p.set(by=KEY, key=1); + vd.associate(p.use()); + + new p3 = directors.shard_param(); + p3.set(by=KEY, key=4294967295); + } + + sub vcl_recv { + return(pass); + } + + sub vcl_backend_fetch { + set bereq.backend=vd.backend(resolve=LAZY); + + if (bereq.url == "/1") { + # default + } else + if (bereq.url == "/2") { + # backend override parameter set + p.set(by=KEY, key=2147483647); + } else + if (bereq.url == "/3") { + # backend override association + vd.backend(resolve=LAZY, param=p3.use()); + } + } + + sub vcl_backend_response { + set beresp.http.backend = bereq.backend; + } +} -start + + +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ech3Ooj" + expect resp.http.backend == "vd" + + txreq -url /2 + rxresp + expect resp.body == "ieQu2qua" + expect resp.http.backend == "vd" + + txreq -url /3 + rxresp + expect resp.body == "xiuFi3Pe" + expect resp.http.backend == "vd" +} -run diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc new file mode 100644 index 0000000..202c485 --- /dev/null +++ b/bin/varnishtest/tests/d00030.vtc @@ -0,0 +1,186 @@ +varnishtest "shard director error handling" + +varnish v1 -vcl { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new shard = directors.shard(); + new p = directors.shard_param(); + p.set(by=BLOB, key_blob=blob.decode(HEX, encoded="")); + } + + sub vcl_recv { + if (req.url == "/1") { + set req.backend_hint = shard.backend( + param=blob.decode(HEX, encoded="")); + } else if (req.url == "/2") { + p.set(by=HASH); + } + } +} -start + +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI {^Rd vcl.load} + expect 0 0 Error {by=BLOB but no or empty key_blob - using key 0} +} -start -wait + +logexpect l2 -v v1 -g raw { + expect * 1001 VCL_Error {shard .backend param invalid} +} -start +logexpect l3 -v v1 -g raw { + expect * 1003 VCL_Error {shard_param.set.. may only be used in vcl_init and in backend context} +} -start + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 503 + expect_close +} -run + +client c1 { + txreq -url "/2" + rxresp + expect resp.status == 503 + expect_close +} -run + +logexpect l2 -wait +logexpect l3 -wait + +varnish v1 -errvcl {shard .associate param invalid} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new shard = directors.shard(); + shard.associate(blob.decode(encoded="")); + } +} + +varnish v1 -errvcl {missing key argument with by=KEY} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(by=KEY); + } +} + +varnish v1 -errvcl {invalid key argument -5 with by=KEY} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(by=KEY, key=-5); + } +} + +varnish v1 -errvcl {missing key_blob argument with by=BLOB} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(by=BLOB); + } +} + +varnish v1 -errvcl {key and key_blob arguments are invalid with by=URL} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(by=URL, key=0); + } +} + +varnish v1 -errvcl {key and key_blob arguments are invalid with by=HASH (default)} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(key=0); + } +} + +varnish v1 -errvcl {invalid alt argument -1} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(alt=-1); + } +} + +varnish v1 -errvcl {invalid warmup argument -0.5} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(warmup=-0.5); + } +} + +varnish v1 -errvcl {invalid warmup argument 1.1} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new p = directors.shard_param(); + p.set(warmup=1.1); + } +} + +varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backend context} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new shard = directors.shard(); + new rr = directors.round_robin(); + rr.add_backend(shard.backend(resolve=LAZY, by=KEY)); + } +} + +varnish v1 -errvcl {resolve=NOW can not be used in vcl_init} { + import directors; + import blob; + + backend dummy { .host = "${bad_backend}"; } + + sub vcl_init { + new shard = directors.shard(); + new rr = directors.round_robin(); + rr.add_backend(shard.backend()); + } +} diff --git a/doc/changes.rst b/doc/changes.rst index e366f63..4d8fb92 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -104,6 +104,31 @@ VCL and bundled VMODs .backend(by=BLOB, key_blob=blobdigest.hash(RS, blob.decode(encoded=))) +* The ``shard`` director now offers resolution at the time the actual + backend connection is made, which is how all other bundled directors + work as well: With the ``resolve=LAZY`` argument, other shard + parameters are saved for later reference and a director object is + returned. + + This enables layering the shard director below other directors. + +* The ``shard`` director now also supports getting other parameters + from a parameter set object: Rather than passing the required + parameters with each ``.backend()`` call, an object can be + associated with a shard director defining the parameters. The + association can be changed in ``vcl_backend_fetch()`` and individual + parameters can be overridden in each ``.backend()`` call. + + The main use case is to segregate shard parameters from director + selection: By associating a parameter object with many directors, + the same load balancing decision can easily be applied independent + of which set of backends is to be used. + +* To support parameter overriding, support for positional arguments of + the shard director ``.backend()`` method had to be removed. In other + words, all parameters to the shard director ``.backend()`` method + now need to be named. + Logging / statistics -------------------- diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index 1275938..f4a59e2 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -13,7 +13,8 @@ libvmod_directors_la_SOURCES = \ shard_dir.c \ shard_dir.h \ tbl_by.h \ - tbl_healthy.h + tbl_healthy.h \ + tbl_resolve.h # Use vmodtool.py generated automake boilerplate include $(srcdir)/automake_boilerplate.am diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index ff5f9af..7a667c3 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -304,27 +304,39 @@ init_state(struct shard_state *state, state->last.hostid = -1; } +/* basically same as vdir_any_healthy + * - XXX we should embed a vdir + * - XXX should we return the health state of the actual backend + * for healthy=IGNORE ? + */ +unsigned +sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, + double *changed) +{ + unsigned retval = 0; + VCL_BACKEND be; + unsigned u; + double c; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + sharddir_rdlock(shardd); + if (changed != NULL) + *changed = 0; + for (u = 0; u < shardd->n_backend; u++) { + be = shardd->backend[u].backend; + CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); + retval = be->healthy(be, bo, &c); + if (changed != NULL && c > *changed) + *changed = c; + if (retval) + break; + } + sharddir_unlock(shardd); + return (retval); +} /* - * core function for the director backend method - * - * while other directors return a reference to their own backend object (on - * which varnish will call the resolve method to resolve to a non-director - * backend), this director immediately reolves in the backend method, to make - * the director choice visible in VCL - * - * consequences: - * - we need no own struct director - * - we can only respect a busy object when being called on the backend side, - * which probably is, for all practical purposes, only relevant when the - * saintmode vmod is used - * - * if we wanted to offer delayed resolution, we'd need something like - * per-request per-director state or we'd need to return a dynamically created - * director object. That should be straight forward once we got director - * refcounting #2072. Until then, we could create it on the workspace, but then - * we'd need to keep other directors from storing any references to our dynamic - * object for longer than the current task - * + * core function for the director backend/resolve method */ VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *shardd, diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 4df4151..f1b1b00 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -41,6 +41,13 @@ enum healthy_e { _HEALTHY_E_MAX }; +enum resolve_e { + _RESOLVE_E_INVALID = 0, +#define VMODENUM(x) x, +#include "tbl_resolve.h" + _RESOLVE_E_MAX +}; + struct vbitmap; struct shard_circlepoint { @@ -121,6 +128,8 @@ void sharddir_new(struct sharddir **sharddp, const char *vcl_name); void sharddir_delete(struct sharddir **sharddp); void sharddir_wrlock(struct sharddir *shardd); void sharddir_unlock(struct sharddir *shardd); +unsigned sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, + double *changed); VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT, VCL_REAL, VCL_BOOL, enum healthy_e); diff --git a/lib/libvmod_directors/tbl_resolve.h b/lib/libvmod_directors/tbl_resolve.h new file mode 100644 index 0000000..c7c6d82 --- /dev/null +++ b/lib/libvmod_directors/tbl_resolve.h @@ -0,0 +1,3 @@ +VMODENUM(NOW) +VMODENUM(LAZY) +#undef VMODENUM diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index ef2fd75..c2106a3 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -3,7 +3,7 @@ # itself. See LICENCE for details. # # Copyright (c) 2013-2015 Varnish Software AS -# Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung +# Copyright 2009-2018 UPLEX - Nils Goroll Systemoptimierung # All rights reserved. # # Authors: Poul-Henning Kamp @@ -233,6 +233,9 @@ Note that the shard director needs to be configured using at least one ``shard.add_backend()`` call(s) **followed by a** ``shard.reconfigure()`` **call** before it can hand out backends. +_Note_ that due to various restrictions (documented below), it is +recommended to use the shard director on the backend side. + Introduction ```````````` @@ -354,6 +357,16 @@ Set the default rampup duration. See `rampup` parameter of Default: 0s (no rampup) +$Method VOID .associate(BLOB param=0) + +Associate a default `obj_shard_param`_ object or clear an association. + +The value of the `param` argument must be a call to the +`func_shard_param.use`_ method. No argument clears the association. + +The association can be changed per backend request using the `param` +argument of `func_shard.backend`_. + $Method BOOL .add_backend(PRIV_TASK, BACKEND backend, STRING ident=0, DURATION rampup=973279260) @@ -409,14 +422,15 @@ To generate sharding keys using other hashes, use a custom vmod like .. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst $Method BACKEND .backend( - ENUM {HASH, URL, KEY, BLOB} by=HASH, - INT key=0, - BLOB key_blob=0, - INT alt=0, - REAL warmup=-1, - BOOL rampup=1, - ENUM {CHOSEN, IGNORE, ALL} healthy=CHOSEN) - + [ ENUM {HASH, URL, KEY, BLOB} by ], + [ INT key ], + [ BLOB key_blob ], + [ INT alt ], + [ REAL warmup ], + [ BOOL rampup ], + [ ENUM {CHOSEN, IGNORE, ALL} healthy ], + [ BLOB param ], + [ ENUM {NOW, LAZY} resolve] ) Lookup a backend on the consistent hashing ring. @@ -519,15 +533,149 @@ is _not_ the order given when backends are added. For `alt > 0`, return the `alt`-th alternative backend of all those healthy, the last healthy backend found or none. +* `resolve` + + * ``NOW``: look up a backend and return it. + + Can not be used in ``vcl_init{}``. + + * ``LAZY``: return an instance of this director for later backend resolution. + + ``LAZY`` mode is required for referencing shard director instances, + for example as backends for other directors (director layering). + + In ``vcl_init{}`` and on the client side, ``LAZY`` mode can not be + used with any other argument. + + On the backend side, parameters from arguments or an associated + parameter set affect the shard director instance for the backend + request irrespective of where it is referenced. + +* `param` + + Use or associate a parameter set. The value of the `param` argument + must be a call to the `func_shard_param.use`_ method. + + default: as set by `func_shard.associate`_ or unset. + + * for ``resolve=NOW`` take parameter defaults from the + `obj_shard_param`_ parameter set + + * for ``resolve=LAZY`` associate the `obj_shard_param`_ parameter + set for this backend request + + Implementation notes for use of parameter sets with + ``resolve=LAZY``: + + * A `param` argument remains associated and any changes to the + associated parameter set affect the sharding decision once the + director resolves to an actual backend. + + * If other paramter arguments are also given, they have preference + and are kept even if the parameter set given by the `param` + argument is subsequently changed within the same backend request. + + * Each call to `func_shard.backend`_ overrides any previous call. + $Method VOID .debug(INT) `intentionally undocumented` +$Object shard_param() + +Create a shard parameter set. + +A parameter set allows for re-use of `func_shard.backend`_ arguments +across many shard director instances and simplifies advanced use cases +(e.g. shard director with custom parameters layered below other +directors). + +Parameter sets have two scopes: + +* per-VCL scope defined in ``vcl_init{}`` +* per backend request scope + +The per-VCL scope defines defaults for the per backend scope. Any +changes to a parameter set in backend context only affect the +respective backend request. + +Parameter sets can not be used in client context. + +$Method VOID .clear() + +Reset the parameter set to default values as documented for +`func_shard.backend`_. + +* in ``vcl_init{}``, resets the parameter set default for this VCL +* in backend context, resets the parameter set for this backend + request to the VCL defaults + +This method may not be used in client context + +$Method VOID .set( + [ ENUM {HASH, URL, KEY, BLOB} by ], + [ INT key ], + [ BLOB key_blob ], + [ INT alt ], + [ REAL warmup ], + [ BOOL rampup ], + [ ENUM {CHOSEN, IGNORE, ALL} healthy ]) + +Change the given parameters of a parameter set as documented for +`func_shard.backend`_. + +* in ``vcl_init{}``, changes the parameter set default for this VCL + +* in backend context, changes the parameter set for this backend + request, keeping the defaults set for this VCL for unspecified + arguments. + +This method may not be used in client context + +$Method STRING .get_by() + +Get a string representation of the `by` enum argument which denotes +how a shard director using this parameter object would derive the +shard key. See `func_shard.backend`_. + +$Method INT .get_key() + +Get the key which a shard director using this parameter object would +use. See `func_shard.backend`_. + +$Method INT .get_alt() + +Get the `alt` paramter which a shard director using this parameter +object would use. See `func_shard.backend`_. + +$Method REAL .get_warmup() + +Get the `warmup` paramter which a shard director using this parameter +object would use. See `func_shard.backend`_. + +$Method BOOL .get_rampup() + +Get the `rampup` paramter which a shard director using this parameter +object would use. See `func_shard.backend`_. + +$Method STRING .get_healthy() + +Get a string representation of the `healthy` enum argument which a +shard director using this parameter object would use. See +`func_shard.backend`_. + +$Method BLOB .use() + +This method may only be used in backend context. + +For use with the `param` argument of `func_shard.backend`_ to associate +this shard parameter set with a shard director. + ACKNOWLEDGEMENTS ================ -Development of a previous version of the shard director was partly sponsored -by Deutsche Telekom AG - Products & Innovation. +Development of a previous version of the shard director was partly +sponsored by Deutsche Telekom AG - Products & Innovation. -Development of this version of the shard director was partly sponsored -by BILD GmbH & Co KG. +Development of a previous version of the shard director was partly +sponsored by BILD GmbH & Co KG. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 655e2ad..2fe2444 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -1,5 +1,5 @@ /*- - * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung + * Copyright 2009-2018 UPLEX - Nils Goroll Systemoptimierung * All rights reserved. * * Authors: Julian Wiesener @@ -33,6 +33,7 @@ #include #include "cache/cache.h" +#include "cache/cache_director.h" #include "vcl.h" #include "vend.h" @@ -41,17 +42,149 @@ #include "shard_dir.h" #include "shard_cfg.h" +/* ------------------------------------------------------------------------- + * method arguments and set parameters bitmask in vmod_directors_shard_param + */ + +#define arg_by ((uint32_t)1) +#define arg_key ((uint32_t)1 << 1) +#define arg_key_blob ((uint32_t)1 << 2) +#define arg_alt ((uint32_t)1 << 3) +#define arg_warmup ((uint32_t)1 << 4) +#define arg_rampup ((uint32_t)1 << 5) +#define arg_healthy ((uint32_t)1 << 6) +#define arg_param ((uint32_t)1 << 7) +#define arg_resolve ((uint32_t)1 << 8) +#define _arg_mask ((arg_resolve << 1) - 1) +/* allowed in shard_param.set */ +#define _arg_mask_set (arg_param - 1) +/* allowed in shard_param */ +#define _arg_mask_param ( _arg_mask_set \ + & ~arg_key \ + & ~arg_key_blob ) + +/* ------------------------------------------------------------------------- + * shard parameters - declaration & defaults + */ +enum vmod_directors_shard_param_scope { + _SCOPE_INVALID = 0, + VMOD, + VCL, + TASK, + STACK +}; + +struct vmod_directors_shard_param; + +struct vmod_directors_shard_param { + unsigned magic; +#define VMOD_SHARD_SHARD_PARAM_MAGIC 0xdf5ca117 + + /* internals */ + uint32_t key; + const char *vcl_name; + const struct vmod_directors_shard_param *defaults; + enum vmod_directors_shard_param_scope scope; + + /* paramters */ + enum by_e by; + enum healthy_e healthy; + uint32_t mask; + VCL_BOOL rampup; + VCL_INT alt; + VCL_REAL warmup; +}; + +static const struct vmod_directors_shard_param shard_param_default = { + .magic = VMOD_SHARD_SHARD_PARAM_MAGIC, + + .key = 0, + .vcl_name = "builtin defaults", + .defaults = NULL, + .scope = VMOD, + + .mask = _arg_mask_param, + .by = BY_HASH, + .healthy = CHOSEN, + .rampup = 1, + .alt = 0, + .warmup = -1, +}; + +static struct vmod_directors_shard_param * +shard_param_stack(struct vmod_directors_shard_param *p, + const struct vmod_directors_shard_param *pa, const char *who); + +static struct vmod_directors_shard_param * +shard_param_task(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *pa); + +static const struct vmod_directors_shard_param * +shard_param_blob(const VCL_BLOB blob); + +static const struct vmod_directors_shard_param * +vmod_shard_param_read(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *p, + struct vmod_directors_shard_param *pstk, const char *who); + +/* ------------------------------------------------------------------------- + * shard vmod interface + */ +static unsigned v_matchproto_(vdi_healthy) +vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, + double *changed); + +static const struct director * v_matchproto_(vdi_resolve_f) +vmod_shard_resolve(const struct director *dir, struct worker *wrk, + struct busyobj *bo); + struct vmod_directors_shard { - unsigned magic; -#define VMOD_SHARD_SHARD_MAGIC 0x6e63e1bf - struct sharddir *shardd; + unsigned magic; +#define VMOD_SHARD_SHARD_MAGIC 0x6e63e1bf + struct sharddir *shardd; + struct director *dir; + const struct vmod_directors_shard_param *param; }; -VCL_VOID v_matchproto_(td_directors_shard__init) -vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, - const char *vcl_name) +static enum by_e +parse_by_e(VCL_ENUM e) +{ +#define VMODENUM(n) if (e == vmod_enum_ ## n) return(BY_ ## n); +#include "tbl_by.h" + WRONG("illegal by enum"); +} + +static enum healthy_e +parse_healthy_e(VCL_ENUM e) +{ +#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#include "tbl_healthy.h" + WRONG("illegal healthy enum"); +} + +static enum resolve_e +parse_resolve_e(VCL_ENUM e) +{ +#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#include "tbl_resolve.h" + WRONG("illegal resolve enum"); +} + +static const char * const by_str[_BY_E_MAX] = { + [_BY_E_INVALID] = "*INVALID*", +#define VMODENUM(n) [BY_ ## n] = #n, +#include "tbl_by.h" +}; + +static const char * const healthy_str[_HEALTHY_E_MAX] = { + [_HEALTHY_E_INVALID] = "*INVALID*", +#define VMODENUM(n) [n] = #n, +#include "tbl_healthy.h" +}; + +static void +shard__assert(void) { - struct vmod_directors_shard *vshard; VCL_INT t1; uint32_t t2a, t2b; @@ -61,7 +194,15 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, t1 = (VCL_INT)t2a; t2b = (uint32_t)t1; assert(t2a == t2b); +} + +VCL_VOID v_matchproto_(td_directors_shard__init) +vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, + const char *vcl_name) +{ + struct vmod_directors_shard *vshard; + shard__assert(); (void)ctx; AN(vshardp); AZ(*vshardp); @@ -70,6 +211,15 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, *vshardp = vshard; sharddir_new(&vshard->shardd, vcl_name); + + vshard->param = &shard_param_default; + ALLOC_OBJ(vshard->dir, DIRECTOR_MAGIC); + AN(vshard->dir); + REPLACE(vshard->dir->vcl_name, vcl_name); + vshard->dir->priv = vshard; + vshard->dir->resolve = vmod_shard_resolve; + vshard->dir->healthy = vmod_shard_healthy; + vshard->dir->admin_health = VDI_AH_HEALTHY; } VCL_VOID v_matchproto_(td_directors_shard__fini) @@ -80,6 +230,8 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp) *vshardp = NULL; CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); sharddir_delete(&vshard->shardd); + free(vshard->dir->vcl_name); + FREE_OBJ(vshard->dir); FREE_OBJ(vshard); } @@ -121,6 +273,28 @@ vmod_shard_set_rampup(VRT_CTX, struct vmod_directors_shard *vshard, shardcfg_set_rampup(vshard->shardd, duration); } +VCL_VOID v_matchproto_(td_directors_shard_associate) +vmod_shard_associate(VRT_CTX, + struct vmod_directors_shard *vshard, VCL_BLOB b) +{ + const struct vmod_directors_shard_param *ppt; + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + + if (b == NULL) { + vshard->param = &shard_param_default; + return; + } + + ppt = shard_param_blob(b); + + if (ppt == NULL) { + VRT_fail(ctx, "shard .associate param invalid"); + return; + } + + vshard->param = ppt; +} + VCL_BOOL v_matchproto_(td_directors_shard_add_backend) vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, struct vmod_priv *priv, @@ -172,14 +346,11 @@ vmod_shard_reconfigure(VRT_CTX, struct vmod_directors_shard *vshard, } static inline uint32_t -get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) +shard_get_key(VRT_CTX, const struct vmod_directors_shard_param *p) { struct http *http; - uint8_t k[4] = { 0 }; - uint8_t *b; - int i, ki; - switch (by) { + switch (p->by) { case BY_HASH: if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -196,92 +367,355 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) return (sharddir_sha256(http->hd[HTTP_HDR_URL].b, vrt_magic_string_end)); case BY_KEY: - return ((uint32_t)key_int); case BY_BLOB: - assert(key_blob); - assert(key_blob->len > 0); - assert(key_blob->priv != NULL); + return (p->key); + default: + WRONG("by enum"); + } +} - if (key_blob->len >= 4) - ki = 0; - else - ki = 4 - key_blob->len; +/* + * merge parameters to resolve all undef values + * key is to be calculated after merging + */ +static void +shard_param_merge(struct vmod_directors_shard_param *to, + const struct vmod_directors_shard_param *from) +{ + CHECK_OBJ_NOTNULL(to, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert((to->mask & ~_arg_mask_param) == 0); - b = key_blob->priv; - for (i = 0; ki < 4; i++, ki++) - k[ki] = b[i]; - assert(i <= key_blob->len); + if (to->mask == _arg_mask_param) + return; - return (vbe32dec(k)); - default: - WRONG("by value"); + CHECK_OBJ_NOTNULL(from, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert((from->mask & ~_arg_mask_param) == 0); + + if ((to->mask & arg_by) == 0 && (from->mask & arg_by) != 0) { + to->by = from->by; + if (from->by == BY_KEY || from->by == BY_BLOB) + to->key = from->key; } + +#define mrg(to, from, field) do { \ + if (((to)->mask & arg_ ## field) == 0 && \ + ((from)->mask & arg_ ## field) != 0) \ + (to)->field = (from)->field; \ + } while(0) + + mrg(to, from, healthy); + mrg(to, from, rampup); + mrg(to, from, alt); + mrg(to, from, warmup); +#undef mrg + + to->mask |= from->mask; + + if (to->mask == _arg_mask_param) + return; + + AN(from->defaults); + shard_param_merge(to, from->defaults); } -static enum by_e -parse_by_e(VCL_ENUM e) +static uint32_t +shard_blob_key(VCL_BLOB key_blob) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(BY_ ## n); -#include "tbl_by.h" - WRONG("illegal by enum"); + uint8_t k[4] = { 0 }; + uint8_t *b; + int i, ki; + + assert(key_blob); + assert(key_blob->len > 0); + assert(key_blob->priv != NULL); + + if (key_blob->len >= 4) + ki = 0; + else + ki = 4 - key_blob->len; + + b = key_blob->priv; + for (i = 0; ki < 4; i++, ki++) + k[ki] = b[i]; + assert(i <= key_blob->len); + + return (vbe32dec(k)); } -static enum healthy_e -parse_healthy_e(VCL_ENUM e) +/* + * convert vmod interface valid_* to our bitmask + */ + +#define tobit(args, name) ((args)->valid_##name ? arg_##name : 0) + +static uint32_t +shard_backend_arg_mask(const struct vmod_shard_backend_arg * const a) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); -#include "tbl_healthy.h" - WRONG("illegal healthy enum"); + return (tobit(a, by) | + tobit(a, key) | + tobit(a, key_blob) | + tobit(a, alt) | + tobit(a, warmup) | + tobit(a, rampup) | + tobit(a, healthy) | + tobit(a, param) | + tobit(a, resolve)); +} +static uint32_t +shard_param_set_mask(const struct vmod_shard_param_set_arg * const a) +{ + return (tobit(a, by) | + tobit(a, key) | + tobit(a, key_blob) | + tobit(a, alt) | + tobit(a, warmup) | + tobit(a, rampup) | + tobit(a, healthy)); +} +#undef tobit + +/* + * check arguments and return in a struct param + */ +static struct vmod_directors_shard_param * +shard_param_args(VRT_CTX, + struct vmod_directors_shard_param *p, const char *who, + uint32_t args, VCL_ENUM by_s, VCL_INT key_int, VCL_BLOB key_blob, + VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup, VCL_ENUM healthy_s) +{ + enum by_e by; + enum healthy_e healthy; + + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + AN(p->vcl_name); + + assert((args & ~_arg_mask_set) == 0); + + by = (args & arg_by) ? parse_by_e(by_s) : BY_HASH; + healthy = (args & arg_healthy) ? parse_healthy_e(healthy_s) : CHOSEN; + + /* by_s / key_int / key_blob */ + if (args & arg_by) { + switch (by) { + case BY_KEY: + if ((args & arg_key) == 0) { + VRT_fail(ctx, "%s %s: " + "missing key argument with by=%s", + who, p->vcl_name, by_s); + return (NULL); + } + if (key_int < 0 || key_int > UINT32_MAX) { + VRT_fail(ctx, "%s %s: " + "invalid key argument %ld with by=%s", + who, p->vcl_name, key_int, by_s); + return (NULL); + } + assert(key_int >= 0); + assert(key_int <= UINT32_MAX); + p->key = (uint32_t)key_int; + break; + case BY_BLOB: + if ((args & arg_key_blob) == 0) { + VRT_fail(ctx, "%s %s: " + "missing key_blob argument with by=%s", + who, p->vcl_name, by_s); + return (NULL); + } + if (key_blob == NULL || key_blob->len <= 0 || + key_blob->priv == NULL) { + sharddir_err(ctx, SLT_Error, "%s %s: " + "by=BLOB but no or empty key_blob " + "- using key 0", + who, p->vcl_name); + p->key = 0; + } else + p->key = shard_blob_key(key_blob); + break; + case BY_HASH: + case BY_URL: + if (args & (arg_key|arg_key_blob)) { + VRT_fail(ctx, "%s %s: " + "key and key_blob arguments are " + "invalid with by=%s", + who, p->vcl_name, by_s); + return (NULL); + } + break; + default: + WRONG("by enum"); + } + p->by = by; + } else { + /* (args & arg_by) == 0 */ + p->by = BY_HASH; + + if (args & (arg_key|arg_key_blob)) { + VRT_fail(ctx, "%s %s: " + "key and key_blob arguments are " + "invalid with by=HASH (default)", + who, p->vcl_name); + return (NULL); + } + } + + if (args & arg_alt) { + if (alt < 0) { + VRT_fail(ctx, "%s %s: " + "invalid alt argument %ld", + who, p->vcl_name, alt); + return (NULL); + } + p->alt = alt; + } + + if (args & arg_warmup) { + if ((warmup < 0 && warmup != -1) || warmup > 1) { + VRT_fail(ctx, "%s %s: " + "invalid warmup argument %f", + who, p->vcl_name, warmup); + return (NULL); + } + p->warmup = warmup; + } + + if (args & arg_rampup) + p->rampup = !!rampup; + + if (args & arg_healthy) + p->healthy = healthy; + + p->mask = args & _arg_mask_param; + return (p); } VCL_BACKEND v_matchproto_(td_directors_shard_backend) vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, - VCL_ENUM by_s, VCL_INT key_int, VCL_BLOB key_blob, VCL_INT alt, - VCL_REAL warmup, VCL_BOOL rampup, VCL_ENUM healthy_s) + struct vmod_shard_backend_arg *a) { - enum by_e by = parse_by_e(by_s); - enum healthy_e healthy = parse_healthy_e(healthy_s); - - uint32_t key; + struct vmod_directors_shard_param pstk; + struct vmod_directors_shard_param *pp = NULL; + const struct vmod_directors_shard_param *ppt; + enum resolve_e resolve; + uint32_t args = shard_backend_arg_mask(a); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + assert((args & ~_arg_mask) == 0); + resolve = (args & arg_resolve) ? parse_resolve_e(a->resolve) : NOW; + + switch (resolve) { + case LAZY: + if ((ctx->method & VCL_MET_TASK_B) == 0) { + if ((args & ~arg_resolve) != 0) { + VRT_fail(ctx, + "shard .backend resolve=LAZY " + "with other parameters can " + "only be used in backend " + "context"); + return (NULL); + } + AN(vshard->dir); + return (vshard->dir); + } - /* TODO #2500 */ - if ((ctx->method & (VCL_MET_TASK_C | VCL_MET_TASK_B)) == 0) { - VRT_fail(ctx, "shard .backend() method may only be used " - "in client and backend context"); - return NULL; - } + assert(ctx->method & VCL_MET_TASK_B); - if (key_int && by != BY_KEY) { - shard_err(ctx, vshard->shardd, - "by=%s but key argument used", by_s); - return NULL; - } + if ((args & ~arg_resolve) == 0) { + /* no other parameters - shortcut */ + AN(vshard->dir); + return (vshard->dir); + } - if (key_blob && by != BY_BLOB) { - shard_err(ctx, vshard->shardd, - "by=%s but key_blob argument used", by_s); - return NULL; + pp = shard_param_task(ctx, vshard, vshard->param); + if (pp == NULL) + return (NULL); + pp->vcl_name = vshard->shardd->name; + break; + case NOW: + if (ctx->method & VCL_MET_TASK_H) { + VRT_fail(ctx, + "shard .backend resolve=NOW can not be " + "used in vcl_init{}/vcl_fini{}"); + return (NULL); + } + pp = shard_param_stack(&pstk, vshard->param, + vshard->shardd->name); + break; + default: + WRONG("resolve enum"); } - if (by == BY_BLOB) { - if (key_blob == NULL || - key_blob->len <= 0 || - key_blob->priv == NULL) { - shard_err0(ctx, vshard->shardd, - "by=BLOB but no or empty key_blob " - "- using key 0"); - by = BY_KEY; - key_int = 0; + AN(pp); + if (args & arg_param) { + ppt = shard_param_blob(a->param); + if (ppt == NULL) { + VRT_fail(ctx, "shard .backend param invalid"); + return (NULL); } + pp->defaults = ppt; } - key = get_key(ctx, by, key_int, key_blob); + pp = shard_param_args(ctx, pp, "shard.backend()", + args & _arg_mask_set, + a->by, a->key, a->key_blob, a->alt, a->warmup, + a->rampup, a->healthy); + if (pp == NULL) + return (NULL); + if (resolve == LAZY) + return (vshard->dir); + + assert(resolve == NOW); + shard_param_merge(pp, pp->defaults); return (sharddir_pick_be(ctx, vshard->shardd, - key, alt, warmup, rampup, healthy)); + shard_get_key(ctx, pp), pp->alt, pp->warmup, + pp->rampup, pp->healthy)); +} + +static unsigned v_matchproto_(vdi_healthy) +vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, + double *changed) +{ + struct vmod_directors_shard *vshard; + + CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); + return (sharddir_any_healthy(vshard->shardd, bo, changed)); +} + +static const struct director * v_matchproto_(vdi_resolve_f) +vmod_shard_resolve(const struct director *dir, struct worker *wrk, + struct busyobj *bo) +{ + struct vmod_directors_shard *vshard; + struct vmod_directors_shard_param pstk[1]; + const struct vmod_directors_shard_param *pp; + struct vrt_ctx ctx[1]; + + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); + + // Ref: vcl_call_method() + INIT_OBJ(ctx, VRT_CTX_MAGIC); + ctx->vsl = bo->vsl; + ctx->vcl = bo->vcl; + ctx->http_bereq = bo->bereq; + ctx->http_beresp = bo->beresp; + ctx->bo = bo; + ctx->sp = bo->sp; + ctx->now = bo->t_prev; + ctx->ws = bo->ws; + ctx->method = VCL_MET_BACKEND_FETCH; + + pp = vmod_shard_param_read(ctx, vshard, + vshard->param, pstk, "shard_resolve"); + if (pp == NULL) + return (NULL); + + return (sharddir_pick_be(ctx, vshard->shardd, + shard_get_key(ctx, pp), pp->alt, pp->warmup, + pp->rampup, pp->healthy)); } VCL_VOID v_matchproto_(td_directors_shard_backend) @@ -293,3 +727,296 @@ vmod_shard_debug(VRT_CTX, struct vmod_directors_shard *vshard, (void)ctx; sharddir_debug(vshard->shardd, i & UINT32_MAX); } + +/* ============================================================= + * shard_param + */ + +VCL_VOID v_matchproto_(td_directors_shard_param__init) +vmod_shard_param__init(VRT_CTX, + struct vmod_directors_shard_param **pp, const char *vcl_name) +{ + struct vmod_directors_shard_param *p; + + (void) ctx; + AN(pp); + AZ(*pp); + ALLOC_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + AN(p); + p->vcl_name = vcl_name; + p->scope = VCL; + p->defaults = &shard_param_default; + + *pp = p; +} + +VCL_VOID v_matchproto_(td_directors_shard_param__fini) +vmod_shard_param__fini(struct vmod_directors_shard_param **pp) +{ + struct vmod_directors_shard_param *p = *pp; + + if (p == NULL) + return; + *pp = NULL; + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + FREE_OBJ(p); +} + +/* + * init a stack param struct defaulting to pa with the given name + */ +static struct vmod_directors_shard_param * +shard_param_stack(struct vmod_directors_shard_param *p, + const struct vmod_directors_shard_param *pa, const char *who) +{ + CHECK_OBJ_NOTNULL(pa, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert(pa->scope > _SCOPE_INVALID); + + AN(p); + INIT_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + p->vcl_name = who; + p->scope = STACK; + p->defaults = pa; + + return (p); +} +/* + * get a task scoped param struct for id defaulting to pa + * if id != pa and pa has VCL scope, also get a task scoped param struct for pa + */ +static struct vmod_directors_shard_param * +shard_param_task(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *pa) +{ + struct vmod_directors_shard_param *p; + struct vmod_priv *task; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(pa, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert(pa->scope > _SCOPE_INVALID); + + task = VRT_priv_task(ctx, id); + + if (task == NULL) { + VRT_fail(ctx, "no priv_task"); + return (NULL); + } + + if (task->priv) { + p = task->priv; + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert(p->scope == TASK); + /* XXX + VSL(SLT_Debug, 0, + "shard_param_task(id %p, pa %p) = %p (found, ws=%p)", + id, pa, p, ctx->ws); + */ + return (p); + } + + p = WS_Alloc(ctx->ws, sizeof *p); + if (p == NULL) { + VRT_fail(ctx, "shard_param_task WS_Alloc failed"); + return (NULL); + } + task->priv = p; + INIT_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + p->vcl_name = pa->vcl_name; + p->scope = TASK; + + if (id == pa || pa->scope != VCL) + p->defaults = pa; + else + p->defaults = shard_param_task(ctx, pa, pa); + + /* XXX + VSL(SLT_Debug, 0, + "shard_param_task(id %p, pa %p) = %p (new, defaults = %p, ws=%p)", + id, pa, p, p->defaults, ctx->ws); + */ + return (p); +} + +static struct vmod_directors_shard_param * +shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p, + const char *who) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + + if (ctx->method & VCL_MET_TASK_C) { + VRT_fail(ctx, "%s may only be used " + "in vcl_init and in backend context", who); + return (NULL); + } else if (ctx->method & VCL_MET_TASK_B) + p = shard_param_task(ctx, p, p); + else + assert(ctx->method & VCL_MET_TASK_H); + + return (p); +} + +VCL_VOID v_matchproto_(td_directors_shard_param_set) +vmod_shard_param_set(VRT_CTX, struct vmod_directors_shard_param *p, + struct vmod_shard_param_set_arg *a) +{ + uint32_t args = shard_param_set_mask(a); + + assert((args & ~_arg_mask_set) == 0); + + p = shard_param_prep(ctx, p, "shard_param.set()"); + if (p == NULL) + return; + (void) shard_param_args(ctx, p, "shard_param.set()", args, + a->by, a->key, a->key_blob, a->alt, a->warmup, + a->rampup, a->healthy); +} + +VCL_VOID v_matchproto_(td_directors_shard_param_clear) +vmod_shard_param_clear(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + p = shard_param_prep(ctx, p, "shard_param.clear()"); + if (p == NULL) + return; + p->mask = 0; +} + +static const struct vmod_directors_shard_param * +vmod_shard_param_read(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *p, + struct vmod_directors_shard_param *pstk, const char *who) +{ + struct vmod_directors_shard_param *pp; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + (void) who; // XXX + + if (ctx->method & VCL_MET_TASK_B) + p = shard_param_task(ctx, id, p); + + if (p == NULL) + return (NULL); + + pp = shard_param_stack(pstk, p, p->vcl_name); + AN(pp); + shard_param_merge(pp, p); + return (pp); +} + +VCL_STRING v_matchproto_(td_directors_shard_param_get_by) +vmod_shard_param_get_by(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_by()"); + if (pp == NULL) + return (NULL); + assert(pp->by > _BY_E_INVALID); + return (by_str[pp->by]); +} + +VCL_INT v_matchproto_(td_directors_shard_param_get_key) +vmod_shard_param_get_key(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_key()"); + if (pp == NULL) + return (-1); + return ((VCL_INT)shard_get_key(ctx, pp)); +} +VCL_INT v_matchproto_(td_directors_shard_param_get_alt) +vmod_shard_param_get_alt(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, + "shard_param.get_alt()"); + if (pp == NULL) + return (-1); + return (pp->alt); +} + +VCL_REAL v_matchproto_(td_directors_shard_param_get_warmup) +vmod_shard_param_get_warmup(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, + "shard_param.get_warmup()"); + if (pp == NULL) + return (-2); + return (pp->warmup); +} + +VCL_BOOL v_matchproto_(td_directors_shard_param_get_rampup) +vmod_shard_param_get_rampup(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, + "shard_param.get_rampup()"); + if (pp == NULL) + return (0); + return (pp->rampup); +} + +VCL_STRING v_matchproto_(td_directors_shard_param_get_healthy) +vmod_shard_param_get_healthy(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_directors_shard_param pstk; + const struct vmod_directors_shard_param *pp; + + pp = vmod_shard_param_read(ctx, p, p, &pstk, + "shard_param.get_healthy()"); + if (pp == NULL) + return (NULL); + assert(pp->healthy > _HEALTHY_E_INVALID); + return (healthy_str[pp->healthy]); + +} + +static const struct vmod_directors_shard_param * +shard_param_blob(const VCL_BLOB blob) +{ + if (blob && blob->priv && + blob->len == sizeof(struct vmod_directors_shard_param) && + *(unsigned *)blob->priv == VMOD_SHARD_SHARD_PARAM_MAGIC) + return (blob->priv); + return (NULL); +} + +VCL_BLOB v_matchproto_(td_directors_shard_param_use) +vmod_shard_param_use(VRT_CTX, + struct vmod_directors_shard_param *p) +{ + struct vmod_priv *blob; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + + blob = (void *)WS_Alloc(ctx->ws, sizeof *blob); + if (blob == NULL) { + VRT_fail(ctx, "Workspace overflow (param.use())"); + return (NULL); + } + + memset(blob, 0, sizeof *blob); + blob->len = sizeof *p; + blob->priv = p; + + return (blob); +} From phk at FreeBSD.org Tue Mar 6 09:01:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 09:01:07 +0000 (UTC) Subject: [master] 19d5f76 This test depends on the specific hash-key, which again depends on the client-varnish connection happening over 127.0.0.1. Fake it for jails and IPv6-oid hosts. Message-ID: <20180306090107.C055AA7FBF@lists.varnish-cache.org> commit 19d5f762f87d892a5072acf12dbc45d21ee5b92f Author: Poul-Henning Kamp Date: Tue Mar 6 08:59:41 2018 +0000 This test depends on the specific hash-key, which again depends on the client-varnish connection happening over 127.0.0.1. Fake it for jails and IPv6-oid hosts. diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc index 358a623..5e99f37 100644 --- a/bin/varnishtest/tests/d00020.vtc +++ b/bin/varnishtest/tests/d00020.vtc @@ -28,6 +28,12 @@ varnish v1 -vcl+backend { key_blob=blob.decode(HEX, encoded="ffffffff00")); } + sub vcl_hash { + hash_data(req.url); + hash_data("127.0.0.1"); + return (lookup); + } + sub vcl_synth { if (req.url == "/def") { set resp.http.sha256 = shard.key(req.url); From nils.goroll at uplex.de Tue Mar 6 10:02:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 10:02:08 +0000 (UTC) Subject: [master] 909c118 whitespace Message-ID: <20180306100208.C56E9A919E@lists.varnish-cache.org> commit 909c1182ba533e4e5d8cc793b8e32bba87ca7476 Author: Nils Goroll Date: Tue Mar 6 11:01:46 2018 +0100 whitespace diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 2fe2444..0e90a40 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -47,9 +47,9 @@ */ #define arg_by ((uint32_t)1) -#define arg_key ((uint32_t)1 << 1) +#define arg_key ((uint32_t)1 << 1) #define arg_key_blob ((uint32_t)1 << 2) -#define arg_alt ((uint32_t)1 << 3) +#define arg_alt ((uint32_t)1 << 3) #define arg_warmup ((uint32_t)1 << 4) #define arg_rampup ((uint32_t)1 << 5) #define arg_healthy ((uint32_t)1 << 6) From phk at FreeBSD.org Tue Mar 6 10:12:06 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 10:12:06 +0000 (UTC) Subject: [master] a60d8d9 Express variable implementations in terms of VCL_* types Message-ID: <20180306101206.A10A8A94D6@lists.varnish-cache.org> commit a60d8d9577f802d82874a212dfdc61af449e47ec Author: Poul-Henning Kamp Date: Tue Mar 6 10:06:51 2018 +0000 Express variable implementations in terms of VCL_* types diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index d781ce7..f5f305f 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -68,7 +68,7 @@ vrt_do_string(VRT_CTX, struct http *hp, int fld, } #define VRT_HDR_L(obj, hdr, fld) \ -void \ +VCL_VOID \ VRT_l_##obj##_##hdr(VRT_CTX, const char *p, ...) \ { \ va_list ap; \ @@ -80,7 +80,7 @@ VRT_l_##obj##_##hdr(VRT_CTX, const char *p, ...) \ } #define VRT_HDR_R(obj, hdr, fld) \ -const char * \ +VCL_STRING \ VRT_r_##obj##_##hdr(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -93,24 +93,27 @@ VRT_r_##obj##_##hdr(VRT_CTX) \ VRT_HDR_R(obj, hdr, fld) #define VRT_STATUS_L(obj) \ -void \ -VRT_l_##obj##_status(VRT_CTX, long num) \ +VCL_VOID \ +VRT_l_##obj##_status(VRT_CTX, VCL_INT num) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC); \ if (num < 0) \ - VRT_fail(ctx, "%s.status (%ld) is negative", #obj, num); \ + VRT_fail(ctx, "%s.status (%jd) is negative", \ + #obj, (intmax_t)num); \ else if (num > 65535) \ - VRT_fail(ctx, "%s.status (%ld) > 65535", #obj, num); \ + VRT_fail(ctx, "%s.status (%jd) > 65535", \ + #obj, (intmax_t)num); \ else if ((num % 1000) < 100) \ - VRT_fail(ctx, "illegal %s.status (%ld) (..0##)", #obj, num); \ + VRT_fail(ctx, "illegal %s.status (%jd) (..0##)", \ + #obj, (intmax_t)num); \ else \ http_SetStatus(ctx->http_##obj, (uint16_t)num); \ } #define VRT_STATUS_R(obj) \ -long \ +VCL_INT \ VRT_r_##obj##_status(VRT_CTX) \ { \ \ @@ -144,7 +147,7 @@ VRT_STATUS_R(beresp) * Pulling things out of the packed object->http */ -long +VCL_INT VRT_r_obj_status(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -154,7 +157,7 @@ VRT_r_obj_status(VRT_CTX) return (HTTP_GetStatusPack(ctx->req->wrk, ctx->req->objcore)); } -const char * +VCL_STRING VRT_r_obj_proto(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -165,7 +168,7 @@ VRT_r_obj_proto(VRT_CTX) H__Proto)); } -const char * +VCL_STRING VRT_r_obj_reason(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -183,7 +186,7 @@ VRT_r_obj_reason(VRT_CTX) #define VBERESPW0(field) #define VBERESPW1(field) \ void \ -VRT_l_beresp_##field(VRT_CTX, unsigned a) \ +VRT_l_beresp_##field(VRT_CTX, VCL_BOOL a) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ @@ -192,7 +195,7 @@ VRT_l_beresp_##field(VRT_CTX, unsigned a) \ #define VBERESPR0(field) #define VBERESPR1(field) \ -unsigned \ +VCL_BOOL \ VRT_r_beresp_##field(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -207,7 +210,7 @@ VRT_r_beresp_##field(VRT_CTX) \ /*--------------------------------------------------------------------*/ -unsigned +VCL_BOOL VRT_r_bereq_is_bgfetch(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -215,7 +218,7 @@ VRT_r_bereq_is_bgfetch(VRT_CTX) return (ctx->bo->is_bgfetch); } -unsigned +VCL_BOOL VRT_r_bereq_uncacheable(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -223,8 +226,8 @@ VRT_r_bereq_uncacheable(VRT_CTX) return (ctx->bo->do_pass); } -void -VRT_l_beresp_uncacheable(VRT_CTX, unsigned a) +VCL_VOID +VRT_l_beresp_uncacheable(VRT_CTX, VCL_BOOL a) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -237,7 +240,7 @@ VRT_l_beresp_uncacheable(VRT_CTX, unsigned a) } } -unsigned +VCL_BOOL VRT_r_beresp_uncacheable(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -247,7 +250,7 @@ VRT_r_beresp_uncacheable(VRT_CTX) /*--------------------------------------------------------------------*/ -const char * +VCL_STRING VRT_r_client_identity(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -257,7 +260,7 @@ VRT_r_client_identity(VRT_CTX) return (SES_Get_String_Attr(ctx->req->sp, SA_CLIENT_IP)); } -void +VCL_VOID VRT_l_client_identity(VRT_CTX, const char *str, ...) { va_list ap; @@ -279,8 +282,8 @@ VRT_l_client_identity(VRT_CTX, const char *str, ...) /*--------------------------------------------------------------------*/ #define BEREQ_TIMEOUT(which) \ -void \ -VRT_l_bereq_##which(VRT_CTX, double num) \ +VCL_VOID \ +VRT_l_bereq_##which(VRT_CTX, VCL_DURATION num) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -288,7 +291,7 @@ VRT_l_bereq_##which(VRT_CTX, double num) \ ctx->bo->which = (num > 0.0 ? num : 0.0); \ } \ \ -double \ +VCL_DURATION \ VRT_r_bereq_##which(VRT_CTX) \ { \ \ @@ -303,7 +306,7 @@ BEREQ_TIMEOUT(between_bytes_timeout) /*--------------------------------------------------------------------*/ -const char * +VCL_STRING VRT_r_beresp_backend_name(VRT_CTX) { @@ -339,7 +342,7 @@ VRT_r_req_storage(VRT_CTX) return (ctx->req->storage); } -void +VCL_VOID VRT_l_req_storage(VRT_CTX, VCL_STEVEDORE stv) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -357,7 +360,7 @@ VRT_r_beresp_storage(VRT_CTX) return (ctx->bo->storage); } -void +VCL_VOID VRT_l_beresp_storage(VRT_CTX, VCL_STEVEDORE stv) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -371,7 +374,7 @@ VRT_l_beresp_storage(VRT_CTX, VCL_STEVEDORE stv) #include "storage/storage.h" -const char * +VCL_STRING VRT_r_beresp_storage_hint(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -382,7 +385,7 @@ VRT_r_beresp_storage_hint(VRT_CTX) return (ctx->bo->storage->vclname); } -void +VCL_VOID VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...) { const char *p; @@ -426,7 +429,7 @@ VRT_r_obj_storage(VRT_CTX) #define REQ_VAR_L(nm, elem, type,extra) \ \ -void \ +VCL_VOID \ VRT_l_req_##nm(VRT_CTX, type arg) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -445,15 +448,15 @@ VRT_r_req_##nm(VRT_CTX) \ return (ctx->req->elem); \ } -REQ_VAR_L(backend_hint, director_hint, const struct director *,) -REQ_VAR_R(backend_hint, director_hint, const struct director *) -REQ_VAR_L(ttl, d_ttl, double, if (!(arg>0.0)) arg = 0;) -REQ_VAR_R(ttl, d_ttl, double) +REQ_VAR_L(backend_hint, director_hint, VCL_BACKEND,) +REQ_VAR_R(backend_hint, director_hint, VCL_BACKEND) +REQ_VAR_L(ttl, d_ttl, VCL_DURATION, if (!(arg>0.0)) arg = 0;) +REQ_VAR_R(ttl, d_ttl, VCL_DURATION) /*--------------------------------------------------------------------*/ -void -VRT_l_bereq_backend(VRT_CTX, const struct director *be) +VCL_VOID +VRT_l_bereq_backend(VRT_CTX, VCL_BACKEND be) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -461,7 +464,7 @@ VRT_l_bereq_backend(VRT_CTX, const struct director *be) ctx->bo->director_req = be; } -const struct director * +VCL_BACKEND VRT_r_bereq_backend(VRT_CTX) { @@ -470,7 +473,7 @@ VRT_r_bereq_backend(VRT_CTX) return (ctx->bo->director_req); } -const struct director * +VCL_BACKEND VRT_r_beresp_backend(VRT_CTX) { @@ -481,7 +484,7 @@ VRT_r_beresp_backend(VRT_CTX) /*--------------------------------------------------------------------*/ -void +VCL_VOID VRT_u_bereq_body(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -495,8 +498,8 @@ VRT_u_bereq_body(VRT_CTX) /*--------------------------------------------------------------------*/ -void -VRT_l_req_esi(VRT_CTX, unsigned process_esi) +VCL_VOID +VRT_l_req_esi(VRT_CTX, VCL_BOOL process_esi) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -510,7 +513,7 @@ VRT_l_req_esi(VRT_CTX, unsigned process_esi) ctx->req->disable_esi = !process_esi; } -unsigned +VCL_BOOL VRT_r_req_esi(VRT_CTX) { @@ -520,7 +523,7 @@ VRT_r_req_esi(VRT_CTX) return (!ctx->req->disable_esi); } -long +VCL_INT VRT_r_req_esi_level(VRT_CTX) { @@ -531,7 +534,7 @@ VRT_r_req_esi_level(VRT_CTX) /*--------------------------------------------------------------------*/ -unsigned +VCL_BOOL VRT_r_req_can_gzip(VRT_CTX) { @@ -542,7 +545,7 @@ VRT_r_req_can_gzip(VRT_CTX) /*--------------------------------------------------------------------*/ -long +VCL_INT VRT_r_req_restarts(VRT_CTX) { @@ -551,7 +554,7 @@ VRT_r_req_restarts(VRT_CTX) return (ctx->req->restarts); } -long +VCL_INT VRT_r_bereq_retries(VRT_CTX) { @@ -572,7 +575,8 @@ VRT_r_bereq_retries(VRT_CTX) * grace&keep are relative to ttl */ -static double ttl_now(VRT_CTX) +static double +ttl_now(VRT_CTX) { if (ctx->bo) { return (ctx->now); @@ -585,8 +589,8 @@ static double ttl_now(VRT_CTX) #define VRT_DO_EXP_L(which, oc, fld, offset) \ \ -void \ -VRT_l_##which##_##fld(VRT_CTX, double a) \ +VCL_VOID \ +VRT_l_##which##_##fld(VRT_CTX, VCL_DURATION a) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -600,7 +604,7 @@ VRT_l_##which##_##fld(VRT_CTX, double a) \ #define VRT_DO_EXP_R(which, oc, fld, offset) \ \ -double \ +VCL_DURATION \ VRT_r_##which##_##fld(VRT_CTX) \ { \ double d; \ @@ -632,7 +636,7 @@ VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore, keep, 0) #define VRT_DO_AGE_R(which, oc) \ \ -double \ +VCL_DURATION \ VRT_r_##which##_##age(VRT_CTX) \ { \ \ @@ -647,7 +651,7 @@ VRT_DO_AGE_R(beresp, ctx->bo->fetch_objcore) * [be]req.xid */ -const char * +VCL_STRING VRT_r_req_xid(VRT_CTX) { @@ -658,7 +662,7 @@ VRT_r_req_xid(VRT_CTX) VXID(ctx->req->vsl->wid))); } -const char * +VCL_STRING VRT_r_bereq_xid(VRT_CTX) { @@ -675,8 +679,8 @@ VRT_r_bereq_xid(VRT_CTX) #define VREQW0(field) #define VREQW1(field) \ -void \ -VRT_l_req_##field(VRT_CTX, unsigned a) \ +VCL_VOID \ +VRT_l_req_##field(VRT_CTX, VCL_BOOL a) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); \ @@ -685,7 +689,7 @@ VRT_l_req_##field(VRT_CTX, unsigned a) \ #define VREQR0(field) #define VREQR1(field) \ -unsigned \ +VCL_BOOL \ VRT_r_req_##field(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -720,7 +724,7 @@ GIP(server) /*--------------------------------------------------------------------*/ -const char* +VCL_STRING VRT_r_server_identity(VRT_CTX) { @@ -731,7 +735,7 @@ VRT_r_server_identity(VRT_CTX) return ("varnishd"); } -const char* +VCL_STRING VRT_r_server_hostname(VRT_CTX) { @@ -743,7 +747,7 @@ VRT_r_server_hostname(VRT_CTX) /*--------------------------------------------------------------------*/ -long +VCL_INT VRT_r_obj_hits(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -752,7 +756,7 @@ VRT_r_obj_hits(VRT_CTX) return (ctx->req->is_hit ? ctx->req->objcore->hits : 0); } -unsigned +VCL_BOOL VRT_r_obj_uncacheable(VRT_CTX) { @@ -764,7 +768,7 @@ VRT_r_obj_uncacheable(VRT_CTX) /*--------------------------------------------------------------------*/ -unsigned +VCL_BOOL VRT_r_resp_is_streaming(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -777,8 +781,8 @@ VRT_r_resp_is_streaming(VRT_CTX) /*--------------------------------------------------------------------*/ -void -VRT_l_resp_do_esi(VRT_CTX, unsigned process_esi) +VCL_VOID +VRT_l_resp_do_esi(VRT_CTX, VCL_BOOL process_esi) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -791,7 +795,7 @@ VRT_l_resp_do_esi(VRT_CTX, unsigned process_esi) ctx->req->disable_esi = !process_esi; } -unsigned +VCL_BOOL VRT_r_resp_do_esi(VRT_CTX) { @@ -804,7 +808,7 @@ VRT_r_resp_do_esi(VRT_CTX) /*--------------------------------------------------------------------*/ #define VRT_BODY_L(which) \ -void \ +VCL_VOID \ VRT_l_##which##_body(VRT_CTX, const char *str, ...) \ { \ va_list ap; \ @@ -847,7 +851,7 @@ VRT_r_bereq_hash(VRT_CTX) /*--------------------------------------------------------------------*/ #define HTTP_VAR(x) \ -struct http * \ +VCL_HTTP \ VRT_r_##x(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ From phk at FreeBSD.org Tue Mar 6 10:12:06 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 10:12:06 +0000 (UTC) Subject: [master] 73e85b5 Be agnostic about size of VCL_INT and VCL_BYTES Message-ID: <20180306101206.C1573A94D9@lists.varnish-cache.org> commit 73e85b57c0bbef23b30f2ed24e719a453ab95c2c Author: Poul-Henning Kamp Date: Tue Mar 6 10:09:51 2018 +0000 Be agnostic about size of VCL_INT and VCL_BYTES diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 053db5c..164c121 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -462,11 +462,11 @@ VRT_IP_string(VRT_CTX, VCL_IP ip) } VCL_STRING v_matchproto_() -VRT_INT_string(VRT_CTX, long num) +VRT_INT_string(VRT_CTX, VCL_INT num) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - return (WS_Printf(ctx->ws, "%ld", num)); + return (WS_Printf(ctx->ws, "%jd", (intmax_t)num)); } VCL_STRING v_matchproto_() @@ -499,7 +499,7 @@ VRT_BACKEND_string(VCL_BACKEND d) } VCL_STRING v_matchproto_() -VRT_BOOL_string(unsigned val) +VRT_BOOL_string(VCL_BOOL val) { return (val ? "true" : "false"); diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 6e5a198..a53d8c4 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -581,8 +581,9 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) } assert(b->len >= 0); if (off + n > b->len) { - VERR(ctx, "size %lld from offset %lld requires more bytes than " - "blob length %d in blob.sub()", n, off, b->len); + VERR(ctx, "size %jd from offset %jd requires more bytes than " + "blob length %d in blob.sub()", + (intmax_t)n, (intmax_t)off, b->len); return NULL; } @@ -595,7 +596,7 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) return NULL; } if ((sub->priv = WS_Alloc(ctx->ws, n)) == NULL) { - VERRNOMEM(ctx, "Allocating %lld bytes in blob.sub()", n); + VERRNOMEM(ctx, "Allocating %jd bytes in blob.sub()", (intmax_t)n); WS_Reset(ctx->ws, snap); return NULL; } diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index adec623..ccd64c2 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -278,9 +278,9 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas) for (i = 0; i < shardd->n_backend; i++) for (j = 0; j < replicas; j++) SHDBG(SHDBG_CIRCLE, shardd, - "hashcircle[%5ld] = " + "hashcircle[%5jd] = " "{point = %8x, host = %2u}\n", - i * replicas + j, + (intmax_t)(i * replicas + j), shardd->hashcircle[i * replicas + j].point, shardd->hashcircle[i * replicas + j].host); } diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 0e90a40..1ecd9f3 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -506,8 +506,9 @@ shard_param_args(VRT_CTX, } if (key_int < 0 || key_int > UINT32_MAX) { VRT_fail(ctx, "%s %s: " - "invalid key argument %ld with by=%s", - who, p->vcl_name, key_int, by_s); + "invalid key argument %jd with by=%s", + who, p->vcl_name, + (intmax_t)key_int, by_s); return (NULL); } assert(key_int >= 0); @@ -561,8 +562,8 @@ shard_param_args(VRT_CTX, if (args & arg_alt) { if (alt < 0) { VRT_fail(ctx, "%s %s: " - "invalid alt argument %ld", - who, p->vcl_name, alt); + "invalid alt argument %jd", + who, p->vcl_name, (intmax_t)alt); return (NULL); } p->alt = alt; From phk at FreeBSD.org Tue Mar 6 10:12:06 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 10:12:06 +0000 (UTC) Subject: [master] 1d68de5 Be agnostic about size of VCL_INT and VCL_BYTES Message-ID: <20180306101206.DD1C8A94DD@lists.varnish-cache.org> commit 1d68de5737bae85e0f2c71ef949653752fb13bc5 Author: Poul-Henning Kamp Date: Tue Mar 6 10:10:22 2018 +0000 Be agnostic about size of VCL_INT and VCL_BYTES diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index d833fb2..0534611 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -156,8 +156,8 @@ xyzzy_argtest(VRT_CTX, struct xyzzy_argtest_arg *arg) char buf[100]; AN(arg); - bprintf(buf, "%s %g %s %s %ld %d %s", - arg->one, arg->two, arg->three, arg->comma, arg->four, + bprintf(buf, "%s %g %s %s %jd %d %s", + arg->one, arg->two, arg->three, arg->comma, (intmax_t)arg->four, arg->valid_opt, arg->valid_opt ? arg->opt : ""); return (WS_Copy(ctx->ws, buf, -1)); } From phk at FreeBSD.org Tue Mar 6 10:12:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 10:12:07 +0000 (UTC) Subject: [master] da72728 Nail size of VCL_INT and VCL_BYTES at 64 bits. Message-ID: <20180306101207.05F66A94E1@lists.varnish-cache.org> commit da727285da6d62e70dd2038dcf6c79d91bba7564 Author: Poul-Henning Kamp Date: Tue Mar 6 10:10:39 2018 +0000 Nail size of VCL_INT and VCL_BYTES at 64 bits. diff --git a/include/vrt.h b/include/vrt.h index 603d24f..7b538cb 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -136,13 +136,13 @@ typedef const struct director * VCL_BACKEND; typedef const struct vmod_priv * VCL_BLOB; typedef const char * VCL_BODY; typedef unsigned VCL_BOOL; -typedef long long VCL_BYTES; +typedef int64_t VCL_BYTES; typedef double VCL_DURATION; typedef const char * VCL_ENUM; typedef const struct gethdr_s * VCL_HEADER; typedef struct http * VCL_HTTP; typedef void VCL_INSTANCE; -typedef long VCL_INT; +typedef int64_t VCL_INT; typedef const struct suckaddr * VCL_IP; typedef const struct vrt_backend_probe * VCL_PROBE; typedef double VCL_REAL; From phk at FreeBSD.org Tue Mar 6 10:25:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 10:25:09 +0000 (UTC) Subject: [master] 5d1dd90 More fixes related to VCL_INT+VCL_BYTES being 64bit Message-ID: <20180306102509.2B83FA9924@lists.varnish-cache.org> commit 5d1dd90aa55918d0d37fa87fd7a0374fa5aabe19 Author: Poul-Henning Kamp Date: Tue Mar 6 10:23:50 2018 +0000 More fixes related to VCL_INT+VCL_BYTES being 64bit diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index ddb1376..90cc00c 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -766,8 +766,8 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) } else if (i || fmt == REAL) e1 = vcc_mk_expr(REAL, "%s%f", sign, d); else - e1 = vcc_mk_expr(INT, "%s%ld", - sign, (unsigned long)d); + e1 = vcc_mk_expr(INT, "%s%jd", + sign, (intmax_t)d); } e1->constant = EXPR_CONST; *e = e1; diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 15cd9d1..bb2be44 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -73,7 +73,7 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) if (r > LONG_MAX || r < LONG_MIN) return (i); - return ((long)r); + return ((VCL_INT)r); } VCL_IP @@ -149,7 +149,7 @@ vmod_real2integer(VRT_CTX, VCL_REAL r, VCL_INT i) r = round(r); if (r > LONG_MAX || r < LONG_MIN) return(i); - return ((long)r); + return ((VCL_INT)r); } VCL_TIME v_matchproto_(td_std_real2time) @@ -173,7 +173,7 @@ vmod_time2integer(VRT_CTX, VCL_TIME t, VCL_INT i) t = round(t); if (t > LONG_MAX || t < LONG_MIN) return(i); - return ((long)t); + return ((VCL_INT)t); } VCL_REAL v_matchproto_(td_std_time2real) From nils.goroll at uplex.de Tue Mar 6 10:53:06 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 11:53:06 +0100 Subject: [master] 9305a2d pull out vcs_version.h / vmod_abi.h generation to own file again In-Reply-To: References: <20180305173809.13D0792299@lists.varnish-cache.org> Message-ID: <03f5763d-7272-9018-997f-0c10fb8d5a03@uplex.de> On 05/03/18 19:32, Dridi Boukelmoune wrote: > > https://travis-ci.org/Dridi/libvmod-querystring/jobs/349421603 > > This is a cron job building vmod-querystring against current master > once a week, and today it fails with: > > version.c:37:10: fatal error: 'vcs_version.h' file not found > #include "vcs_version.h" > ^~~~~~~~~~~~~~~ > 1 error generated. > > FYI, this is a source tree downloaded from github (same as > git-archive) and should result in NOGIT since this is not a dist > archive. d-oh, I was not aware of that use case - sorry. You mentioned you wanted to look after this - please keep in mind that if you _do_ have a dist, vcs_version should not be touched, iow, you should not get NOGIT when building dist. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 473 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Tue Mar 6 11:01:39 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 6 Mar 2018 12:01:39 +0100 Subject: [master] 9305a2d pull out vcs_version.h / vmod_abi.h generation to own file again In-Reply-To: <03f5763d-7272-9018-997f-0c10fb8d5a03@uplex.de> References: <20180305173809.13D0792299@lists.varnish-cache.org> <03f5763d-7272-9018-997f-0c10fb8d5a03@uplex.de> Message-ID: > d-oh, I was not aware of that use case - sorry. > > You mentioned you wanted to look after this - please keep in mind that if you > _do_ have a dist, vcs_version should not be touched, iow, you should not get > NOGIT when building dist. Yes, I have a fix covering all 3 cases and will submit it for review. Cheers From phk at FreeBSD.org Tue Mar 6 12:46:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 12:46:07 +0000 (UTC) Subject: [master] 5345cb0 More VCL- vs. C-type cleanup Message-ID: <20180306124607.D69ABAE1C3@lists.varnish-cache.org> commit 5345cb0316c64634700be1ec06a8629372543a8f Author: Poul-Henning Kamp Date: Tue Mar 6 11:36:11 2018 +0000 More VCL- vs. C-type cleanup diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 164c121..11c8e5a 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -50,13 +50,13 @@ const void * const vrt_magic_string_unset = &vrt_magic_string_unset; /*--------------------------------------------------------------------*/ -void -VRT_synth(VRT_CTX, unsigned code, const char *reason) +VCL_VOID +VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - if (code < 100) + if (code < 100 || code > 65535) code = 503; ctx->req->err_code = (uint16_t)code; ctx->req->err_reason = reason ? reason @@ -87,7 +87,9 @@ VRT_acl_match(VRT_CTX, VCL_ACL acl, VCL_IP ip) return (acl->match(ctx, ip)); } -void +/*--------------------------------------------------------------------*/ + +VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION d) { struct objcore *oc; @@ -109,7 +111,7 @@ VRT_hit_for_pass(VRT_CTX, VCL_DURATION d) /*--------------------------------------------------------------------*/ -struct http * +VCL_HTTP VRT_selecthttp(VRT_CTX, enum gethdr_e where) { struct http *hp; @@ -139,7 +141,7 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where) /*--------------------------------------------------------------------*/ -const char * +VCL_STRING VRT_GetHdr(VRT_CTX, const struct gethdr_s *hs) { const char *p; @@ -322,7 +324,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap) * Copy and merge a STRING_LIST on the current workspace */ -const char * +VCL_STRING VRT_CollectString(VRT_CTX, const char *p, ...) { va_list ap; @@ -340,7 +342,7 @@ VRT_CollectString(VRT_CTX, const char *p, ...) /*--------------------------------------------------------------------*/ -void +VCL_VOID VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs, const char *p, ...) { @@ -370,7 +372,7 @@ VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs, /*--------------------------------------------------------------------*/ -void +VCL_VOID VRT_handling(VRT_CTX, unsigned hand) { @@ -383,7 +385,7 @@ VRT_handling(VRT_CTX, unsigned hand) /*--------------------------------------------------------------------*/ -void +VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) { va_list ap; @@ -405,7 +407,7 @@ VRT_fail(VRT_CTX, const char *fmt, ...) * Feed data into the hash calculation */ -void +VCL_VOID VRT_hashdata(VRT_CTX, const char *str, ...) { va_list ap; @@ -432,7 +434,7 @@ VRT_hashdata(VRT_CTX, const char *str, ...) /*--------------------------------------------------------------------*/ -double +VCL_TIME VRT_r_now(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -470,7 +472,7 @@ VRT_INT_string(VRT_CTX, VCL_INT num) } VCL_STRING v_matchproto_() -VRT_REAL_string(VRT_CTX, double num) +VRT_REAL_string(VRT_CTX, VCL_REAL num) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -478,7 +480,7 @@ VRT_REAL_string(VRT_CTX, double num) } VCL_STRING v_matchproto_() -VRT_TIME_string(VRT_CTX, double t) +VRT_TIME_string(VRT_CTX, VCL_TIME t) { char *p; @@ -507,8 +509,8 @@ VRT_BOOL_string(VCL_BOOL val) /*--------------------------------------------------------------------*/ -void -VRT_Rollback(VRT_CTX, const struct http *hp) +VCL_VOID +VRT_Rollback(VRT_CTX, VCL_HTTP hp) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -528,7 +530,7 @@ VRT_Rollback(VRT_CTX, const struct http *hp) /*--------------------------------------------------------------------*/ -void +VCL_VOID VRT_synth_page(VRT_CTX, const char *str, ...) { va_list ap; @@ -553,8 +555,8 @@ VRT_synth_page(VRT_CTX, const char *str, ...) /*--------------------------------------------------------------------*/ -void -VRT_ban_string(VRT_CTX, const char *str) +VCL_VOID +VRT_ban_string(VRT_CTX, VCL_STRING str) { char *a1, *a2, *a3; char **av; @@ -645,8 +647,8 @@ VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize) * purges */ -unsigned -VRT_purge(VRT_CTX, double ttl, double grace, double keep) +VCL_INT +VRT_purge(VRT_CTX, VCL_DURATION ttl, VCL_DURATION grace, VCL_DURATION keep) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -711,16 +713,16 @@ VRT_memmove(void *dst, const void *src, unsigned len) (void)memmove(dst, src, len); } -int -VRT_ipcmp(const struct suckaddr *sua1, const struct suckaddr *sua2) +VCL_BOOL +VRT_ipcmp(VCL_IP sua1, VCL_IP sua2) { if (sua1 == NULL || sua2 == NULL) return(1); return (VSA_Compare_IP(sua1, sua2)); } -struct vmod_priv * -VRT_blob(VRT_CTX, const char *err, const uint8_t *src, size_t len) +VCL_BLOB +VRT_blob(VRT_CTX, const char *err, const void *src, size_t len) { struct vmod_priv *p; void *d; diff --git a/include/vrt.h b/include/vrt.h index 7b538cb..c29f8d4 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -364,8 +364,8 @@ struct gethdr_s { const char *what; }; -struct http *VRT_selecthttp(VRT_CTX, enum gethdr_e); -const char *VRT_GetHdr(VRT_CTX, const struct gethdr_s *); +VCL_HTTP VRT_selecthttp(VRT_CTX, enum gethdr_e); +VCL_STRING VRT_GetHdr(VRT_CTX, const struct gethdr_s *); /*********************************************************************** * req related @@ -376,28 +376,26 @@ VCL_BYTES VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize); /* Regexp related */ const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *); -void VRT_ban_string(VRT_CTX, const char *); -unsigned VRT_purge(VRT_CTX, double ttl, double grace, double keep); +VCL_VOID VRT_ban_string(VRT_CTX, VCL_STRING); +VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION); +VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING); +VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION); -void VRT_synth(VRT_CTX, unsigned, const char *); -void VRT_hit_for_pass(VRT_CTX, VCL_DURATION); - -void VRT_SetHdr(VRT_CTX, const struct gethdr_s *, const char *, ...); -void VRT_handling(VRT_CTX, unsigned hand); -void VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3); - -void VRT_hashdata(VRT_CTX, const char *str, ...); +VCL_VOID VRT_SetHdr(VRT_CTX, const struct gethdr_s *, const char *, ...); +VCL_VOID VRT_handling(VRT_CTX, unsigned hand); +VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3); +VCL_VOID VRT_hashdata(VRT_CTX, const char *str, ...); /* Simple stuff */ int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); -int VRT_ipcmp(const struct suckaddr *sua1, const struct suckaddr *sua2); -struct vmod_priv *VRT_blob(VRT_CTX, const char *, const uint8_t *, size_t); +VCL_BOOL VRT_ipcmp(VCL_IP, VCL_IP); +VCL_BLOB VRT_blob(VRT_CTX, const char *, const void *, size_t); -void VRT_Rollback(VRT_CTX, const struct http *); +VCL_VOID VRT_Rollback(VRT_CTX, VCL_HTTP); /* Synthetic pages */ -void VRT_synth_page(VRT_CTX, const char *, ...); +VCL_VOID VRT_synth_page(VRT_CTX, const char *, ...); /* Backend related */ struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *); From nils.goroll at uplex.de Tue Mar 6 13:40:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 13:40:10 +0000 (UTC) Subject: [master] 9b7aa04 Also run generate.py if vmod_abi.h or vcs_version.h are missing Message-ID: <20180306134010.48E6FAF131@lists.varnish-cache.org> commit 9b7aa043bca96b30be2ba053e99d7c234ace5916 Author: Nils Goroll Date: Tue Mar 6 13:12:23 2018 +0100 Also run generate.py if vmod_abi.h or vcs_version.h are missing Fixes #2597 Merges #2600 slightly modified diff --git a/include/Makefile.am b/include/Makefile.am index 80f6569..0831e2e 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -124,7 +124,9 @@ GENERATED_H = vcl.h $(GEN_H) ## except when building from a distribution vcs_version.h: - $(AM_V_GEN) if test -d $(top_srcdir)/.git ; then \ + @if test -e $(top_srcdir)/.git || \ + ! test -f vmod_abi.h || \ + ! test -f vcs_version.h ; then \ @PYTHON@ $(srcdir)/generate.py \ $(top_srcdir) $(top_builddir) ; \ fi From dridi.boukelmoune at gmail.com Tue Mar 6 13:54:08 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Mar 2018 13:54:08 +0000 (UTC) Subject: [master] 126841f Simplify include/generate.py Message-ID: <20180306135408.2E73BAF56D@lists.varnish-cache.org> commit 126841f6a7ecb8c76b45e9795edb5b631fd8f537 Author: Dridi Boukelmoune Date: Tue Mar 6 14:50:58 2018 +0100 Simplify include/generate.py Closes #2598 diff --git a/include/generate.py b/include/generate.py index f6cd339..0d9d9b3 100755 --- a/include/generate.py +++ b/include/generate.py @@ -50,21 +50,18 @@ def file_header(fo): fo.write("""/* * NB: This file is machine generated, DO NOT EDIT! * - * Edit and run lib/libvcc/generate.py instead. + * Edit and run include/generate.py instead. */ """) ####################################################################### -if os.path.isdir(os.path.join(srcroot, ".git")): - v = subprocess.check_output([ - "git --git-dir=" + os.path.join(srcroot, ".git") + - " show -s --pretty=format:%H" - ], shell=True, universal_newlines=True) - v = v.strip() -else: - v = "NOGIT" + +v = subprocess.check_output([ + "git --git-dir=%s rev-parse HEAD 2>/dev/null || echo NOGIT" % + (os.path.join(srcroot, ".git")) + ], shell=True, universal_newlines=True).strip() vcsfn = os.path.join(srcroot, "include", "vcs_version.h") From nils.goroll at uplex.de Tue Mar 6 15:35:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 15:35:10 +0000 (UTC) Subject: [master] 05f8503 fix out-of-tree builds / distcheck Message-ID: <20180306153510.9D5FCB127F@lists.varnish-cache.org> commit 05f850387eefde7d2dc8f917e8c366c16d76df38 Author: Nils Goroll Date: Tue Mar 6 16:34:06 2018 +0100 fix out-of-tree builds / distcheck diff --git a/include/Makefile.am b/include/Makefile.am index 0831e2e..d0d88c9 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -125,8 +125,8 @@ GENERATED_H = vcl.h $(GEN_H) vcs_version.h: @if test -e $(top_srcdir)/.git || \ - ! test -f vmod_abi.h || \ - ! test -f vcs_version.h ; then \ + ! test -f $(srcdir)/vmod_abi.h || \ + ! test -f $(srcdir)/vcs_version.h ; then \ @PYTHON@ $(srcdir)/generate.py \ $(top_srcdir) $(top_builddir) ; \ fi From nils.goroll at uplex.de Tue Mar 6 16:57:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 16:57:07 +0000 (UTC) Subject: [master] 5dd54f8 ban lurker edge case: make sure we stop at the ban list tail Message-ID: <20180306165707.18A1AB29E1@lists.varnish-cache.org> commit 5dd54f8390739c62c201e62c36eb515b1e03c2ee Author: Nils Goroll Date: Tue Mar 6 16:03:50 2018 +0100 ban lurker edge case: make sure we stop at the ban list tail in ban_lurker_test_ban() we remove completed bans from the obans list so the ban list tail might not be present in the obans list. To ensure that we stop at the tail, we now find our obans iteration termination ban walking the bans list backwards until we find an active ban, the head or the first obans element. In the latter two cases we are done. If there are any bans left to mark completed, the last active ban must be contained in the obans list so we should terminate correctly. Fixes #2556 diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 974cf9c..52a3c79 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -349,12 +349,30 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) if (ban_cleantail(VTAILQ_FIRST(&obans))) return (dt); + if (VTAILQ_FIRST(&obans) == NULL) + return (dt); + /* * Mark remaining bans completed: the tail of the obans list is now - * removed, but iterating over it is safe until we hit the new tail ban + * removed, but iterating over it is safe until we hit the new ban list + * tail + * + * bans at the tail of the list may have been completed by other means + * and, consequently, may have been removed from obans, so we skip all + * already completed bans at the tail. + * + * While traversing the ban list backwards, we check if we pass by the + * first oban, in which case we're done. */ - Lck_Lock(&ban_mtx); bd = VTAILQ_LAST(&ban_head, banhead_s); + while (bd->flags & BANS_FLAG_COMPLETED) { + if (bd == VTAILQ_FIRST(&ban_head) || + bd == VTAILQ_FIRST(&obans)) + return (dt); + bd = VTAILQ_PREV(bd, banhead_s, list); + } + + Lck_Lock(&ban_mtx); VTAILQ_FOREACH(b, &obans, l_list) { ban_mark_completed(b); if (b == bd) From nils.goroll at uplex.de Tue Mar 6 16:57:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 16:57:07 +0000 (UTC) Subject: [master] b130914 RFC2616_Ttl() in english Message-ID: <20180306165707.2E37DB29E4@lists.varnish-cache.org> commit b130914d5607cdd118eb298d98e263bacb403d4e Author: Nils Goroll Date: Tue Mar 6 17:42:26 2018 +0100 RFC2616_Ttl() in english Feel free to improve further! diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index 47b183f..b9951c8 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -403,6 +403,52 @@ Note: Backend conditional requests are independent of client conditional requests, so clients may receive 304 responses no matter if a backend request was conditional. +Before calling `vcl_backend_response`, core code sets ``beresp.ttl`` +based on the response status and the response headers ``Age``, +``Cache-Control`` or ``Expires`` and ``Date`` as follows: + +* If present and valid, the value of the ``Age`` header is effectively + deduced from all ttl calculations. + +* For status codes 200, 203, 204, 300, 401, 304, 404, 410 and 414: + + * If ``Cache-Control`` contains an ``s-maxage`` or ``max-age`` field + (in that order of preference), the ttl is set to the respective + non-negative value or 0 if negative. + + * Otherwise, if no ``Expires`` header exists, the default ttl is + used. + + * Otherwise, if ``Expires`` contains a time stamp before ``Date``, + the ttl is set to 0. + + * Otherwise, if no ``Date`` header is present or the ``Date`` header + timestamp differs from the local clock by no more than the + `clock_skew` parameter, the ttl is set to + + * 0 if ``Expires`` denotes a past timestamp or + + * the difference between the local clock and the ``Expires`` + header otherwise. + + * Otherwise, the ttl is set to the difference between ``Expires`` + and ``Date`` + +* For status codes 302 and 307, the calculation is identical except + that the default ttl is not used and -1 is returned if neither + ``Cache-Control`` nor ``Expires`` exists. + +* For all other status codes, ttl -1 is returned. + +``beresp.grace`` defaults to the `default_grace` parameter. + +For a non-negative ttl, if ``Cache-Control`` contains a +``stale-while-revalidate`` field value, ``beresp.grace`` is +set to that value if non-negative or 0 otherwise. + +``beresp.keep`` defaults to the `default_keep` parameter. + + The `vcl_backend_response` subroutine may terminate with calling ``return()`` with one of the following keywords: From nils.goroll at uplex.de Tue Mar 6 16:57:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Mar 2018 16:57:07 +0000 (UTC) Subject: [master] 59aa485 fix and restructure Built in subroutines documentation Message-ID: <20180306165707.4B622B29E8@lists.varnish-cache.org> commit 59aa48589808ae35da20a0c9ca71982343733deb Author: Nils Goroll Date: Tue Mar 6 17:54:12 2018 +0100 fix and restructure Built in subroutines documentation the section/subsection/subsubsection structure was broken. Regarding the 304 and beresp.ttl / beresp.grace / beresp.keep I am missing a fourth level, input welcome. diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index b9951c8..d8c9861 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -1,6 +1,5 @@ .. _vcl-built-in-subs: -==================== Built in subroutines ==================== @@ -19,7 +18,7 @@ similar across subroutines, so differences are only documented where relevant. common return keywords -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- .. _fail: @@ -63,7 +62,6 @@ common return keywords preserved except for ``req.restarts`` and ``req.xid``, which need to change by design. ------------ client side ----------- @@ -322,7 +320,6 @@ following keywords: Directly deliver the object defined by `vcl_synth` to the client without calling `vcl_deliver`. ------------- Backend Side ------------ @@ -384,6 +381,36 @@ vcl_backend_response Called after the response headers have been successfully retrieved from the backend. +The `vcl_backend_response` subroutine may terminate with calling +``return()`` with one of the following keywords: + + ``fail`` + see `fail`_ + + ``deliver`` + For a 304 response, create an updated cache object. + Otherwise, fetch the object body from the backend and initiate + delivery to any waiting client requests, possibly in parallel + (streaming). + + ``retry`` + Retry the backend transaction. Increases the `retries` counter. + If the number of retries is higher than *max_retries*, + control will be passed to :ref:`vcl_backend_error`. + + ``abandon`` + Abandon the backend request. Unless the backend request was a + background fetch, control is passed to :ref:`vcl_synth` on the + client side with ``resp.status`` preset to 503. + + ``pass(duration)`` + Mark the object as a hit-for-pass for the given duration. Subsequent + lookups hitting this object will be turned into passed transactions, + as if ``vcl_recv`` had returned ``pass``. + +304 handling +~~~~~~~~~~~~ + For a 304 response, varnish core code amends ``beresp`` before calling `vcl_backend_response`: @@ -403,6 +430,9 @@ Note: Backend conditional requests are independent of client conditional requests, so clients may receive 304 responses no matter if a backend request was conditional. +beresp.ttl / beresp.grace / beresp.keep +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Before calling `vcl_backend_response`, core code sets ``beresp.ttl`` based on the response status and the response headers ``Age``, ``Cache-Control`` or ``Expires`` and ``Date`` as follows: @@ -448,34 +478,6 @@ set to that value if non-negative or 0 otherwise. ``beresp.keep`` defaults to the `default_keep` parameter. - -The `vcl_backend_response` subroutine may terminate with calling -``return()`` with one of the following keywords: - - ``fail`` - see `fail`_ - - ``deliver`` - For a 304 response, create an updated cache object. - Otherwise, fetch the object body from the backend and initiate - delivery to any waiting client requests, possibly in parallel - (streaming). - - ``retry`` - Retry the backend transaction. Increases the `retries` counter. - If the number of retries is higher than *max_retries*, - control will be passed to :ref:`vcl_backend_error`. - - ``abandon`` - Abandon the backend request. Unless the backend request was a - background fetch, control is passed to :ref:`vcl_synth` on the - client side with ``resp.status`` preset to 503. - - ``pass(duration)`` - Mark the object as a hit-for-pass for the given duration. Subsequent - lookups hitting this object will be turned into passed transactions, - as if ``vcl_recv`` had returned ``pass``. - .. _vcl_backend_error: vcl_backend_error @@ -504,7 +506,6 @@ with one of the following keywords: :ref:`vcl_synth` on the client side is called with ``resp.status`` preset to 503. ----------------------- vcl.load / vcl.discard ---------------------- From phk at FreeBSD.org Tue Mar 6 17:49:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Mar 2018 17:49:07 +0000 (UTC) Subject: [master] 168fb22 Neuter a bit more of libvgz Message-ID: <20180306174907.A8541B39C7@lists.varnish-cache.org> commit 168fb22d4b90eb1d82b10b44bb407b0009049930 Author: Poul-Henning Kamp Date: Tue Mar 6 17:47:38 2018 +0000 Neuter a bit more of libvgz diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index b40a64e..2a25447 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -833,6 +833,8 @@ int ZEXPORT deflate (strm, flush) /* Write the header */ if (s->status == INIT_STATE) { +#ifdef NOVGZ + abort(); /* zlib header */ uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; uInt level_flags; @@ -865,6 +867,7 @@ int ZEXPORT deflate (strm, flush) s->last_flush = -1; return Z_OK; } +#endif } #ifdef GZIP if (s->status == GZIP_STATE) { From daghf at varnish-software.com Wed Mar 7 10:30:13 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 7 Mar 2018 10:30:13 +0000 (UTC) Subject: [master] e6e1b93 Exercise the H2_Send window handling Message-ID: <20180307103013.201BA9A3F8@lists.varnish-cache.org> commit e6e1b93a0dd9296918a695455d5e93c3045138ee Author: Dag Haavi Finstad Date: Wed Mar 7 11:27:49 2018 +0100 Exercise the H2_Send window handling diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc new file mode 100644 index 0000000..f9bf333 --- /dev/null +++ b/bin/varnishtest/tests/t02014.vtc @@ -0,0 +1,49 @@ +varnishtest "Exercise h/2 sender flow control code" + +barrier b1 sock 3 + +server s1 { + rxreq + txresp -bodylen 1000 +} -start + +varnish v1 -vcl+backend { + import vtc; + + sub vcl_deliver { + vtc.barrier_sync("${b1_sock}"); + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "param.set feature +http2" + +client c1 { + stream 0 { + txsettings -winsize 256 + rxsettings + barrier b1 sync + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + } -start + + stream 1 { + txreq + barrier b1 sync + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + rxresp + expect resp.status == 200 + expect resp.bodylen == 1000 + } -run + + stream 0 -wait +} -run From nils.goroll at uplex.de Wed Mar 7 11:58:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 7 Mar 2018 11:58:07 +0000 (UTC) Subject: [master] 2da6c5e Document desupported VSL tags in varnishncsa Message-ID: <20180307115807.356A99BFE7@lists.varnish-cache.org> commit 2da6c5e539f8f858c446812d30f00de600ecc3e4 Author: Nils Goroll Date: Wed Mar 7 12:55:48 2018 +0100 Document desupported VSL tags in varnishncsa Ref: 63807dd22725653e23c20071a9eb958642413f36 Found while working on https://code.uplex.de/uplex-varnish/varnishevent diff --git a/doc/changes.rst b/doc/changes.rst index 4d8fb92..e7cab57 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -144,6 +144,15 @@ Logging / statistics bytes was attributed to the body, including any protocol overhead (ie chunked encoding). +bundled tools +------------- + +* ``varnishncsa`` refuses output formats (as defined with the ``-F`` + command line argument) for tags which could contain control or + binary characters. At the time of writing, these are: + ``%{H2RxHdr}x``, ``%{H2RxBody}x``, ``%{H2TxHdr}x``, ``%{H2TxBody}x``, + ``%{Debug}x``, ``%{HttpGarbage}x`` and ``%{Hash}x`` + C APIs (for vmod and utility authors) ------------------------------------- From daghf at varnish-software.com Wed Mar 7 13:45:09 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 7 Mar 2018 13:45:09 +0000 (UTC) Subject: [master] ea65120 Adjust flow-control windows for active streams Message-ID: <20180307134509.D326FA328B@lists.varnish-cache.org> commit ea6512044a71d78ce60b82a66c9384e6efd717ec Author: Dag Haavi Finstad Date: Wed Mar 7 14:34:16 2018 +0100 Adjust flow-control windows for active streams On reception of a SETTINGS frame with a new value for INITIAL_WINDOW_SIZE, we also need to adjust the flow-control window for any active streams. rfc7540, section 6.9.2 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index efe91f2..7b56b44 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -404,6 +404,32 @@ static const struct h2_setting_s * const h2_setting_tbl[] = { #define H2_SETTING_TBL_LEN (sizeof(h2_setting_tbl)/sizeof(h2_setting_tbl[0])) +static void +h2_win_adjust(struct h2_sess *h2, uint32_t oldval, uint32_t newval) +{ + struct h2_req *r2; + + Lck_AssertHeld(&h2->sess->mtx); + // rfc7540,l,2668,2674 + VTAILQ_FOREACH(r2, &h2->streams, list) { + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + if (r2 == h2->req0) + continue; // rfc7540,l,2699,2699 + switch (r2->state) { + case H2_S_IDLE: + case H2_S_OPEN: + case H2_S_CLOS_REM: + /* + * We allow a window to go negative, as per + * rfc7540,l,2676,2680 + */ + r2->t_window += (int64_t)newval - oldval; + default: + break; + } + } +} + h2_error h2_set_setting(struct h2_sess *h2, const uint8_t *d) { @@ -433,6 +459,8 @@ h2_set_setting(struct h2_sess *h2, const uint8_t *d) return (s->range_error); } Lck_Lock(&h2->sess->mtx); + if (s == H2_SET_INITIAL_WINDOW_SIZE) + h2_win_adjust(h2, h2->remote_settings.initial_window_size, y); VSLb(h2->vsl, SLT_Debug, "H2SETTING %s=0x%08x", s->name, y); Lck_Unlock(&h2->sess->mtx); AN(s->setfunc); From daghf at varnish-software.com Wed Mar 7 14:14:08 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 7 Mar 2018 14:14:08 +0000 (UTC) Subject: [master] ee0e804 t02014: Also exercise connection flow-control bits Message-ID: <20180307141408.59ED2A3B3E@lists.varnish-cache.org> commit ee0e804923f11f786751c29cc6ce428c2e337563 Author: Dag Haavi Finstad Date: Wed Mar 7 15:12:43 2018 +0100 t02014: Also exercise connection flow-control bits diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc index f9bf333..bc49f4e 100644 --- a/bin/varnishtest/tests/t02014.vtc +++ b/bin/varnishtest/tests/t02014.vtc @@ -4,7 +4,7 @@ barrier b1 sock 3 server s1 { rxreq - txresp -bodylen 1000 + txresp -bodylen 66300 } -start varnish v1 -vcl+backend { @@ -20,8 +20,6 @@ varnish v1 -cliok "param.set feature +http2" client c1 { stream 0 { - txsettings -winsize 256 - rxsettings barrier b1 sync delay .5 txwinup -size 256 @@ -42,7 +40,7 @@ client c1 { txwinup -size 256 rxresp expect resp.status == 200 - expect resp.bodylen == 1000 + expect resp.bodylen == 66300 } -run stream 0 -wait From geoff at uplex.de Wed Mar 7 14:24:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 7 Mar 2018 14:24:08 +0000 (UTC) Subject: [master] 9bc756d Add VCL var sess.xid (VCL >= 4.1). Message-ID: <20180307142408.E1EEDA3E70@lists.varnish-cache.org> commit 9bc756d0cc4981624ce8c1d933ef52c82d6e1731 Author: Geoff Simmons Date: Wed Mar 7 15:22:48 2018 +0100 Add VCL var sess.xid (VCL >= 4.1). diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index f5f305f..05f058a 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -648,7 +648,7 @@ VRT_DO_AGE_R(obj, ctx->req->objcore) VRT_DO_AGE_R(beresp, ctx->bo->fetch_objcore) /*-------------------------------------------------------------------- - * [be]req.xid + * [[be]req|sess].xid */ VCL_STRING @@ -673,6 +673,20 @@ VRT_r_bereq_xid(VRT_CTX) VXID(ctx->bo->vsl->wid))); } +VCL_STRING +VRT_r_sess_xid(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (VALID_OBJ(ctx->req, REQ_MAGIC)) + return (WS_Printf(ctx->req->http->ws, "%u", + VXID(ctx->req->sp->vxid))); + + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + return (WS_Printf(ctx->bo->bereq->ws, "%u", + VXID(ctx->bo->sp->vxid))); +} + /*-------------------------------------------------------------------- * req fields */ diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index cb7dc75..b3cbc1a 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -7,6 +7,10 @@ server s1 { -hdr "H234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789I: foo" \ -hdr "Foo: H234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789I" \ -bodylen 10 + rxreq + txresp + rxreq + txresp } -start varnish v1 -vcl+backend { @@ -65,3 +69,34 @@ process p1 -stop # shell {cat ${tmpdir}/vlog} shell -match {1001 H2TxHdr c \[000006040000000000\]} \ {cat ${tmpdir}/vlog} + +# While we're here, test sess.xid over H2 as well + +varnish v1 -syntax 4.1 -vcl+backend { + sub vcl_backend_response { + set beresp.http.B-Sess-XID = sess.xid; + } + + sub vcl_deliver { + set resp.http.C-Sess-XID = sess.xid; + } +} + +client c1 { + stream 7 { + txreq -url "/uncached" + rxresp + expect resp.status == 200 + expect resp.http.C-Sess-XID ~ "^[0-9]+$" + expect resp.http.B-Sess-XID ~ "^[0-9]+$" + expect resp.http.C-Sess-XID == resp.http.B-Sess-XID + } -run + stream 9 { + txreq -url "/still_not_cached" + rxresp + expect resp.status == 200 + expect resp.http.C-Sess-XID ~ "^[0-9]+$" + expect resp.http.B-Sess-XID ~ "^[0-9]+$" + expect resp.http.C-Sess-XID == resp.http.B-Sess-XID + } -run +} -run diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 544240b..9a20422 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -3,6 +3,8 @@ varnishtest "More VCL coverage" server s1 { rxreq txresp + rxreq + txresp } -start varnish v1 -syntax 4.0 -arg "-i J.F.Nobody" -vcl+backend { @@ -91,3 +93,34 @@ client c1 { txreq rxresp } -run + +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'sess.xid' (Only available when 4.1 <= VCL syntax)} { + sub vcl_recv { + set req.http.Sess-XID = sess.xid; + } +} + +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'sess.xid' (Only available when 4.1 <= VCL syntax)} { + sub vcl_backend_fetch { + set bereq.http.Sess-XID = sess.xid; + } +} + +varnish v1 -syntax 4.1 -vcl+backend { + sub vcl_backend_response { + set beresp.http.B-Sess-XID = sess.xid; + } + + sub vcl_deliver { + set resp.http.C-Sess-XID = sess.xid; + } +} + +client c1 { + txreq -url "/uncached" + rxresp + expect resp.status == 200 + expect resp.http.C-Sess-XID ~ "^[0-9]+$" + expect resp.http.B-Sess-XID ~ "^[0-9]+$" + expect resp.http.C-Sess-XID == resp.http.B-Sess-XID +} -run diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 37e88c9..7c18b36 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1120,7 +1120,23 @@ now When converted to STRING in expressions it returns a formatted timestamp like `Tue, 20 Feb 2018 09:30:31 GMT` - + +sess +~~~~ + +A session corresponds to the "conversation" that Varnish has with a +single client connection, over which one or more request/response +transactions may take place. It may comprise the traffic over an +HTTP/1 keep-alive connection, or the multiplexed traffic over an +HTTP/2 connection. + +sess.xid ``VCL >= 4.1`` + + Type: STRING + + Readable from: client, backend + + Unique ID of this session. storage ~~~~~~~ From geoff at uplex.de Wed Mar 7 22:17:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 7 Mar 2018 22:17:08 +0000 (UTC) Subject: [master] 7877283 First batch of docs concerning UDS and VCL in "Upgrading to 6.0". Message-ID: <20180307221708.C8930AE2F1@lists.varnish-cache.org> commit 78772832944c550d65269f4ad8436902b850684e Author: Geoff Simmons Date: Wed Mar 7 23:15:07 2018 +0100 First batch of docs concerning UDS and VCL in "Upgrading to 6.0". This anticipates some features planned for 6.0 that are presently not yet implemented: - UDS requires VCL 4.1. - VCL variables local.socket and local.endpoint diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index d2af1ac..ff0c6ec 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -30,10 +30,158 @@ Changes to VCL XXX: ... intro paragraph -XXX VCL subhead 1 -~~~~~~~~~~~~~~~~~ - -XXX: ... etc. +Unix domain sockets and VCL +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We have made an effort to adapt the support of Unix domain sockets in +VCL so that you may not have to change anything in your VCL deployment +at all, other than changing the version to 4.1. Most importantly, this +affects the meaning of the ``*.ip`` variables and the use of ACLs; and +there are a number of other details you should consider. + +If you don't use UDSen, then nothing about VCL changes. UDS support +requires version 4.1, so if you are keeping your VCL level at 4.0 (and +hence are staying with IP addresses), then none of the following is of +concern. + +``client.ip``, ``server.ip``, ``local.ip`` and ``remote.ip`` +------------------------------------------------------------ + +When referring to a connection that was in fact addressed as a UDS, +the ``*.ip`` variables always have a value equivalent to the IPv4 +address ``0.0.0.0:0`` -- the "any IPv4" address with port 0. + +Remember that if you are using the PROXY protocol, then ``client.ip`` +and ``server.ip`` are set to the addresses sent in the PROXY header by +the forwarding component. So these two variables may have "real" IP +address values, even when the Varnish listener address is a UDS, if +PROXY was specified. ``local.ip`` and ``remote.ip``, however, are +always ``0.0.0.0`` for a UDS listener. + +If you have more than one UDS listener (more than one ``-a`` +command-line argument specifying a socket path), then you may not be +able to use the ``*.ip`` variables to tell them apart, especially +since ``local.ip`` will be ``0.0.0.0`` for all of them. If you need to +distinguish such addresses in VCL, you can use ``local.socket`` (which +is the name given for the ``-a`` argument; ``a0``, ``a1`` etc. by +default) or ``local.endpoint``, which in the case of UDS is the path +given in the ``-a`` argument. You can, for example, use string +operations such as regex matching on ``local.endpoint`` to determine +properties of the path address:: + + # admin requests allowed only on the listener whose path ends in + # "admin.sock" + if (req.url ~ "^/admin") { + if (local.endpoint !~ "admin.sock$") { + # wrong listener, respond with "403 Forbidden" + return( synth(403) ); + } + else { + # process the admin request ... + } + } + + # superadmin requests only allowed on the "superadmin.sock" listener + if (req.url ~ "^/superadmin") { + if (local.endpoint !~ "superadmin.sock$") { + return( synth(403) ); + } + else { + # superadmin request ... + } + } + +ACLs +---- + +As before, ACLs can only specify ranges of IP addresses, and matches +against ACLs can only be run against IP-valued elements. + +This means that if a ``*.ip`` variable whose value is ``0.0.0.0`` due +to the use of UDS is matched against an ACL, the match can only +succeed if the ACL includes ``0.0.0.0``. If you currently have a +security requirement that depends on an ACL matching a range of IP +addresses, then that will continue to work with a UDS listener (since +you almost certainly have not included ``0.0.0.0`` in that range). + +Recall again that ``client.ip`` and ``server.ip`` are set by the PROXY +protocol. So if you have a UDS listener configured to use PROXY and +are using an ACL to match against one of those two variables, the +matches will continue working against the "real" IPs sent via PROXY. + +You can of course define an ACL to match in the UDS case, by including +``0.0.0.0``:: + + # matches local.ip and remote.ip when the listener is UDS + acl uds { + "0.0.0.0"; + } + +But such an ACL cannot distinguish different UDS listeners, if you +have more than one. For that, you can achieve a similar effect by +inspecting ``local.socket`` and/or ``local.endpoint``, as discussed +above. + +``client.identity`` and the hash and shard directors +---------------------------------------------------- + +As before, ``client.identity`` defaults to ``client.ip``; that is, if +its value has not been explicitly set in VCL, then it returns the same +value as ``client.ip`` when it is read. + +A common use of ``client.identity`` is to configure the hash and shard +directors (see :ref:`vmod_directors(3)`). This is a way to achieve +"client-sticky" distribution of requests to backends -- requests from +the same clients are always sent to the same backends. + +Such a configuration will almost certainly not do what you want if: + +* The listener is set to a UDS address. +* PROXY is not used to set ``client.ip``. +* ``client.identity`` is not set to a distinct value before it is + used to configure the director. + +Since ``client.identity`` defaults to ``client.ip``, which is always +``0.0.0.0`` under these conditions, the result will be that the +director sends all requests to just one backend, and no requests to +any other backend. + +To avoid that result, change one of the conditions listed above -- use +PROXY to set distinct values for ``client.ip``, or set +``client.identity`` to distinct values before it is used. + +``server.ip`` and default hashing for the cache +----------------------------------------------- + +The default algorithm for computing a hash value for the cache (the +implementation of ``vcl_hash`` in ``builtin.vcl``) mixes ``req.url`` +and the Host header (``req.http.Host``) into the hash data. If there +is no Host header, then ``server.ip`` is used instead. Considering the +Host header or ``server.ip`` is a way of achieving a kind of "virtual +hosting" -- if your site receives requests with different Host headers +or at distinct server addresses, then requests for the same URL will +not hit the same cached response, if the requests are different in +those other respects. + +If you have UDS listeners and are not using PROXY to set distinct +values of ``server.ip``, then requests without a Host header will have +the same value of ``server.ip == 0.0.0.0`` mixed into the hash. In +that case, requests with the same URL will result in the same hash +value, and hit the same cached responses. + +That doesn't matter, of course, if you don't need the "virtual +hosting" effect -- you only have one listener, you never receive +different host headers, or you never receive the same URL for what +should lead to distinct responses. + +But if you need to avoid that result, then you can make one or more +of these changes: + +* Use the PROXY protocol to set distinct ``server.ip`` values. +* Write your own implementation of ``vcl_hash``, for example to + mix ``local.socket`` or ``local.endpoint`` into the hash. +* Set ``req.http.Host`` to a distinct value if it is absent before + ``vcl_hash`` is entered. VCL variables ~~~~~~~~~~~~~ From phk at FreeBSD.org Thu Mar 8 00:40:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 00:40:07 +0000 (UTC) Subject: [master] a125b10 Discontinue backend.ip variable in VCL4.1 Message-ID: <20180308004007.AB515B0A4B@lists.varnish-cache.org> commit a125b10d8e38a1dfc1c912b28189e687fd5d4c6a Author: Poul-Henning Kamp Date: Wed Mar 7 21:58:59 2018 +0000 Discontinue backend.ip variable in VCL4.1 diff --git a/bin/varnishtest/tests/r01398.vtc b/bin/varnishtest/tests/r01398.vtc index 3f929c3..5bedab4 100644 --- a/bin/varnishtest/tests/r01398.vtc +++ b/bin/varnishtest/tests/r01398.vtc @@ -1,6 +1,6 @@ varnishtest "ticket 1398" -varnish v1 -vcl { +varnish v1 -syntax 4.0 -vcl { backend foo { .host = "${bad_backend}"; } diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 7c18b36..8f20afc 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -849,7 +849,7 @@ beresp.backend.name Same as beresp.backend. -beresp.backend.ip +beresp.backend.ip ``VCL <= 4.0`` Type: IP From phk at FreeBSD.org Thu Mar 8 00:40:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 00:40:07 +0000 (UTC) Subject: [master] 688ccfc Rename struct vtp to pfd, it will become non-TCP specific. Message-ID: <20180308004007.C33F0B0A4E@lists.varnish-cache.org> commit 688ccfcebe8af7896dc6e462f69eb48bd453f820 Author: Poul-Henning Kamp Date: Wed Mar 7 23:06:11 2018 +0000 Rename struct vtp to pfd, it will become non-TCP specific. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 204c3c7..5d49763 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -73,11 +73,11 @@ static struct lock backends_mtx; * Get a connection to the backend */ -static struct vtp * +static struct pfd * vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, unsigned force_fresh) { - struct vtp *vtp; + struct pfd *pfd; double tmod; char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE]; char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE]; @@ -113,8 +113,8 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - vtp = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh); - if (vtp == NULL) { + pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh); + if (pfd == NULL) { VSLb(bo->vsl, SLT_FetchError, "backend %s: fail", bp->director->display_name); // XXX: Per backend stats ? @@ -123,8 +123,8 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, return (NULL); } - assert(vtp->fd >= 0); - AN(vtp->addr); + assert(pfd->fd >= 0); + AN(pfd->priv); Lck_Lock(&bp->mtx); bp->n_conn++; @@ -133,21 +133,21 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, Lck_Unlock(&bp->mtx); if (bp->proxy_header != 0) - VPX_Send_Proxy(vtp->fd, bp->proxy_header, bo->sp); + VPX_Send_Proxy(pfd->fd, bp->proxy_header, bo->sp); - VTCP_myname(vtp->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - VTCP_hisname(vtp->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); + VTCP_myname(pfd->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); + VTCP_hisname(pfd->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", - vtp->fd, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); + pfd->fd, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); INIT_OBJ(bo->htc, HTTP_CONN_MAGIC); - bo->htc->priv = vtp; - bo->htc->rfd = &vtp->fd; + bo->htc->priv = pfd; + bo->htc->rfd = &pfd->fd; FIND_TMO(first_byte_timeout, bo->htc->first_byte_timeout, bo, bp); FIND_TMO(between_bytes_timeout, bo->htc->between_bytes_timeout, bo, bp); - return (vtp); + return (pfd); } static unsigned v_matchproto_(vdi_healthy_f) @@ -167,7 +167,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, struct busyobj *bo) { struct backend *bp; - struct vtp *vtp; + struct pfd *pfd; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -175,24 +175,24 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(vtp, bo->htc->priv, VTP_MAGIC); + CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC); bo->htc->priv = NULL; - if (vtp->state != VTP_STATE_USED) + if (pfd->state != PFD_STATE_USED) assert(bo->htc->doclose == SC_TX_PIPE || bo->htc->doclose == SC_RX_TIMEOUT); if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { - VSLb(bo->vsl, SLT_BackendClose, "%d %s", vtp->fd, + VSLb(bo->vsl, SLT_BackendClose, "%d %s", pfd->fd, bp->director->display_name); - VTP_Close(&vtp); - AZ(vtp); + VTP_Close(&pfd); + AZ(pfd); Lck_Lock(&bp->mtx); } else { - assert (vtp->state == VTP_STATE_USED); - VSLb(bo->vsl, SLT_BackendReuse, "%d %s", vtp->fd, + assert (pfd->state == PFD_STATE_USED); + VSLb(bo->vsl, SLT_BackendReuse, "%d %s", pfd->fd, bp->director->display_name); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; - VTP_Recycle(wrk, &vtp); + VTP_Recycle(wrk, &pfd); } assert(bp->n_conn > 0); bp->n_conn--; @@ -210,7 +210,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, { int i, extrachance = 1; struct backend *bp; - struct vtp *vtp; + struct pfd *pfd; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -226,18 +226,18 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr); do { - vtp = vbe_dir_getfd(wrk, bp, bo, extrachance == 0); - if (vtp == NULL) + pfd = vbe_dir_getfd(wrk, bp, bo, extrachance == 0); + if (pfd == NULL) return (-1); AN(bo->htc); - if (vtp->state != VTP_STATE_STOLEN) + if (pfd->state != PFD_STATE_STOLEN) extrachance = 0; i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, &bo->acct.bereq_bodybytes, 0); - if (vtp->state != VTP_STATE_USED) { - if (VTP_Wait(wrk, vtp, VTIM_real() + + if (pfd->state != PFD_STATE_USED) { + if (VTP_Wait(wrk, pfd, VTIM_real() + bo->htc->first_byte_timeout) != 0) { bo->htc->doclose = SC_RX_TIMEOUT; VSLb(bo->vsl, SLT_FetchError, @@ -247,7 +247,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, } if (bo->htc->doclose == SC_NULL) { - assert(vtp->state == VTP_STATE_USED); + assert(pfd->state == PFD_STATE_USED); if (i == 0) i = V1F_FetchRespHdr(bo); if (i == 0) { @@ -278,15 +278,15 @@ static const struct suckaddr * v_matchproto_(vdi_getip_f) vbe_dir_getip(const struct director *d, struct worker *wrk, struct busyobj *bo) { - struct vtp *vtp; + struct pfd *pfd; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(vtp, bo->htc->priv, VTP_MAGIC); + CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC); - return (vtp->addr); + return (pfd->priv); } /*--------------------------------------------------------------------*/ @@ -298,7 +298,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) enum sess_close retval; struct backend *bp; struct v1p_acct v1a; - struct vtp *vtp; + struct pfd *pfd; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -313,16 +313,16 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) req->res_mode = RES_PIPE; - vtp = vbe_dir_getfd(req->wrk, bp, bo, 0); + pfd = vbe_dir_getfd(req->wrk, bp, bo, 0); - if (vtp == NULL) { + if (pfd == NULL) { retval = SC_TX_ERROR; } else { CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1); VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk)); if (i == 0) - V1P_Process(req, vtp->fd, &v1a); + V1P_Process(req, pfd->fd, &v1a); VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk)); bo->htc->doclose = SC_TX_PIPE; vbe_dir_finish(d, req->wrk, bo); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index a6c945d..a374f70 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -277,7 +277,7 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VTP_Open(vt->tcp_pool, t_end - t_now, &sa); + s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa); if (s < 0) { /* Got no connection: failed */ return; diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index feaeed1..1c0ca90 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -56,10 +56,10 @@ struct tcp_pool { int refcnt; struct lock mtx; - VTAILQ_HEAD(, vtp) connlist; + VTAILQ_HEAD(, pfd) connlist; int n_conn; - VTAILQ_HEAD(, vtp) killlist; + VTAILQ_HEAD(, pfd) killlist; int n_kill; int n_used; @@ -75,39 +75,39 @@ static VTAILQ_HEAD(, tcp_pool) tcp_pools = VTAILQ_HEAD_INITIALIZER(tcp_pools); static void v_matchproto_(waiter_handle_f) tcp_handle(struct waited *w, enum wait_event ev, double now) { - struct vtp *vtp; + struct pfd *pfd; struct tcp_pool *tp; - CAST_OBJ_NOTNULL(vtp, w->priv1, VTP_MAGIC); + CAST_OBJ_NOTNULL(pfd, w->priv1, PFD_MAGIC); (void)ev; (void)now; - CHECK_OBJ_NOTNULL(vtp->tcp_pool, TCP_POOL_MAGIC); - tp = vtp->tcp_pool; + CHECK_OBJ_NOTNULL(pfd->tcp_pool, TCP_POOL_MAGIC); + tp = pfd->tcp_pool; Lck_Lock(&tp->mtx); - switch (vtp->state) { - case VTP_STATE_STOLEN: - vtp->state = VTP_STATE_USED; - VTAILQ_REMOVE(&tp->connlist, vtp, list); - AN(vtp->cond); - AZ(pthread_cond_signal(vtp->cond)); + switch (pfd->state) { + case PFD_STATE_STOLEN: + pfd->state = PFD_STATE_USED; + VTAILQ_REMOVE(&tp->connlist, pfd, list); + AN(pfd->cond); + AZ(pthread_cond_signal(pfd->cond)); break; - case VTP_STATE_AVAIL: - VTCP_close(&vtp->fd); - VTAILQ_REMOVE(&tp->connlist, vtp, list); + case PFD_STATE_AVAIL: + VTCP_close(&pfd->fd); + VTAILQ_REMOVE(&tp->connlist, pfd, list); tp->n_conn--; - FREE_OBJ(vtp); + FREE_OBJ(pfd); break; - case VTP_STATE_CLEANUP: - VTCP_close(&vtp->fd); + case PFD_STATE_CLEANUP: + VTCP_close(&pfd->fd); tp->n_kill--; - VTAILQ_REMOVE(&tp->killlist, vtp, list); - memset(vtp, 0x11, sizeof *vtp); - free(vtp); + VTAILQ_REMOVE(&tp->killlist, pfd, list); + memset(pfd, 0x11, sizeof *pfd); + free(pfd); break; default: - WRONG("Wrong vtp state"); + WRONG("Wrong pfd state"); } Lck_Unlock(&tp->mtx); } @@ -194,7 +194,7 @@ void VTP_Rel(struct tcp_pool **tpp) { struct tcp_pool *tp; - struct vtp *vtp, *vtp2; + struct pfd *pfd, *pfd2; TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); @@ -211,13 +211,13 @@ VTP_Rel(struct tcp_pool **tpp) free(tp->ip4); free(tp->ip6); Lck_Lock(&tp->mtx); - VTAILQ_FOREACH_SAFE(vtp, &tp->connlist, list, vtp2) { - VTAILQ_REMOVE(&tp->connlist, vtp, list); + VTAILQ_FOREACH_SAFE(pfd, &tp->connlist, list, pfd2) { + VTAILQ_REMOVE(&tp->connlist, pfd, list); tp->n_conn--; - assert(vtp->state == VTP_STATE_AVAIL); - vtp->state = VTP_STATE_CLEANUP; - (void)shutdown(vtp->fd, SHUT_WR); - VTAILQ_INSERT_TAIL(&tp->killlist, vtp, list); + assert(pfd->state == PFD_STATE_AVAIL); + pfd->state = PFD_STATE_CLEANUP; + (void)shutdown(pfd->fd, SHUT_WR); + VTAILQ_INSERT_TAIL(&tp->killlist, pfd, list); tp->n_kill++; } while (tp->n_kill) { @@ -239,7 +239,7 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) +VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) { int s; int msec; @@ -248,17 +248,17 @@ VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) msec = (int)floor(tmo * 1000.0); if (cache_param->prefer_ipv6) { - *sa = tp->ip6; + *privp = tp->ip6; s = VTCP_connect(tp->ip6, msec); if (s >= 0) return (s); } - *sa = tp->ip4; + *privp = tp->ip4; s = VTCP_connect(tp->ip4, msec); if (s >= 0) return (s); if (!cache_param->prefer_ipv6) { - *sa = tp->ip6; + *privp = tp->ip6; s = VTCP_connect(tp->ip6, msec); } return (s); @@ -269,43 +269,43 @@ VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) */ void -VTP_Recycle(const struct worker *wrk, struct vtp **vtpp) +VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) { - struct vtp *vtp; + struct pfd *pfd; struct tcp_pool *tp; int i = 0; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - vtp = *vtpp; - *vtpp = NULL; - CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); - tp = vtp->tcp_pool; + pfd = *pfdp; + *pfdp = NULL; + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + tp = pfd->tcp_pool; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - assert(vtp->state == VTP_STATE_USED); - assert(vtp->fd > 0); + assert(pfd->state == PFD_STATE_USED); + assert(pfd->fd > 0); Lck_Lock(&tp->mtx); tp->n_used--; - vtp->waited->priv1 = vtp; - vtp->waited->fd = vtp->fd; - vtp->waited->idle = VTIM_real(); - vtp->state = VTP_STATE_AVAIL; - vtp->waited->func = tcp_handle; - vtp->waited->tmo = &cache_param->backend_idle_timeout; - if (Wait_Enter(wrk->pool->waiter, vtp->waited)) { - VTCP_close(&vtp->fd); - memset(vtp, 0x33, sizeof *vtp); - free(vtp); + pfd->waited->priv1 = pfd; + pfd->waited->fd = pfd->fd; + pfd->waited->idle = VTIM_real(); + pfd->state = PFD_STATE_AVAIL; + pfd->waited->func = tcp_handle; + pfd->waited->tmo = &cache_param->backend_idle_timeout; + if (Wait_Enter(wrk->pool->waiter, pfd->waited)) { + VTCP_close(&pfd->fd); + memset(pfd, 0x33, sizeof *pfd); + free(pfd); // XXX: stats - vtp = NULL; + pfd = NULL; } else { - VTAILQ_INSERT_HEAD(&tp->connlist, vtp, list); + VTAILQ_INSERT_HEAD(&tp->connlist, pfd, list); i++; } - if (vtp != NULL) + if (pfd != NULL) tp->n_conn++; Lck_Unlock(&tp->mtx); @@ -314,7 +314,7 @@ VTP_Recycle(const struct worker *wrk, struct vtp **vtpp) * In varnishtest we do not have the luxury of using * multiple backend connections, so whenever we end up * in the "pending" case, take a short nap to let the - * waiter catch up and put the vtp back into circulations. + * waiter catch up and put the pfd back into circulations. * * In particular ESI:include related tests suffer random * failures without this. @@ -332,33 +332,33 @@ VTP_Recycle(const struct worker *wrk, struct vtp **vtpp) */ void -VTP_Close(struct vtp **vtpp) +VTP_Close(struct pfd **pfdp) { - struct vtp *vtp; + struct pfd *pfd; struct tcp_pool *tp; - vtp = *vtpp; - *vtpp = NULL; - CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); - tp = vtp->tcp_pool; + pfd = *pfdp; + *pfdp = NULL; + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + tp = pfd->tcp_pool; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - assert(vtp->fd > 0); + assert(pfd->fd > 0); Lck_Lock(&tp->mtx); - assert(vtp->state == VTP_STATE_USED || vtp->state == VTP_STATE_STOLEN); + assert(pfd->state == PFD_STATE_USED || pfd->state == PFD_STATE_STOLEN); tp->n_used--; - if (vtp->state == VTP_STATE_STOLEN) { - (void)shutdown(vtp->fd, SHUT_RDWR); - VTAILQ_REMOVE(&tp->connlist, vtp, list); - vtp->state = VTP_STATE_CLEANUP; - VTAILQ_INSERT_HEAD(&tp->killlist, vtp, list); + if (pfd->state == PFD_STATE_STOLEN) { + (void)shutdown(pfd->fd, SHUT_RDWR); + VTAILQ_REMOVE(&tp->connlist, pfd, list); + pfd->state = PFD_STATE_CLEANUP; + VTAILQ_INSERT_HEAD(&tp->killlist, pfd, list); tp->n_kill++; } else { - assert(vtp->state == VTP_STATE_USED); - VTCP_close(&vtp->fd); - memset(vtp, 0x44, sizeof *vtp); - free(vtp); + assert(pfd->state == PFD_STATE_USED); + VTCP_close(&pfd->fd); + memset(pfd, 0x44, sizeof *pfd); + free(pfd); } Lck_Unlock(&tp->mtx); } @@ -367,69 +367,69 @@ VTP_Close(struct vtp **vtpp) * Get a connection */ -struct vtp * +struct pfd * VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, unsigned force_fresh) { - struct vtp *vtp; + struct pfd *pfd; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); Lck_Lock(&tp->mtx); - vtp = VTAILQ_FIRST(&tp->connlist); - CHECK_OBJ_ORNULL(vtp, VTP_MAGIC); - if (force_fresh || vtp == NULL || vtp->state == VTP_STATE_STOLEN) - vtp = NULL; + pfd = VTAILQ_FIRST(&tp->connlist); + CHECK_OBJ_ORNULL(pfd, PFD_MAGIC); + if (force_fresh || pfd == NULL || pfd->state == PFD_STATE_STOLEN) + pfd = NULL; else { - assert(vtp->tcp_pool == tp); - assert(vtp->state == VTP_STATE_AVAIL); - VTAILQ_REMOVE(&tp->connlist, vtp, list); - VTAILQ_INSERT_TAIL(&tp->connlist, vtp, list); + assert(pfd->tcp_pool == tp); + assert(pfd->state == PFD_STATE_AVAIL); + VTAILQ_REMOVE(&tp->connlist, pfd, list); + VTAILQ_INSERT_TAIL(&tp->connlist, pfd, list); tp->n_conn--; VSC_C_main->backend_reuse++; - vtp->state = VTP_STATE_STOLEN; - vtp->cond = &wrk->cond; + pfd->state = PFD_STATE_STOLEN; + pfd->cond = &wrk->cond; } tp->n_used++; // Opening mostly works Lck_Unlock(&tp->mtx); - if (vtp != NULL) - return (vtp); - - ALLOC_OBJ(vtp, VTP_MAGIC); - AN(vtp); - INIT_OBJ(vtp->waited, WAITED_MAGIC); - vtp->state = VTP_STATE_USED; - vtp->tcp_pool = tp; - vtp->fd = VTP_Open(tp, tmo, &vtp->addr); - if (vtp->fd < 0) { - FREE_OBJ(vtp); + if (pfd != NULL) + return (pfd); + + ALLOC_OBJ(pfd, PFD_MAGIC); + AN(pfd); + INIT_OBJ(pfd->waited, WAITED_MAGIC); + pfd->state = PFD_STATE_USED; + pfd->tcp_pool = tp; + pfd->fd = VTP_Open(tp, tmo, &pfd->priv); + if (pfd->fd < 0) { + FREE_OBJ(pfd); Lck_Lock(&tp->mtx); tp->n_used--; // Nope, didn't work after all. Lck_Unlock(&tp->mtx); } else VSC_C_main->backend_conn++; - return (vtp); + return (pfd); } /*-------------------------------------------------------------------- */ int -VTP_Wait(struct worker *wrk, struct vtp *vtp, double tmo) +VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo) { struct tcp_pool *tp; int r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); - tp = vtp->tcp_pool; + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + tp = pfd->tcp_pool; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - assert(vtp->cond == &wrk->cond); + assert(pfd->cond == &wrk->cond); Lck_Lock(&tp->mtx); - while (vtp->state == VTP_STATE_STOLEN) { + while (pfd->state == PFD_STATE_STOLEN) { r = Lck_CondWait(&wrk->cond, &tp->mtx, tmo); if (r != 0) { if (r == EINTR) @@ -439,8 +439,8 @@ VTP_Wait(struct worker *wrk, struct vtp *vtp, double tmo) return (1); } } - assert(vtp->state == VTP_STATE_USED); - vtp->cond = NULL; + assert(pfd->state == PFD_STATE_USED); + pfd->cond = NULL; Lck_Unlock(&tp->mtx); return (0); diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index f91cac6..ceae8a9 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -32,17 +32,17 @@ struct tcp_pool; -struct vtp { +struct pfd { unsigned magic; -#define VTP_MAGIC 0x0c5e6592 +#define PFD_MAGIC 0x0c5e6593 int fd; - VTAILQ_ENTRY(vtp) list; - const struct suckaddr *addr; + VTAILQ_ENTRY(pfd) list; + const void *priv; uint8_t state; -#define VTP_STATE_AVAIL (1<<0) -#define VTP_STATE_USED (1<<1) -#define VTP_STATE_STOLEN (1<<2) -#define VTP_STATE_CLEANUP (1<<3) +#define PFD_STATE_AVAIL (1<<0) +#define PFD_STATE_USED (1<<1) +#define PFD_STATE_STOLEN (1<<2) +#define PFD_STATE_CLEANUP (1<<3) struct waited waited[1]; struct tcp_pool *tcp_pool; @@ -72,28 +72,28 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(const struct tcp_pool *, double tmo, const struct suckaddr **); +int VTP_Open(const struct tcp_pool *, double tmo, const void **); /* * Open a new connection and return the adress used. */ -void VTP_Close(struct vtp **); +void VTP_Close(struct pfd **); /* * Close a connection. */ -void VTP_Recycle(const struct worker *, struct vtp **); +void VTP_Recycle(const struct worker *, struct pfd **); /* * Recycle an open connection. */ -struct vtp *VTP_Get(struct tcp_pool *, double tmo, struct worker *, +struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *, unsigned force_fresh); /* * Get a (possibly) recycled connection. */ -int VTP_Wait(struct worker *, struct vtp *, double tmo); +int VTP_Wait(struct worker *, struct pfd *, double tmo); /* * If the connection was recycled (state != VTP_STATE_USED) call this * function before attempting to receive on the connection. From phk at FreeBSD.org Thu Mar 8 00:40:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 00:40:07 +0000 (UTC) Subject: [master] 0933928 Call VTCP_open/close through method pointers Message-ID: <20180308004007.D9ABBB0A52@lists.varnish-cache.org> commit 0933928075d0d29b692975f72319687e3eb48aa3 Author: Poul-Henning Kamp Date: Wed Mar 7 23:19:58 2018 +0000 Call VTCP_open/close through method pointers diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 1c0ca90..fff58c0 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -44,31 +44,83 @@ #include "cache_tcp_pool.h" #include "cache_pool.h" +typedef int cp_open_f(const struct tcp_pool *, double tmo, const void **privp); +typedef void cp_close_f(struct pfd *); + +struct cp_methods { + cp_open_f *open; + cp_close_f *close; +}; + struct tcp_pool { - unsigned magic; -#define TCP_POOL_MAGIC 0x28b0e42a + unsigned magic; +#define TCP_POOL_MAGIC 0x28b0e42a + + const struct cp_methods *methods; - const void *id; - struct suckaddr *ip4; - struct suckaddr *ip6; + const void *id; + struct suckaddr *ip4; + struct suckaddr *ip6; - VTAILQ_ENTRY(tcp_pool) list; - int refcnt; - struct lock mtx; + VTAILQ_ENTRY(tcp_pool) list; + int refcnt; + struct lock mtx; - VTAILQ_HEAD(, pfd) connlist; - int n_conn; + VTAILQ_HEAD(, pfd) connlist; + int n_conn; - VTAILQ_HEAD(, pfd) killlist; - int n_kill; + VTAILQ_HEAD(, pfd) killlist; + int n_kill; - int n_used; + int n_used; }; static struct lock tcp_pools_mtx; static VTAILQ_HEAD(, tcp_pool) tcp_pools = VTAILQ_HEAD_INITIALIZER(tcp_pools); /*-------------------------------------------------------------------- + */ + +static int v_matchproto_(cp_open_f) +vtp_open(const struct tcp_pool *tp, double tmo, const void **privp) +{ + int s; + int msec; + + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + + msec = (int)floor(tmo * 1000.0); + if (cache_param->prefer_ipv6) { + *privp = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + if (s >= 0) + return (s); + } + *privp = tp->ip4; + s = VTCP_connect(tp->ip4, msec); + if (s >= 0) + return (s); + if (!cache_param->prefer_ipv6) { + *privp = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + } + return (s); +} + +static void v_matchproto_(cp_close_f) +vtp_close(struct pfd *pfd) +{ + + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + VTCP_close(&pfd->fd); +} + +static const struct cp_methods vtp_methods = { + .open = vtp_open, + .close = vtp_close, +}; + +/*-------------------------------------------------------------------- * Waiter-handler */ @@ -94,13 +146,13 @@ tcp_handle(struct waited *w, enum wait_event ev, double now) AZ(pthread_cond_signal(pfd->cond)); break; case PFD_STATE_AVAIL: - VTCP_close(&pfd->fd); + tp->methods->close(pfd); VTAILQ_REMOVE(&tp->connlist, pfd, list); tp->n_conn--; FREE_OBJ(pfd); break; case PFD_STATE_CLEANUP: - VTCP_close(&pfd->fd); + tp->methods->close(pfd); tp->n_kill--; VTAILQ_REMOVE(&tp->killlist, pfd, list); memset(pfd, 0x11, sizeof *pfd); @@ -154,6 +206,7 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) ALLOC_OBJ(tp, TCP_POOL_MAGIC); AN(tp); + tp->methods = &vtp_methods; if (ip4 != NULL) tp->ip4 = VSA_Clone(ip4); if (ip6 != NULL) @@ -241,27 +294,8 @@ VTP_Rel(struct tcp_pool **tpp) int VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) { - int s; - int msec; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - - msec = (int)floor(tmo * 1000.0); - if (cache_param->prefer_ipv6) { - *privp = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - if (s >= 0) - return (s); - } - *privp = tp->ip4; - s = VTCP_connect(tp->ip4, msec); - if (s >= 0) - return (s); - if (!cache_param->prefer_ipv6) { - *privp = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - } - return (s); + return (vtp_open(tp, tmo, privp)); } /*-------------------------------------------------------------------- @@ -295,7 +329,7 @@ VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) pfd->waited->func = tcp_handle; pfd->waited->tmo = &cache_param->backend_idle_timeout; if (Wait_Enter(wrk->pool->waiter, pfd->waited)) { - VTCP_close(&pfd->fd); + tp->methods->close(pfd); memset(pfd, 0x33, sizeof *pfd); free(pfd); // XXX: stats @@ -356,7 +390,7 @@ VTP_Close(struct pfd **pfdp) tp->n_kill++; } else { assert(pfd->state == PFD_STATE_USED); - VTCP_close(&pfd->fd); + tp->methods->close(pfd); memset(pfd, 0x44, sizeof *pfd); free(pfd); } @@ -402,7 +436,7 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; pfd->tcp_pool = tp; - pfd->fd = VTP_Open(tp, tmo, &pfd->priv); + pfd->fd = tp->methods->open(tp, tmo, &pfd->priv); if (pfd->fd < 0) { FREE_OBJ(pfd); Lck_Lock(&tp->mtx); From phk at FreeBSD.org Thu Mar 8 00:40:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 00:40:07 +0000 (UTC) Subject: [master] d15c9bd Extract a protocol independent connection pool from the tcp-pool, and implement the tcp-pool with it. Message-ID: <20180308004007.F3576B0A56@lists.varnish-cache.org> commit d15c9bdbba9003148d24968ab34ec57952dacac9 Author: Poul-Henning Kamp Date: Thu Mar 8 00:37:28 2018 +0000 Extract a protocol independent connection pool from the tcp-pool, and implement the tcp-pool with it. It should now be trivial to implement other pools, for instance UDS using this. Filenames subject to change still (s/tcp_pool/conn_pool/ ?) diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 5d49763..9c0231b 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -38,7 +38,6 @@ #include "vtcp.h" #include "vtim.h" -#include "waiter/waiter.h" #include "cache_director.h" #include "cache_backend.h" @@ -78,6 +77,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, unsigned force_fresh) { struct pfd *pfd; + int *fdp; double tmod; char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE]; char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE]; @@ -123,8 +123,9 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, return (NULL); } - assert(pfd->fd >= 0); - AN(pfd->priv); + fdp = PFD_Fd(pfd); + AN(fdp); + assert(*fdp >= 0); Lck_Lock(&bp->mtx); bp->n_conn++; @@ -133,16 +134,16 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, Lck_Unlock(&bp->mtx); if (bp->proxy_header != 0) - VPX_Send_Proxy(pfd->fd, bp->proxy_header, bo->sp); + VPX_Send_Proxy(*fdp, bp->proxy_header, bo->sp); - VTCP_myname(pfd->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - VTCP_hisname(pfd->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); + VTCP_myname(*fdp, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); + VTCP_hisname(*fdp, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", - pfd->fd, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); + *fdp, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); INIT_OBJ(bo->htc, HTTP_CONN_MAGIC); bo->htc->priv = pfd; - bo->htc->rfd = &pfd->fd; + bo->htc->rfd = fdp; FIND_TMO(first_byte_timeout, bo->htc->first_byte_timeout, bo, bp); FIND_TMO(between_bytes_timeout, @@ -175,20 +176,20 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC); + pfd = bo->htc->priv; bo->htc->priv = NULL; - if (pfd->state != PFD_STATE_USED) + if (PFD_State(pfd) != PFD_STATE_USED) assert(bo->htc->doclose == SC_TX_PIPE || bo->htc->doclose == SC_RX_TIMEOUT); if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { - VSLb(bo->vsl, SLT_BackendClose, "%d %s", pfd->fd, + VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd), bp->director->display_name); VTP_Close(&pfd); AZ(pfd); Lck_Lock(&bp->mtx); } else { - assert (pfd->state == PFD_STATE_USED); - VSLb(bo->vsl, SLT_BackendReuse, "%d %s", pfd->fd, + assert (PFD_State(pfd) == PFD_STATE_USED); + VSLb(bo->vsl, SLT_BackendReuse, "%d %s", *PFD_Fd(pfd), bp->director->display_name); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; @@ -230,13 +231,13 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, if (pfd == NULL) return (-1); AN(bo->htc); - if (pfd->state != PFD_STATE_STOLEN) + if (PFD_State(pfd) != PFD_STATE_STOLEN) extrachance = 0; i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, &bo->acct.bereq_bodybytes, 0); - if (pfd->state != PFD_STATE_USED) { + if (PFD_State(pfd) != PFD_STATE_USED) { if (VTP_Wait(wrk, pfd, VTIM_real() + bo->htc->first_byte_timeout) != 0) { bo->htc->doclose = SC_RX_TIMEOUT; @@ -247,7 +248,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, } if (bo->htc->doclose == SC_NULL) { - assert(pfd->state == PFD_STATE_USED); + assert(PFD_State(pfd) == PFD_STATE_USED); if (i == 0) i = V1F_FetchRespHdr(bo); if (i == 0) { @@ -284,9 +285,9 @@ vbe_dir_getip(const struct director *d, struct worker *wrk, CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC); + pfd = bo->htc->priv; - return (pfd->priv); + return (VTP_getip(pfd)); } /*--------------------------------------------------------------------*/ @@ -322,7 +323,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1); VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk)); if (i == 0) - V1P_Process(req, pfd->fd, &v1a); + V1P_Process(req, *PFD_Fd(pfd), &v1a); VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk)); bo->htc->doclose = SC_TX_PIPE; vbe_dir_finish(d, req->wrk, bo); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index a374f70..b5522dc 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -48,7 +48,6 @@ #include "vsa.h" #include "vtcp.h" #include "vtim.h" -#include "waiter/waiter.h" #include "cache_director.h" #include "cache_backend.h" diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index fff58c0..73b7f4f 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -44,25 +44,61 @@ #include "cache_tcp_pool.h" #include "cache_pool.h" -typedef int cp_open_f(const struct tcp_pool *, double tmo, const void **privp); +struct conn_pool; + +/*-------------------------------------------------------------------- + */ + +struct pfd { + unsigned magic; +#define PFD_MAGIC 0x0c5e6593 + int fd; + VTAILQ_ENTRY(pfd) list; + const void *priv; + uint8_t state; + struct waited waited[1]; + struct conn_pool *conn_pool; + + pthread_cond_t *cond; +}; + +unsigned +PFD_State(const struct pfd *p) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + return (p->state); +} + +int * +PFD_Fd(struct pfd *p) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + return (&(p->fd)); +} + +/*-------------------------------------------------------------------- + */ + +typedef int cp_open_f(const struct conn_pool *, double tmo, const void **privp); typedef void cp_close_f(struct pfd *); +typedef int cp_cmp_f(const struct conn_pool *, const void *priv); struct cp_methods { cp_open_f *open; cp_close_f *close; + cp_cmp_f *cmp; }; -struct tcp_pool { +struct conn_pool { unsigned magic; -#define TCP_POOL_MAGIC 0x28b0e42a +#define CONN_POOL_MAGIC 0x85099bc3 const struct cp_methods *methods; const void *id; - struct suckaddr *ip4; - struct suckaddr *ip6; + void *priv; - VTAILQ_ENTRY(tcp_pool) list; + VTAILQ_ENTRY(conn_pool) list; int refcnt; struct lock mtx; @@ -75,273 +111,218 @@ struct tcp_pool { int n_used; }; -static struct lock tcp_pools_mtx; -static VTAILQ_HEAD(, tcp_pool) tcp_pools = VTAILQ_HEAD_INITIALIZER(tcp_pools); - -/*-------------------------------------------------------------------- - */ - -static int v_matchproto_(cp_open_f) -vtp_open(const struct tcp_pool *tp, double tmo, const void **privp) -{ - int s; - int msec; - - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - - msec = (int)floor(tmo * 1000.0); - if (cache_param->prefer_ipv6) { - *privp = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - if (s >= 0) - return (s); - } - *privp = tp->ip4; - s = VTCP_connect(tp->ip4, msec); - if (s >= 0) - return (s); - if (!cache_param->prefer_ipv6) { - *privp = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - } - return (s); -} - -static void v_matchproto_(cp_close_f) -vtp_close(struct pfd *pfd) -{ - - CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - VTCP_close(&pfd->fd); -} +struct tcp_pool { + unsigned magic; +#define TCP_POOL_MAGIC 0x28b0e42a -static const struct cp_methods vtp_methods = { - .open = vtp_open, - .close = vtp_close, + struct suckaddr *ip4; + struct suckaddr *ip6; + struct conn_pool cp[1]; }; +static struct lock conn_pools_mtx; +static VTAILQ_HEAD(, conn_pool) conn_pools = + VTAILQ_HEAD_INITIALIZER(conn_pools); + /*-------------------------------------------------------------------- * Waiter-handler */ static void v_matchproto_(waiter_handle_f) -tcp_handle(struct waited *w, enum wait_event ev, double now) +vcp_handle(struct waited *w, enum wait_event ev, double now) { struct pfd *pfd; - struct tcp_pool *tp; + struct conn_pool *cp; CAST_OBJ_NOTNULL(pfd, w->priv1, PFD_MAGIC); (void)ev; (void)now; - CHECK_OBJ_NOTNULL(pfd->tcp_pool, TCP_POOL_MAGIC); - tp = pfd->tcp_pool; + CHECK_OBJ_NOTNULL(pfd->conn_pool, CONN_POOL_MAGIC); + cp = pfd->conn_pool; - Lck_Lock(&tp->mtx); + Lck_Lock(&cp->mtx); switch (pfd->state) { case PFD_STATE_STOLEN: pfd->state = PFD_STATE_USED; - VTAILQ_REMOVE(&tp->connlist, pfd, list); + VTAILQ_REMOVE(&cp->connlist, pfd, list); AN(pfd->cond); AZ(pthread_cond_signal(pfd->cond)); break; case PFD_STATE_AVAIL: - tp->methods->close(pfd); - VTAILQ_REMOVE(&tp->connlist, pfd, list); - tp->n_conn--; + cp->methods->close(pfd); + VTAILQ_REMOVE(&cp->connlist, pfd, list); + cp->n_conn--; FREE_OBJ(pfd); break; case PFD_STATE_CLEANUP: - tp->methods->close(pfd); - tp->n_kill--; - VTAILQ_REMOVE(&tp->killlist, pfd, list); + cp->methods->close(pfd); + cp->n_kill--; + VTAILQ_REMOVE(&cp->killlist, pfd, list); memset(pfd, 0x11, sizeof *pfd); free(pfd); break; default: WRONG("Wrong pfd state"); } - Lck_Unlock(&tp->mtx); + Lck_Unlock(&cp->mtx); } /*-------------------------------------------------------------------- - * Reference a TCP pool given by {ip4, ip6} pair. Create if it - * doesn't exist already. */ -struct tcp_pool * -VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) +static struct conn_pool * +VCP_Ref(const void *id, const void *priv) { - struct tcp_pool *tp; + struct conn_pool *cp; - assert(ip4 != NULL || ip6 != NULL); - Lck_Lock(&tcp_pools_mtx); - VTAILQ_FOREACH(tp, &tcp_pools, list) { - assert(tp->refcnt > 0); - if (tp->id != id) + Lck_Lock(&conn_pools_mtx); + VTAILQ_FOREACH(cp, &conn_pools, list) { + assert(cp->refcnt > 0); + if (cp->id != id) continue; - if (ip4 == NULL) { - if (tp->ip4 != NULL) - continue; - } else { - if (tp->ip4 == NULL) - continue; - if (VSA_Compare(ip4, tp->ip4)) - continue; - } - if (ip6 == NULL) { - if (tp->ip6 != NULL) - continue; - } else { - if (tp->ip6 == NULL) - continue; - if (VSA_Compare(ip6, tp->ip6)) - continue; - } - tp->refcnt++; - Lck_Unlock(&tcp_pools_mtx); - return (tp); + if (cp->methods->cmp(cp, priv)) + continue; + cp->refcnt++; + Lck_Unlock(&conn_pools_mtx); + return (cp); } - Lck_Unlock(&tcp_pools_mtx); + Lck_Unlock(&conn_pools_mtx); + return (NULL); +} - ALLOC_OBJ(tp, TCP_POOL_MAGIC); - AN(tp); - tp->methods = &vtp_methods; - if (ip4 != NULL) - tp->ip4 = VSA_Clone(ip4); - if (ip6 != NULL) - tp->ip6 = VSA_Clone(ip6); - tp->refcnt = 1; - tp->id = id; - Lck_New(&tp->mtx, lck_tcp_pool); - VTAILQ_INIT(&tp->connlist); - VTAILQ_INIT(&tp->killlist); +/*-------------------------------------------------------------------- + */ - Lck_Lock(&tcp_pools_mtx); - VTAILQ_INSERT_HEAD(&tcp_pools, tp, list); - Lck_Unlock(&tcp_pools_mtx); +static void * +VCP_New(struct conn_pool *cp, const void *id, void *priv, + const struct cp_methods *cm) +{ - return (tp); + AN(cp); + AN(cm); + AN(cm->open); + AN(cm->close); + AN(cm->cmp); + + INIT_OBJ(cp, CONN_POOL_MAGIC); + cp->id = id; + cp->priv = priv; + cp->methods = cm; + cp->refcnt = 1; + Lck_New(&cp->mtx, lck_tcp_pool); + VTAILQ_INIT(&cp->connlist); + VTAILQ_INIT(&cp->killlist); + + Lck_Lock(&conn_pools_mtx); + VTAILQ_INSERT_HEAD(&conn_pools, cp, list); + Lck_Unlock(&conn_pools_mtx); + + return (priv); } + /*-------------------------------------------------------------------- - * Add a reference to a tcp_pool */ -void -VTP_AddRef(struct tcp_pool *tp) +static void +VCP_AddRef(struct conn_pool *cp) { - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); - Lck_Lock(&tcp_pools_mtx); - assert(tp->refcnt > 0); - tp->refcnt++; - Lck_Unlock(&tcp_pools_mtx); + Lck_Lock(&conn_pools_mtx); + assert(cp->refcnt > 0); + cp->refcnt++; + Lck_Unlock(&conn_pools_mtx); } /*-------------------------------------------------------------------- - * Release TCP pool, destroy if last reference. + * Release Conn pool, destroy if last reference. */ -void -VTP_Rel(struct tcp_pool **tpp) +static int +VCP_Rel(struct conn_pool *cp) { - struct tcp_pool *tp; struct pfd *pfd, *pfd2; - TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); - Lck_Lock(&tcp_pools_mtx); - assert(tp->refcnt > 0); - if (--tp->refcnt > 0) { - Lck_Unlock(&tcp_pools_mtx); - return; + Lck_Lock(&conn_pools_mtx); + assert(cp->refcnt > 0); + if (--cp->refcnt > 0) { + Lck_Unlock(&conn_pools_mtx); + return (1); } - AZ(tp->n_used); - VTAILQ_REMOVE(&tcp_pools, tp, list); - Lck_Unlock(&tcp_pools_mtx); - - free(tp->ip4); - free(tp->ip6); - Lck_Lock(&tp->mtx); - VTAILQ_FOREACH_SAFE(pfd, &tp->connlist, list, pfd2) { - VTAILQ_REMOVE(&tp->connlist, pfd, list); - tp->n_conn--; + AZ(cp->n_used); + VTAILQ_REMOVE(&conn_pools, cp, list); + Lck_Unlock(&conn_pools_mtx); + + Lck_Lock(&cp->mtx); + VTAILQ_FOREACH_SAFE(pfd, &cp->connlist, list, pfd2) { + VTAILQ_REMOVE(&cp->connlist, pfd, list); + cp->n_conn--; assert(pfd->state == PFD_STATE_AVAIL); pfd->state = PFD_STATE_CLEANUP; (void)shutdown(pfd->fd, SHUT_WR); - VTAILQ_INSERT_TAIL(&tp->killlist, pfd, list); - tp->n_kill++; + VTAILQ_INSERT_TAIL(&cp->killlist, pfd, list); + cp->n_kill++; } - while (tp->n_kill) { - Lck_Unlock(&tp->mtx); + while (cp->n_kill) { + Lck_Unlock(&cp->mtx); (void)usleep(20000); - Lck_Lock(&tp->mtx); + Lck_Lock(&cp->mtx); } - Lck_Unlock(&tp->mtx); - Lck_Delete(&tp->mtx); - AZ(tp->n_conn); - AZ(tp->n_kill); - - FREE_OBJ(tp); -} - -/*-------------------------------------------------------------------- - * Open a new connection from pool. This is a distinct function since - * probing cannot use a recycled connection. - */ - -int -VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) -{ - - return (vtp_open(tp, tmo, privp)); + Lck_Unlock(&cp->mtx); + Lck_Delete(&cp->mtx); + AZ(cp->n_conn); + AZ(cp->n_kill); + return (0); } /*-------------------------------------------------------------------- * Recycle a connection. */ -void -VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) +static void +VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) { struct pfd *pfd; - struct tcp_pool *tp; + struct conn_pool *cp; int i = 0; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); pfd = *pfdp; *pfdp = NULL; CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - tp = pfd->tcp_pool; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + cp = pfd->conn_pool; + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); assert(pfd->state == PFD_STATE_USED); assert(pfd->fd > 0); - Lck_Lock(&tp->mtx); - tp->n_used--; + Lck_Lock(&cp->mtx); + cp->n_used--; pfd->waited->priv1 = pfd; pfd->waited->fd = pfd->fd; pfd->waited->idle = VTIM_real(); pfd->state = PFD_STATE_AVAIL; - pfd->waited->func = tcp_handle; + pfd->waited->func = vcp_handle; pfd->waited->tmo = &cache_param->backend_idle_timeout; if (Wait_Enter(wrk->pool->waiter, pfd->waited)) { - tp->methods->close(pfd); + cp->methods->close(pfd); memset(pfd, 0x33, sizeof *pfd); free(pfd); // XXX: stats pfd = NULL; } else { - VTAILQ_INSERT_HEAD(&tp->connlist, pfd, list); + VTAILQ_INSERT_HEAD(&cp->connlist, pfd, list); i++; } if (pfd != NULL) - tp->n_conn++; - Lck_Unlock(&tp->mtx); + cp->n_conn++; + Lck_Unlock(&cp->mtx); if (i && DO_DEBUG(DBG_VTC_MODE)) { /* @@ -361,72 +342,73 @@ VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) } } + /*-------------------------------------------------------------------- * Close a connection. */ -void -VTP_Close(struct pfd **pfdp) +static void +VCP_Close(struct pfd **pfdp) { struct pfd *pfd; - struct tcp_pool *tp; + struct conn_pool *cp; pfd = *pfdp; *pfdp = NULL; CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - tp = pfd->tcp_pool; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + cp = pfd->conn_pool; + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); assert(pfd->fd > 0); - Lck_Lock(&tp->mtx); + Lck_Lock(&cp->mtx); assert(pfd->state == PFD_STATE_USED || pfd->state == PFD_STATE_STOLEN); - tp->n_used--; + cp->n_used--; if (pfd->state == PFD_STATE_STOLEN) { (void)shutdown(pfd->fd, SHUT_RDWR); - VTAILQ_REMOVE(&tp->connlist, pfd, list); + VTAILQ_REMOVE(&cp->connlist, pfd, list); pfd->state = PFD_STATE_CLEANUP; - VTAILQ_INSERT_HEAD(&tp->killlist, pfd, list); - tp->n_kill++; + VTAILQ_INSERT_HEAD(&cp->killlist, pfd, list); + cp->n_kill++; } else { assert(pfd->state == PFD_STATE_USED); - tp->methods->close(pfd); + cp->methods->close(pfd); memset(pfd, 0x44, sizeof *pfd); free(pfd); } - Lck_Unlock(&tp->mtx); + Lck_Unlock(&cp->mtx); } /*-------------------------------------------------------------------- * Get a connection */ -struct pfd * -VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, +static struct pfd * +VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, unsigned force_fresh) { struct pfd *pfd; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - Lck_Lock(&tp->mtx); - pfd = VTAILQ_FIRST(&tp->connlist); + Lck_Lock(&cp->mtx); + pfd = VTAILQ_FIRST(&cp->connlist); CHECK_OBJ_ORNULL(pfd, PFD_MAGIC); if (force_fresh || pfd == NULL || pfd->state == PFD_STATE_STOLEN) pfd = NULL; else { - assert(pfd->tcp_pool == tp); + assert(pfd->conn_pool == cp); assert(pfd->state == PFD_STATE_AVAIL); - VTAILQ_REMOVE(&tp->connlist, pfd, list); - VTAILQ_INSERT_TAIL(&tp->connlist, pfd, list); - tp->n_conn--; + VTAILQ_REMOVE(&cp->connlist, pfd, list); + VTAILQ_INSERT_TAIL(&cp->connlist, pfd, list); + cp->n_conn--; VSC_C_main->backend_reuse++; pfd->state = PFD_STATE_STOLEN; pfd->cond = &wrk->cond; } - tp->n_used++; // Opening mostly works - Lck_Unlock(&tp->mtx); + cp->n_used++; // Opening mostly works + Lck_Unlock(&cp->mtx); if (pfd != NULL) return (pfd); @@ -435,13 +417,13 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, AN(pfd); INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; - pfd->tcp_pool = tp; - pfd->fd = tp->methods->open(tp, tmo, &pfd->priv); + pfd->conn_pool = cp; + pfd->fd = cp->methods->open(cp, tmo, &pfd->priv); if (pfd->fd < 0) { FREE_OBJ(pfd); - Lck_Lock(&tp->mtx); - tp->n_used--; // Nope, didn't work after all. - Lck_Unlock(&tp->mtx); + Lck_Lock(&cp->mtx); + cp->n_used--; // Nope, didn't work after all. + Lck_Unlock(&cp->mtx); } else VSC_C_main->backend_conn++; @@ -451,39 +433,244 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, /*-------------------------------------------------------------------- */ -int -VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo) +static int +VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo) { - struct tcp_pool *tp; + struct conn_pool *cp; int r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - tp = pfd->tcp_pool; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + cp = pfd->conn_pool; + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); assert(pfd->cond == &wrk->cond); - Lck_Lock(&tp->mtx); + Lck_Lock(&cp->mtx); while (pfd->state == PFD_STATE_STOLEN) { - r = Lck_CondWait(&wrk->cond, &tp->mtx, tmo); + r = Lck_CondWait(&wrk->cond, &cp->mtx, tmo); if (r != 0) { if (r == EINTR) continue; assert(r == ETIMEDOUT); - Lck_Unlock(&tp->mtx); + Lck_Unlock(&cp->mtx); return (1); } } assert(pfd->state == PFD_STATE_USED); pfd->cond = NULL; - Lck_Unlock(&tp->mtx); + Lck_Unlock(&cp->mtx); return (0); } +/*-------------------------------------------------------------------- + */ + +static int v_matchproto_(cp_open_f) +vtp_open(const struct conn_pool *cp, double tmo, const void **privp) +{ + int s; + int msec; + struct tcp_pool *tp; + + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); + + msec = (int)floor(tmo * 1000.0); + if (cache_param->prefer_ipv6) { + *privp = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + if (s >= 0) + return (s); + } + *privp = tp->ip4; + s = VTCP_connect(tp->ip4, msec); + if (s >= 0) + return (s); + if (!cache_param->prefer_ipv6) { + *privp = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + } + return (s); +} + +static void v_matchproto_(cp_close_f) +vtp_close(struct pfd *pfd) +{ + + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + VTCP_close(&pfd->fd); +} + +struct vtp_cs { + unsigned magic; +#define VTP_CS_MAGIC 0xc1e40447 + const struct suckaddr *ip4; + const struct suckaddr *ip6; +}; + +static int v_matchproto_(cp_cmp_f) +vtp_cmp(const struct conn_pool *cp, const void *priv) +{ + const struct vtp_cs *vcs; + const struct tcp_pool *tp; + + CAST_OBJ_NOTNULL(vcs, priv, VTP_CS_MAGIC); + CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); + if (tp->ip4 == NULL && vcs->ip4 != NULL) + return (1); + if (tp->ip4 != NULL && vcs->ip4 == NULL) + return (1); + if (tp->ip6 == NULL && vcs->ip6 != NULL) + return (1); + if (tp->ip6 != NULL && vcs->ip6 == NULL) + return (1); + if (tp->ip4 != NULL && vcs->ip4 != NULL && + VSA_Compare(tp->ip4, vcs->ip4)) + return (1); + if (tp->ip6 != NULL && vcs->ip6 != NULL && + VSA_Compare(tp->ip6, vcs->ip6)) + return (1); + return (0); +} + +static const struct cp_methods vtp_methods = { + .open = vtp_open, + .close = vtp_close, + .cmp = vtp_cmp, +}; + + +/*-------------------------------------------------------------------- + * Reference a TCP pool given by {ip4, ip6} pair. Create if it + * doesn't exist already. + */ + +struct tcp_pool * +VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) +{ + struct tcp_pool *tp; + struct conn_pool *cp; + struct vtp_cs vcs; + + assert(ip4 != NULL || ip6 != NULL); + INIT_OBJ(&vcs, VTP_CS_MAGIC); + vcs.ip4 = ip4; + vcs.ip6 = ip6; + + cp = VCP_Ref(id, &vcs); + if (cp != NULL) + return (cp->priv); + + ALLOC_OBJ(tp, TCP_POOL_MAGIC); + AN(tp); + if (ip4 != NULL) + tp->ip4 = VSA_Clone(ip4); + if (ip6 != NULL) + tp->ip6 = VSA_Clone(ip6); + return(VCP_New(tp->cp, id, tp, &vtp_methods)); +} + +/*-------------------------------------------------------------------- + * Add a reference to a tcp_pool + */ + +void +VTP_AddRef(struct tcp_pool *tp) +{ + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + VCP_AddRef(tp->cp); +} + +/*-------------------------------------------------------------------- + * Release TCP pool, destroy if last reference. + */ + +void +VTP_Rel(struct tcp_pool **tpp) +{ + struct tcp_pool *tp; + + TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); + if (VCP_Rel(tp->cp)) + return; + + free(tp->ip4); + free(tp->ip6); + FREE_OBJ(tp); +} + +/*-------------------------------------------------------------------- + * Open a new connection from pool. This is a distinct function since + * probing cannot use a recycled connection. + */ + +int +VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) +{ + + return (vtp_open(tp->cp, tmo, privp)); +} + +/*-------------------------------------------------------------------- + * Recycle a connection. + */ + +void +VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) +{ + + VCP_Recycle(wrk, pfdp); +} + +/*-------------------------------------------------------------------- + * Close a connection. + */ + +void +VTP_Close(struct pfd **pfdp) +{ + + VCP_Close(pfdp); +} + +/*-------------------------------------------------------------------- + * Get a connection + */ + +struct pfd * +VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, + unsigned force_fresh) +{ + + return VCP_Get(tp->cp, tmo, wrk, force_fresh); +} + +/*-------------------------------------------------------------------- + */ + +int +VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo) +{ + return (VCP_Wait(wrk, pfd, tmo)); +} + +/*-------------------------------------------------------------------- + */ + +const struct suckaddr * +VTP_getip(struct pfd *pfd) +{ + struct tcp_pool *tp; + + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + CAST_OBJ_NOTNULL(tp, pfd->conn_pool->priv, TCP_POOL_MAGIC); + return (pfd->priv); +} + /*--------------------------------------------------------------------*/ void VTP_Init(void) { - Lck_New(&tcp_pools_mtx, lck_tcp_pool); + Lck_New(&conn_pools_mtx, lck_tcp_pool); } diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index ceae8a9..f5e25c5 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -31,25 +31,20 @@ */ struct tcp_pool; - -struct pfd { - unsigned magic; -#define PFD_MAGIC 0x0c5e6593 - int fd; - VTAILQ_ENTRY(pfd) list; - const void *priv; - uint8_t state; +struct pfd; #define PFD_STATE_AVAIL (1<<0) #define PFD_STATE_USED (1<<1) #define PFD_STATE_STOLEN (1<<2) #define PFD_STATE_CLEANUP (1<<3) - struct waited waited[1]; - struct tcp_pool *tcp_pool; - pthread_cond_t *cond; -}; +/*--------------------------------------------------------------------- + */ + +unsigned PFD_State(const struct pfd *); +int *PFD_Fd(struct pfd *); /*--------------------------------------------------------------------- + * Prototypes */ @@ -98,3 +93,6 @@ int VTP_Wait(struct worker *, struct pfd *, double tmo); * If the connection was recycled (state != VTP_STATE_USED) call this * function before attempting to receive on the connection. */ + +const struct suckaddr *VTP_getip(struct pfd *); + diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 0925a2b..8ec868d 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -94,6 +94,7 @@ +libh mgt_event.h +-sem(VCP_New, custodial(3)) -sem(vsmw_addseg, custodial(2)) -sem(BAN_Free, custodial(1)) -sem(EXP_Inject, custodial(1)) From geoff at uplex.de Thu Mar 8 06:27:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Thu, 8 Mar 2018 06:27:08 +0000 (UTC) Subject: [master] dfeb320 Document the effect of UDS on X-Forwarded-For in "Upgrading to 6.0". Message-ID: <20180308062708.062D761722@lists.varnish-cache.org> commit dfeb32051c96c2a2c1700c83c62a12a75cef1f0a Author: Geoff Simmons Date: Thu Mar 8 07:25:20 2018 +0100 Document the effect of UDS on X-Forwarded-For in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index ff0c6ec..8cfcb67 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -183,6 +183,29 @@ of these changes: * Set ``req.http.Host`` to a distinct value if it is absent before ``vcl_hash`` is entered. +X-Forwarded-For +--------------- + +Varnish automatically appends the value of ``client.ip`` to the +``X-Forwarded-For`` request header that is passed on to backends, or +it creates the header with that value if it is not already present in +the client request. + +If the client request is received over a UDS listener and the PROXY +protocol is not used, then ``0.0.0.0`` will be added to +``X-Forwarded-For``. If you prefer, you can change that in VCL:: + + sub vcl_backend_fetch { + # Assuming that server.identity has been set to an IP + # address with the -i command-line argument. + set bereq.http.X-Forwarded-For + = regsub(bereq.http-X-Forwarded-For, "0.0.0.0$", server.identity); + # ... + } + +Again, this is probably not a concern if ``client.ip`` is set via the +PROXY protocol. + VCL variables ~~~~~~~~~~~~~ From phk at FreeBSD.org Thu Mar 8 08:50:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 08:50:09 +0000 (UTC) Subject: [master] 62c00fd More work to slim down the cache.h API Message-ID: <20180308085009.DFD466406D@lists.varnish-cache.org> commit 62c00fda8ae345093404c308100cecdd7ed2566c Author: Poul-Henning Kamp Date: Thu Mar 8 08:49:15 2018 +0000 More work to slim down the cache.h API diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9f564f1..fb97ede 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -604,18 +604,6 @@ const char *BAN_AddTest(struct ban_proto *, const char *BAN_Commit(struct ban_proto *b); void BAN_Abandon(struct ban_proto *b); -/* for stevedoes resurrecting bans */ -void BAN_Hold(void); -void BAN_Release(void); -void BAN_Reload(const uint8_t *ban, unsigned len); -struct ban *BAN_FindBan(double t0); -void BAN_RefBan(struct objcore *oc, struct ban *); -double BAN_Time(const struct ban *ban); - -/* cache_busyobj.c */ -struct busyobj *VBO_GetBusyObj(struct worker *, const struct req *); -void VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **busyobj); - /* cache_cli.c [CLI] */ extern pthread_t cli_thread; #define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0) @@ -829,13 +817,6 @@ void ObjSendEvent(struct worker *, struct objcore *oc, unsigned event); const char *body_status_2str(enum body_status e); const char *sess_close_2str(enum sess_close sc, int want_desc); -/* cache_req.c */ -struct req *Req_New(const struct worker *, struct sess *); -void Req_Release(struct req *); -void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); -void Req_Fail(struct req *req, enum sess_close reason); -void Req_AcctLogCharge(struct VSC_main *, struct req *); - /* cache_req_body.c */ int VRB_Ignore(struct req *); ssize_t VRB_Cache(struct req *, ssize_t maxsize); @@ -888,22 +869,14 @@ VSLb_ts_busyobj(struct busyobj *bo, const char *event, double now) VSLb_ts(bo->vsl, event, bo->t_first, &bo->t_prev, now); } - -void VSL_Flush(struct vsl_log *, int overflow); - /* cache_vcl.c */ const char *VCL_Name(const struct vcl *); -const char *VCL_Method_Name(unsigned); -#define VCL_MET_MAC(l,u,t,b) \ - void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \ - struct busyobj *bo, void *specific); -#include "tbl/vcl_returns.h" /* cache_vrt.c */ - /* * These prototypes go here, because we do not want to pollute vrt.h * with va_list. VCC never generates direct calls to them. + * XXX: We should deprecate these (ref: STRANDS) */ const char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap); char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap); @@ -947,10 +920,6 @@ void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); void RFC2616_Response_Body(const struct worker *, const struct busyobj *); -/* stevedore.c */ -int STV_NewObject(struct worker *, struct objcore *, - const struct stevedore *, unsigned len); - /* * A normal pointer difference is signed, but we never want a negative value * so this little tool will make sure we don't get that. @@ -989,13 +958,6 @@ Tlen(const txt t) */ #define W_TIM_real(w) ((w)->lastused = VTIM_real()) -/* - * XXX should cache.h refer to cache_param if common_param.h - * is not in $DIST ? - */ -#define FEATURE(x) COM_FEATURE(cache_param->feature_bits, x) -#define DO_DEBUG(x) COM_DO_DEBUG(cache_param->debug_bits, x) - #define DSL(debug_bit, id, ...) \ do { \ if (DO_DEBUG(debug_bit)) \ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index ae1ceec..c81ac3b 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -63,6 +63,20 @@ void VTP_Init(void); /* cache_backend_poll.c */ void VBP_Init(void); +/* cache_ban.c */ + +/* for stevedoes resurrecting bans */ +void BAN_Hold(void); +void BAN_Release(void); +void BAN_Reload(const uint8_t *ban, unsigned len); +struct ban *BAN_FindBan(double t0); +void BAN_RefBan(struct objcore *oc, struct ban *); +double BAN_Time(const struct ban *ban); + +/* cache_busyobj.c */ +struct busyobj *VBO_GetBusyObj(struct worker *, const struct req *); +void VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **busyobj); + /* cache_director.c */ int VDI_GetHdr(struct worker *, struct busyobj *); int VDI_GetBody(struct worker *, struct busyobj *); @@ -158,6 +172,13 @@ int Pool_Task_Any(struct pool_task *task, enum task_prio prio); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, const char *r); +/* cache_req.c */ +struct req *Req_New(const struct worker *, struct sess *); +void Req_Release(struct req *); +void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); +void Req_Fail(struct req *req, enum sess_close reason); +void Req_AcctLogCharge(struct VSC_main *, struct req *); + /* cache_req_fsm.c [CNT] */ enum req_fsm_nxt { @@ -195,6 +216,7 @@ void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len); void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, uint32_t vxid); void VSL_End(struct vsl_log *vsl); +void VSL_Flush(struct vsl_log *, int overflow); /* cache_vary.c */ int VRY_Create(struct busyobj *bo, struct vsb **psb); @@ -214,6 +236,12 @@ void VCL_Ref(struct vcl *); void VCL_Refresh(struct vcl **); void VCL_Rel(struct vcl **); const char *VCL_Return_Name(unsigned); +const char *VCL_Method_Name(unsigned); +#define VCL_MET_MAC(l,u,t,b) \ + void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \ + struct busyobj *bo, void *specific); +#include "tbl/vcl_returns.h" + typedef int vcl_be_func(struct cli *, struct director *, void *); @@ -244,9 +272,15 @@ const struct stevedore *STV_next(void); int STV_BanInfoDrop(const uint8_t *ban, unsigned len); int STV_BanInfoNew(const uint8_t *ban, unsigned len); void STV_BanExport(const uint8_t *banlist, unsigned len); +int STV_NewObject(struct worker *, struct objcore *, + const struct stevedore *, unsigned len); + #if WITH_PERSISTENT_STORAGE /* storage_persistent.c */ void SMP_Ready(void); #endif +#define FEATURE(x) COM_FEATURE(cache_param->feature_bits, x) +#define DO_DEBUG(x) COM_DO_DEBUG(cache_param->debug_bits, x) + From dridi at varni.sh Thu Mar 8 08:55:54 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 8 Mar 2018 09:55:54 +0100 Subject: [master] 62c00fd More work to slim down the cache.h API In-Reply-To: <20180308085009.DFD466406D@lists.varnish-cache.org> References: <20180308085009.DFD466406D@lists.varnish-cache.org> Message-ID: > -/* > - * XXX should cache.h refer to cache_param if common_param.h > - * is not in $DIST ? > - */ > -#define FEATURE(x) COM_FEATURE(cache_param->feature_bits, x) > -#define DO_DEBUG(x) COM_DO_DEBUG(cache_param->debug_bits, x) > - > #define DSL(debug_bit, id, ...) \ > do { \ > if (DO_DEBUG(debug_bit)) \ DO_DEBUG is removed from the API but used right below in DSL(). Also should we really hide those flags? There's at least vtc_mode that could be really useful to VMOD authors. Dridi From phk at phk.freebsd.dk Thu Mar 8 09:08:42 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Thu, 08 Mar 2018 09:08:42 +0000 Subject: [master] 62c00fd More work to slim down the cache.h API In-Reply-To: References: <20180308085009.DFD466406D@lists.varnish-cache.org> Message-ID: <33698.1520500122@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> -/* >> - * XXX should cache.h refer to cache_param if common_param.h >> - * is not in $DIST ? >> - */ >> -#define FEATURE(x) COM_FEATURE(cache_param->feature_bits, x) >> -#define DO_DEBUG(x) COM_DO_DEBUG(cache_param->debug_bits, x) >> - >> #define DSL(debug_bit, id, ...) \ >> do { \ >> if (DO_DEBUG(debug_bit)) \ > >DO_DEBUG is removed from the API but used right below in DSL(). Also >should we really hide those flags? There's at least vtc_mode that >could be really useful to VMOD authors. We don't expose the layout of the param structure at the cache.h API level, so these macros are useless. I will move DSL as well. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Thu Mar 8 09:10:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 8 Mar 2018 09:10:10 +0000 (UTC) Subject: [master] c8e2aba Also move DSL out of cache.h API Message-ID: <20180308091010.447EE64754@lists.varnish-cache.org> commit c8e2aba9bd97a9b7a0ec57071b87055975e07d1e Author: Poul-Henning Kamp Date: Thu Mar 8 09:09:31 2018 +0000 Also move DSL out of cache.h API Spotted by: dridi diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index fb97ede..eb1c43d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -958,12 +958,6 @@ Tlen(const txt t) */ #define W_TIM_real(w) ((w)->lastused = VTIM_real()) -#define DSL(debug_bit, id, ...) \ - do { \ - if (DO_DEBUG(debug_bit)) \ - VSL(SLT_Debug, (id), __VA_ARGS__); \ - } while (0) - #define PAN_CheckMagic(vsb, ptr, exp) \ do { \ if ((ptr)->magic != (exp)) \ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index c81ac3b..bad6380 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -284,3 +284,9 @@ void SMP_Ready(void); #define FEATURE(x) COM_FEATURE(cache_param->feature_bits, x) #define DO_DEBUG(x) COM_DO_DEBUG(cache_param->debug_bits, x) +#define DSL(debug_bit, id, ...) \ + do { \ + if (DO_DEBUG(debug_bit)) \ + VSL(SLT_Debug, (id), __VA_ARGS__); \ + } while (0) + From geoff at uplex.de Thu Mar 8 09:24:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Thu, 8 Mar 2018 09:24:07 +0000 (UTC) Subject: [master] f214d59 Document effects of UDS on std.port() and std.set_ip_tos(). Message-ID: <20180308092407.B7EB164B97@lists.varnish-cache.org> commit f214d59f0b989ca1c395c0ae2c3e88fa68b2de2d Author: Geoff Simmons Date: Thu Mar 8 10:22:32 2018 +0100 Document effects of UDS on std.port() and std.set_ip_tos(). diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 62579b5..fe7fd63 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -60,7 +60,8 @@ $Function VOID set_ip_tos(INT tos) Description Sets the IP type-of-service (TOS) field for the current session - to *tos*. + to *tos*. Silently ignored if the listen address is a Unix + domain socket. Please note that the TOS field is not removed by the end of the request so probably want to set it on every request should you utilize it. @@ -231,7 +232,9 @@ Description $Function INT port(IP ip) Description - Returns the port number of the IP address *ip*. + Returns the port number of the IP address *ip*. Always returns + 0 for a ``*.ip`` variable whose value is ``0.0.0.0`` because + the listen address is a Unix domain socket. $Function VOID rollback(HTTP h) From geoff at uplex.de Thu Mar 8 09:35:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Thu, 8 Mar 2018 09:35:09 +0000 (UTC) Subject: [master] 8943203 Mention UDS effects on VMOD std in "Upgrading to 6.0". Message-ID: <20180308093509.97CA064EF0@lists.varnish-cache.org> commit 89432039dc5eeca8810b6b0de890326026af5cdc Author: Geoff Simmons Date: Thu Mar 8 10:34:07 2018 +0100 Mention UDS effects on VMOD std in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 8cfcb67..db8c9cd 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -206,6 +206,14 @@ protocol is not used, then ``0.0.0.0`` will be added to Again, this is probably not a concern if ``client.ip`` is set via the PROXY protocol. +VMOD std +-------- + +The VMOD std function :ref:`func_port` always returns 0 when applied +to a ``*.ip`` variable whose value is set to ``0.0.0.0`` because the +listener is UDS. :ref:`func_set_ip_tos` is silently ignored when the +listener is UDS. + VCL variables ~~~~~~~~~~~~~ From daghf at varnish-software.com Thu Mar 8 09:42:07 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 8 Mar 2018 09:42:07 +0000 (UTC) Subject: [master] d7f25da Constify Message-ID: <20180308094207.9153F65126@lists.varnish-cache.org> commit d7f25da2a6d0b14380ab13831fdfc626c1a81d84 Author: Dag Haavi Finstad Date: Thu Mar 8 10:41:12 2018 +0100 Constify diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 7b56b44..63401e0 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -405,7 +405,7 @@ static const struct h2_setting_s * const h2_setting_tbl[] = { #define H2_SETTING_TBL_LEN (sizeof(h2_setting_tbl)/sizeof(h2_setting_tbl[0])) static void -h2_win_adjust(struct h2_sess *h2, uint32_t oldval, uint32_t newval) +h2_win_adjust(const struct h2_sess *h2, uint32_t oldval, uint32_t newval) { struct h2_req *r2; From geoff at uplex.de Thu Mar 8 09:55:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Thu, 8 Mar 2018 09:55:08 +0000 (UTC) Subject: [master] 148893f Some editorial polish in "Upgrading to 6.0". Message-ID: <20180308095508.1ECE465545@lists.varnish-cache.org> commit 148893f11ed897719c7fb29635ecafae4d96c793 Author: Geoff Simmons Date: Thu Mar 8 10:53:52 2018 +0100 Some editorial polish in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index db8c9cd..444c301 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -100,9 +100,10 @@ against ACLs can only be run against IP-valued elements. This means that if a ``*.ip`` variable whose value is ``0.0.0.0`` due to the use of UDS is matched against an ACL, the match can only succeed if the ACL includes ``0.0.0.0``. If you currently have a -security requirement that depends on an ACL matching a range of IP -addresses, then that will continue to work with a UDS listener (since -you almost certainly have not included ``0.0.0.0`` in that range). +security requirement that depends on IP addresses *not* matching an +ACL unless they belong to a specified range, then that will continue +to work with a UDS listener (since you almost certainly have not +included ``0.0.0.0`` in that range). Recall again that ``client.ip`` and ``server.ip`` are set by the PROXY protocol. So if you have a UDS listener configured to use PROXY and @@ -257,7 +258,7 @@ Other changes file when it executes ``bind(2)``. To make it easier for other processes to connect to the socket, the server's umask is temporarily set to 0 before the listen is attempted, to minimize - issues with permissions. No further attempted is made to set the + issues with permissions. No further attempt is made to set the socket's permissions. To test a Varnish instance listening at a UDS, just use the From dridi at varnish-software.com Thu Mar 8 11:02:07 2018 From: dridi at varnish-software.com (Dridi Boukelmoune) Date: Thu, 8 Mar 2018 11:02:07 +0000 (UTC) Subject: [master] 82c00f6 Keep the child as "starting" until it's fully launched Message-ID: <20180308110207.2A82490B9C@lists.varnish-cache.org> commit 82c00f6f37b708a9cc6780a921ead474180c233c Author: Dridi Boukelmoune Date: Tue Mar 6 22:15:49 2018 +0100 Keep the child as "starting" until it's fully launched diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 88acd0e..fe16d89 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -395,7 +395,6 @@ mgt_launch_child(struct cli *cli) mgt_cli_start_child(child_cli_in, child_cli_out); child_pid = pid; - child_state = CH_RUNNING; if (mgt_push_vcls(cli, &u, &p)) { VCLI_SetResult(cli, u); @@ -414,6 +413,8 @@ mgt_launch_child(struct cli *cli) MCH_Stop_Child(); return; } + + child_state = CH_RUNNING; } /*===================================================================== @@ -575,7 +576,7 @@ void MCH_Cli_Fail(void) { - if (child_state != CH_RUNNING) + if (child_state != CH_RUNNING && child_state != CH_STARTING) return; if (child_pid < 0) return; @@ -597,7 +598,7 @@ void MCH_Stop_Child(void) { - if (child_state != CH_RUNNING) + if (child_state != CH_RUNNING && child_state != CH_STARTING) return; child_state = CH_STOPPING; From daghf at varnish-software.com Thu Mar 8 12:43:07 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 8 Mar 2018 12:43:07 +0000 (UTC) Subject: [master] 7f076ef Allow an h/2 flow control to go negative Message-ID: <20180308124307.1DFD092CBD@lists.varnish-cache.org> commit 7f076ef2bc40d43ea92d3492023482e411f79826 Author: Dag Haavi Finstad Date: Thu Mar 8 13:41:05 2018 +0100 Allow an h/2 flow control to go negative diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 63401e0..1425084 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -358,7 +358,7 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) else if (r2->cond != NULL) AZ(pthread_cond_signal(r2->cond)); Lck_Unlock(&h2->sess->mtx); - if (r2->t_window >= (1LLU << 31)) + if (r2->t_window >= (1LL << 31)) return (H2SE_FLOW_CONTROL_ERROR); return (0); } From daghf at varnish-software.com Thu Mar 8 13:56:06 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 8 Mar 2018 13:56:06 +0000 (UTC) Subject: [master] b3691b3 Tidy up Message-ID: <20180308135606.8B83C942CB@lists.varnish-cache.org> commit b3691b396c0da62ef355d3c67aa6715eedf2fb42 Author: Dag Haavi Finstad Date: Thu Mar 8 14:55:29 2018 +0100 Tidy up diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 1425084..542a10e 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -271,7 +271,7 @@ h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { ASSERT_RXTHR(h2); - (void)r2; + if (h2->rxf_len != 8) // rfc7540,l,2364,2366 return (H2CE_FRAME_SIZE_ERROR); AZ(h2->rxf_stream); // rfc7540,l,2359,2362 From guillaume at varnish-software.com Thu Mar 8 18:22:07 2018 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 8 Mar 2018 18:22:07 +0000 (UTC) Subject: [master] 9e174d4 Log cacheability Message-ID: <20180308182207.47C7B99204@lists.varnish-cache.org> commit 9e174d4c1d38b752e029d25c28127f9eeace39c9 Author: Guillaume Quintard Date: Tue Mar 6 12:27:16 2018 +0100 Log cacheability diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index cef9b39..ddecb9c 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -198,9 +198,10 @@ RFC2616_Ttl(struct busyobj *bo, double now, double *t_origin, } VSLb(bo->vsl, SLT_TTL, - "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", + "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %s", *ttl, *grace, *keep, now, - *t_origin, h_date, h_expires, max_age); + *t_origin, h_date, h_expires, max_age, + bo->uncacheable ? "uncacheable" : "cacheable"); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 11c8e5a..5ac0a00 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -105,7 +105,7 @@ VRT_hit_for_pass(VRT_CTX, VCL_DURATION d) oc->ttl = d; oc->grace = 0.0; oc->keep = 0.0; - VSLb(ctx->vsl, SLT_TTL, "HFP %.0f %.0f %.0f %.0f", + VSLb(ctx->vsl, SLT_TTL, "HFP %.0f %.0f %.0f %.0f uncacheable", oc->ttl, oc->grace, oc->keep, oc->t_origin); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 05f058a..2d08ffa 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -229,8 +229,11 @@ VRT_r_bereq_uncacheable(VRT_CTX) VCL_VOID VRT_l_beresp_uncacheable(VRT_CTX, VCL_BOOL a) { + struct objcore *oc; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->fetch_objcore, OBJCORE_MAGIC); if (ctx->bo->uncacheable && !a) { VSLb(ctx->vsl, SLT_VCL_Error, @@ -238,6 +241,11 @@ VRT_l_beresp_uncacheable(VRT_CTX, VCL_BOOL a) } else if (a) { ctx->bo->uncacheable = 1; } + oc = ctx->bo->fetch_objcore; + + VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %s", \ + oc->ttl, oc->grace, oc->keep, oc->t_origin, \ + ctx->bo->uncacheable ? "uncacheable" : "cacheable");\ } VCL_BOOL @@ -598,8 +606,9 @@ VRT_l_##which##_##fld(VRT_CTX, VCL_DURATION a) \ if (a < 0.0) \ a = 0.0; \ oc->fld = a; \ - VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f", \ - oc->ttl, oc->grace, oc->keep, oc->t_origin); \ + VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %s", \ + oc->ttl, oc->grace, oc->keep, oc->t_origin, \ + ctx->bo->uncacheable ? "uncacheable" : "cacheable");\ } #define VRT_DO_EXP_R(which, oc, fld, offset) \ diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 728011b..89a422a 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -209,25 +209,27 @@ SLTM(LostHeader, 0, "Failed attempt to set HTTP header", SLTM(TTL, 0, "TTL set on object", "A TTL record is emitted whenever the ttl, grace or keep" - " values for an object is set.\n\n" + " values for an object is set as well as whether the object is " + " cacheable or not.\n\n" "The format is::\n\n" - "\t%s %d %d %d %d [ %d %d %u %u ]\n" - "\t| | | | | | | | |\n" - "\t| | | | | | | | +- Max-Age from Cache-Control header\n" - "\t| | | | | | | +---- Expires header\n" - "\t| | | | | | +------- Date header\n" - "\t| | | | | +---------- Age (incl Age: header value)\n" - "\t| | | | +--------------- Reference time for TTL\n" - "\t| | | +------------------ Keep\n" - "\t| | +--------------------- Grace\n" - "\t| +------------------------ TTL\n" - "\t+--------------------------- \"RFC\", \"VCL\" or \"HFP\"\n" + "\t%s %d %d %d %d [ %d %d %u %u ] %s\n" + "\t| | | | | | | | | |\n" + "\t| | | | | | | | | +- \"cacheable\" or \"uncacheable\"\n" + "\t| | | | | | | | +------ Max-Age from Cache-Control header\n" + "\t| | | | | | | +--------- Expires header\n" + "\t| | | | | | +------------ Date header\n" + "\t| | | | | +--------------- Age (incl Age: header value)\n" + "\t| | | | +-------------------- Reference time for TTL\n" + "\t| | | +----------------------- Keep\n" + "\t| | +-------------------------- Grace\n" + "\t| +----------------------------- TTL\n" + "\t+-------------------------------- \"RFC\", \"VCL\" or \"HFP\"\n" "\n" - "The last four fields are only present in \"RFC\" headers.\n\n" + "The four optional fields are only present in \"RFC\" headers.\n\n" "Examples::\n\n" - "\tRFC 60 10 -1 1312966109 1312966109 1312966109 0 60\n" - "\tVCL 120 10 0 1312966111\n" - "\tHFP 2 0 0 1312966113\n" + "\tRFC 60 10 -1 1312966109 1312966109 1312966109 0 60 cacheable\n" + "\tVCL 120 10 0 1312966111 uncacheable\n" + "\tHFP 2 0 0 1312966113 uncacheable\n" "\n" ) From geoff at uplex.de Fri Mar 9 08:36:13 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 9 Mar 2018 08:36:13 +0000 (UTC) Subject: [master] 3088b6f Document -a UDS in "Upgrading to 6.0". Message-ID: <20180309083613.94AADAF24A@lists.varnish-cache.org> commit 3088b6fd42bbca90da3126d8e73ea27d78de5fb3 Author: Geoff Simmons Date: Fri Mar 9 09:34:48 2018 +0100 Document -a UDS in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 444c301..27772f4 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -9,6 +9,49 @@ XXX: Most important change first XXX ... +Unix domain sockets as listen addresses +======================================= + +The ``varnishd -a`` command-line argument now has this form, where the +``address`` may be a Unix domain socket, identified as such when it +begins with ``/`` (see varnishd :ref:`ref-varnishd-options`):: + + -a [name=][address][:port][,PROTO][,user=][,group=][,mode=] + +That means that an absolute path must always be specified for the +socket file. The socket file is created when Varnish starts, and any +file that may exist at that path is unlinked first. You can use the +optional ``user``, ``group`` and ``mode`` sub-arguments to set +permissions of the new socket file; use names for ``user`` and +``group`` (not numeric IDs), and a 3-digit octal number for +``mode``. This is done by the management process, so creating the +socket file and setting permissions are done with the privileges of +the management process owner. + +There are some platform-specific restrictions on the use of UDSen to +which you will have to conform. Here are some things we know of, but +this list is by no means authoritative or exhaustive; always consult +your platform documentation (usually in ``man unix``): + +* There is a maximum permitted length of the path for a socket file, + considerably shorter than the maximum for the file system; usually a + bit over 100 bytes. + +* On FreeBSD and other BSD-derived systems, the permissions of the + socket file do not restrict which processes can connect to the + socket. + +* On Linux, a process connecting to the socket must have write + permissions on the socket file. + +On any system, a process connecting to the socket must be able to +access the socket file. So you can reliably restrict access by +restricting permissions on the directory containing the socket (but +that must be done outside of the Varnish configuration). + +If you continue using only IP addresses in your ``-a`` arguments, you +won't have to change them. + varnishd parameters =================== From geoff at uplex.de Fri Mar 9 09:02:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 9 Mar 2018 09:02:08 +0000 (UTC) Subject: [master] f98d28c Some editorial fiddling with "Upgrading to 6.0". Message-ID: <20180309090208.58A4CAF989@lists.varnish-cache.org> commit f98d28c5cf1f377feef26fb6fe684e3c663e14ac Author: Geoff Simmons Date: Fri Mar 9 10:01:14 2018 +0100 Some editorial fiddling with "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 27772f4..f40c47d 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -78,9 +78,13 @@ Unix domain sockets and VCL We have made an effort to adapt the support of Unix domain sockets in VCL so that you may not have to change anything in your VCL deployment -at all, other than changing the version to 4.1. Most importantly, this -affects the meaning of the ``*.ip`` variables and the use of ACLs; and -there are a number of other details you should consider. +at all, other than changing the version to 4.1. + +The short story is that where VCL requires an IP value, the value is +``0.0.0.0:0`` for a connection that was addressed as a UDS -- the "any +IPv4" address with port 0. So your use of IP-valued elements in VCL +will continue to work and may not have to change, but there are some +consequences that you should consider, covered in the following. If you don't use UDSen, then nothing about VCL changes. UDS support requires version 4.1, so if you are keeping your VCL level at 4.0 (and @@ -90,9 +94,8 @@ concern. ``client.ip``, ``server.ip``, ``local.ip`` and ``remote.ip`` ------------------------------------------------------------ -When referring to a connection that was in fact addressed as a UDS, -the ``*.ip`` variables always have a value equivalent to the IPv4 -address ``0.0.0.0:0`` -- the "any IPv4" address with port 0. +These variables have the value ``0.0.0.0`` for a connection that was +addressed as a UDS. Remember that if you are using the PROXY protocol, then ``client.ip`` and ``server.ip`` are set to the addresses sent in the PROXY header by @@ -105,9 +108,9 @@ If you have more than one UDS listener (more than one ``-a`` command-line argument specifying a socket path), then you may not be able to use the ``*.ip`` variables to tell them apart, especially since ``local.ip`` will be ``0.0.0.0`` for all of them. If you need to -distinguish such addresses in VCL, you can use ``local.socket`` (which -is the name given for the ``-a`` argument; ``a0``, ``a1`` etc. by -default) or ``local.endpoint``, which in the case of UDS is the path +distinguish such addresses in VCL, you can use ``local.socket``, which +is the name given for the ``-a`` argument (``a0``, ``a1`` etc. by +default), or ``local.endpoint``, which in the case of UDS is the path given in the ``-a`` argument. You can, for example, use string operations such as regex matching on ``local.endpoint`` to determine properties of the path address:: From phk at FreeBSD.org Fri Mar 9 09:32:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 9 Mar 2018 09:32:08 +0000 (UTC) Subject: [master] bc60965 Flexelinting. VUT_Error never returns. Message-ID: <20180309093208.295D5B0235@lists.varnish-cache.org> commit bc609650ca0c97d2d031b711010c5e54f0e13a4a Author: Poul-Henning Kamp Date: Fri Mar 9 09:31:02 2018 +0000 Flexelinting. VUT_Error never returns. diff --git a/bin/varnishadm/flint.lnt b/bin/varnishadm/flint.lnt index e69de29..c67ba85 100644 --- a/bin/varnishadm/flint.lnt +++ b/bin/varnishadm/flint.lnt @@ -0,0 +1 @@ +-sem(usage, r_no) diff --git a/bin/varnishhist/flint.lnt b/bin/varnishhist/flint.lnt index 78ae771..a4289bf 100644 --- a/bin/varnishhist/flint.lnt +++ b/bin/varnishhist/flint.lnt @@ -1,2 +1,3 @@ -efile(451, "varnishhist_profiles.h") -efile(451, "varnishhist_options.h") +-sem(usage, r_no) diff --git a/bin/varnishlog/flint.lnt b/bin/varnishlog/flint.lnt index 509f2fe..204e850 100644 --- a/bin/varnishlog/flint.lnt +++ b/bin/varnishlog/flint.lnt @@ -10,3 +10,4 @@ -e788 // enum constant '___' not used within defaulted switch -e641 // Converting enum '___' to '___' +-sem(usage,r_no) diff --git a/bin/varnishncsa/flint.lnt b/bin/varnishncsa/flint.lnt index 5270ff2..f7767d1 100644 --- a/bin/varnishncsa/flint.lnt +++ b/bin/varnishncsa/flint.lnt @@ -1,2 +1,3 @@ -efile(451, "varnishncsa_options.h") +-sem(usage, r_no) diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 5573407..cd3f5a0 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1146,9 +1146,8 @@ read_format(const char *formatfile) free(fmt); if (feof(fmtfile)) VUT_Error(vut, 1, "Empty format file"); - else - VUT_Error(vut, 1, "Can't read format from file (%s)", - strerror(errno)); + VUT_Error(vut, 1, "Can't read format from file (%s)", + strerror(errno)); } fclose(fmtfile); if (fmt[fmtlen - 1] == '\n') diff --git a/bin/varnishstat/flint.lnt b/bin/varnishstat/flint.lnt index b3d6bb7..0dbb9d8 100644 --- a/bin/varnishstat/flint.lnt +++ b/bin/varnishstat/flint.lnt @@ -3,22 +3,4 @@ -efile(451, varnishstat_options.h) --esym(850, av) - --e712 // 14 Info 712 Loss of precision (___) (___ to ___) --e747 // 16 Info 747 Significant prototype coercion (___) ___ to ___ - --e763 // Redundant declaration for symbol '...' previously declared - --e732 // Loss of sign (arg. no. 2) (int to unsigned --e713 // Loss of precision (assignment) (unsigned long long to long long) - -/////////////////////////////////////////////////////////////////////// -// Varnishstat specific - --emacro(774, MAC_STAT) --emacro(506, MAC_STAT) --emacro(525, MAC_STAT) --efile(451, "*/include/stat_field.h") --e850 // for loop index variable '___' whose type category is '___' is modified in body of the for loop that began at '___' - +-sem(usage, r_no) diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index d498cca..ad9c221 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -118,7 +118,7 @@ static int sample = 0; static int scale = 1; static double t_sample = 0.; static double interval = 1.; -static int vsm_status = 0; +static unsigned vsm_status = 0; static void init_hitrate(void) @@ -395,16 +395,16 @@ make_windows(void) } static void -print_duration(WINDOW *w, time_t t) +print_duration(WINDOW *w, uint64_t t) { - wprintw(w, "%4jd+%02jd:%02jd:%02jd", - (intmax_t)t / 86400, (intmax_t)(t % 86400) / 3600, - (intmax_t)(t % 3600) / 60, (intmax_t)t % 60); + wprintw(w, "%4ju+%02ju:%02ju:%02ju", + (uintmax_t)t / 86400, (uintmax_t)(t % 86400) / 3600, + (uintmax_t)(t % 3600) / 60, (uintmax_t)t % 60); } static void -running(WINDOW *w, time_t up, int flg) +running(WINDOW *w, uint64_t up, int flg) { if (vsm_status & flg) { print_duration(w_status, up); @@ -418,8 +418,8 @@ running(WINDOW *w, time_t up, int flg) static void draw_status(void) { - time_t up_mgt = 0; - time_t up_chld = 0; + uint64_t up_mgt = 0; + uint64_t up_chld = 0; AN(w_status); diff --git a/bin/varnishtop/flint.lnt b/bin/varnishtop/flint.lnt index 4d95f0a..696ce2d 100644 --- a/bin/varnishtop/flint.lnt +++ b/bin/varnishtop/flint.lnt @@ -8,6 +8,9 @@ -e732 // Loss of sign (arg. no. 2) (int to unsigned -e713 // Loss of precision (assignment) (unsigned long long to long long) +-sem(t_order_VRB_INSERT, custodial(2)) +-sem(t_key_VRB_INSERT, custodial(2)) + /////////////////////////////////////////////////////////////////////// // Varnishstat specific diff --git a/flint.lnt b/flint.lnt index 8b055f8..e49efe5 100644 --- a/flint.lnt +++ b/flint.lnt @@ -92,6 +92,11 @@ -esym(765, vmod_enum_*) // external '___' (___) could be made static /////////////////////////////////////////////////////////////////////// +// + +-sem(VUT_Error, r_no) + +/////////////////////////////////////////////////////////////////////// // -sem(VAS_Fail, r_no) // does not return @@ -130,8 +135,6 @@ -esym(534, VSB_vprintf) -esym(534, VSB_putc) - - /////////////////////////////////////////////////////////////////////// // diff --git a/include/vut.h b/include/vut.h index 9b8c25c..2528de3 100644 --- a/include/vut.h +++ b/include/vut.h @@ -69,7 +69,7 @@ struct VUT { }; void VUT_Error(struct VUT *, int status, const char *fmt, ...) - v_printflike_(3, 4); + v_noreturn_ v_printflike_(3, 4); int VUT_Arg(struct VUT *, int opt, const char *arg); diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 4d4572f..e852a0d 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -124,6 +124,7 @@ VUT_Error(struct VUT *vut, int status, const char *fmt, ...) va_start(ap, fmt); vut->error_f(vut, status, fmt, ap); va_end(ap); + exit(2); } int From phk at FreeBSD.org Fri Mar 9 09:53:06 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 9 Mar 2018 09:53:06 +0000 (UTC) Subject: [master] 33ed161 More Flexelinting Message-ID: <20180309095306.F1642B081A@lists.varnish-cache.org> commit 33ed16191d786a878787fb97bd23f562f6a60d24 Author: Poul-Henning Kamp Date: Fri Mar 9 09:52:26 2018 +0000 More Flexelinting diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index fdc773a..3f72597 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -187,14 +187,14 @@ update(void) scale, nhist, delay); for (j = 2; j < LINES - 3; j += 5) - mvprintw(j, 0, "%u_", (LINES - 3 - j) * scale); + mvprintw(j, 0, "%u_", ((LINES - 3) - j) * scale); /* show them */ for (i = 0; i < n; ++i) { for (j = 0; j < bm[i] / scale; ++j) - (void)mvaddch(LINES - 3 - j, i, '#'); + (void)mvaddch((LINES - 3) - j, i, '#'); for (; j < (bm[i] + bh[i]) / scale; ++j) - (void)mvaddch(LINES - 3 - j, i, '|'); + (void)mvaddch((LINES - 3) - j, i, '|'); } refresh(); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index f4307b6..8f917d4 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -2704,9 +2704,9 @@ b64_settings(const struct http *hp, const char *s) if (*s >= 'A' && *s <= 'Z') v |= (uint64_t)(*s - 'A') << shift; else if (*s >= 'a' && *s <= 'z') - v |= (uint64_t)(*s - 'a' + 26) << shift; + v |= (uint64_t)((*s - 'a') + 26) << shift; else if (*s >= '0' && *s <= '9') - v |= (uint64_t)(*s - '0' + 52) << shift; + v |= (uint64_t)((*s - '0') + 52) << shift; else if (*s == '-') v |= (uint64_t)62 << shift; else if (*s == '_') diff --git a/flint.lnt b/flint.lnt index e49efe5..a585940 100644 --- a/flint.lnt +++ b/flint.lnt @@ -215,5 +215,7 @@ -e459 // unlocked access from func-ptr -e679 // Suspicious Truncation in arithmetic expression combining with pointer -e712 // Loss of precision (___) (___ to ___) +-e713 // Loss of precision (___) (___ to ___) -e732 // Loss of sign (___) (___ to ___) +-e734 // Loss of precision (___) (___ bits to ___ bits) -e747 // Significant prototype coercion (___) ___ to ___ diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index c5cf838..36d7377 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -176,7 +176,7 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg) sz = strtoul(vg->av[3], NULL, 10); assert(sz > 0); assert(of >= off); - len = RUP2(of - off + sz, ps); + len = RUP2((of - off) + sz, ps); vsb = VSB_new_auto(); AN(vsb); From geoff at uplex.de Fri Mar 9 10:27:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 9 Mar 2018 10:27:07 +0000 (UTC) Subject: [master] 6ebe080 Edit "Upgrading" again to emphasize that UDS+PROXY is a good idea. Message-ID: <20180309102707.E552FB11B7@lists.varnish-cache.org> commit 6ebe080e2b36f4d4339c14d66739f69c90b5aea6 Author: Geoff Simmons Date: Fri Mar 9 11:25:44 2018 +0100 Edit "Upgrading" again to emphasize that UDS+PROXY is a good idea. diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index f40c47d..3931993 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -86,6 +86,11 @@ IPv4" address with port 0. So your use of IP-valued elements in VCL will continue to work and may not have to change, but there are some consequences that you should consider, covered in the following. +As we shall see, for a variety of reasons you get the best results if +the component forwarding to Varnish via UDS uses the PROXY protocol, +which sets ``client.ip`` and ``server.ip`` to the addresses sent in +the PROXY header. + If you don't use UDSen, then nothing about VCL changes. UDS support requires version 4.1, so if you are keeping your VCL level at 4.0 (and hence are staying with IP addresses), then none of the following is of @@ -95,14 +100,10 @@ concern. ------------------------------------------------------------ These variables have the value ``0.0.0.0`` for a connection that was -addressed as a UDS. - -Remember that if you are using the PROXY protocol, then ``client.ip`` -and ``server.ip`` are set to the addresses sent in the PROXY header by -the forwarding component. So these two variables may have "real" IP -address values, even when the Varnish listener address is a UDS, if -PROXY was specified. ``local.ip`` and ``remote.ip``, however, are -always ``0.0.0.0`` for a UDS listener. +addressed as a UDS. If you are using the PROXY protocol, then +``client.ip`` and ``server.ip`` have the "real" IP address values sent +via PROXY, but ``local.ip`` and ``remote.ip`` are always ``0.0.0.0`` +for a UDS listener. If you have more than one UDS listener (more than one ``-a`` command-line argument specifying a socket path), then you may not be From nils.goroll at uplex.de Fri Mar 9 13:01:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 13:01:07 +0000 (UTC) Subject: [master] 59fd455 shard director: test layering Message-ID: <20180309130107.E691BB3E4A@lists.varnish-cache.org> commit 59fd455f846d3b68e59dd466ea0db2b9726ec4c7 Author: Nils Goroll Date: Fri Mar 9 13:36:04 2018 +0100 shard director: test layering diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc index c29e03f..c62e77e 100644 --- a/bin/varnishtest/tests/d00021.vtc +++ b/bin/varnishtest/tests/d00021.vtc @@ -1,16 +1,16 @@ -varnishtest "shard director LAZY" +varnishtest "shard director LAZY and layering" -server s1 { +server s1 -repeat 2 { rxreq txresp -body "ech3Ooj" } -start -server s2 { +server s2 -repeat 2 { rxreq txresp -body "ieQu2qua" } -start -server s3 { +server s3 -repeat 2 { rxreq txresp -body "xiuFi3Pe" } -start @@ -33,30 +33,75 @@ varnish v1 -vcl+backend { if (!vd.reconfigure(replicas=25)) { std.log("reconfigure failed"); } + + new l = directors.shard(); + new lp = directors.shard_param(); + l.associate(lp.use()); + if (!l.add_backend(s1)) { + std.log("add s1 failed"); + } + if (!l.add_backend(s2)) { + std.log("add s2 failed"); + } + if (!l.add_backend(s3)) { + std.log("add s3 failed"); + } + if (!l.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + new ll = directors.round_robin(); + ll.add_backend(l.backend(resolve=LAZY)); } sub vcl_recv { return(pass); } - sub vcl_backend_fetch { + sub backend_fetch_direct { if (bereq.url == "/1") { set bereq.backend = vd.backend(resolve=LAZY, by=KEY, key=1); - } + } else if (bereq.url == "/2") { set bereq.backend = vd.backend(resolve=LAZY, by=KEY, key=2147483647); - } + } else if (bereq.url == "/3") { set bereq.backend = vd.backend(resolve=LAZY, by=KEY, key=4294967295); } } + sub backend_fetch_layered { + set bereq.backend = ll.backend(); + if (bereq.url == "/1") { + lp.set(by=KEY, key=1); + } else + if (bereq.url == "/2") { + lp.set(by=KEY, key=2147483647); + } else + if (bereq.url == "/3") { + lp.set(by=KEY, key=4294967295); + } + } + + sub vcl_backend_fetch { + if (bereq.http.layered) { + call backend_fetch_layered; + } else { + call backend_fetch_direct; + } + } + sub vcl_backend_response { - set beresp.http.healthy = std.healthy( - vd.backend(resolve=LAZY, by=KEY, key=1)); + if (bereq.http.layered) { + set beresp.http.healthy = std.healthy(ll.backend()); + } else { + set beresp.http.healthy = std.healthy( + vd.backend(resolve=LAZY, by=KEY, key=1)); + } + set beresp.http.director = bereq.backend; + set beresp.http.backend = beresp.backend; } } -start @@ -66,14 +111,41 @@ client c1 { rxresp expect resp.body == "ech3Ooj" expect resp.http.healthy == "true" + expect resp.http.director == "vd" + expect resp.http.backend == "s1" txreq -url /2 rxresp expect resp.body == "ieQu2qua" expect resp.http.healthy == "true" + expect resp.http.director == "vd" + expect resp.http.backend == "s2" txreq -url /3 rxresp expect resp.body == "xiuFi3Pe" expect resp.http.healthy == "true" + expect resp.http.director == "vd" + expect resp.http.backend == "s3" + + txreq -url /1 -hdr "layered: true" + rxresp + expect resp.body == "ech3Ooj" + expect resp.http.healthy == "true" + expect resp.http.director == "ll" + expect resp.http.backend == "s1" + + txreq -url /2 -hdr "layered: true" + rxresp + expect resp.body == "ieQu2qua" + expect resp.http.healthy == "true" + expect resp.http.director == "ll" + expect resp.http.backend == "s2" + + txreq -url /3 -hdr "layered: true" + rxresp + expect resp.body == "xiuFi3Pe" + expect resp.http.healthy == "true" + expect resp.http.director == "ll" + expect resp.http.backend == "s3" } -run From daghf at varnish-software.com Fri Mar 9 14:38:08 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 9 Mar 2018 14:38:08 +0000 (UTC) Subject: [master] b12b829 Avoid leaving the h2sess hanging on return (fail) Message-ID: <20180309143808.CA584B5A1F@lists.varnish-cache.org> commit b12b8294cd431c9f82b54bafac5614d76e9ce744 Author: Dag Haavi Finstad Date: Fri Mar 9 15:35:01 2018 +0100 Avoid leaving the h2sess hanging on return (fail) If we fail a stream early from VCL via return (fail) after sending a request body, the h2 sess will be stuck waiting forever. diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 542a10e..8b61020 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -509,6 +509,7 @@ h2_do_req(struct worker *wrk, void *priv) { struct req *req; struct h2_req *r2; + struct h2_sess *h2; CAST_OBJ_NOTNULL(req, priv, REQ_MAGIC); CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); @@ -517,10 +518,16 @@ h2_do_req(struct worker *wrk, void *priv) req->http->conds = 1; if (CNT_Request(wrk, req) != REQ_FSM_DISEMBARK) { AZ(req->ws->r); + h2 = r2->h2sess; + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + Lck_Lock(&h2->sess->mtx); r2->scheduled = 0; r2->state = H2_S_CLOSED; - if (r2->h2sess->error) - AZ(pthread_cond_signal(r2->h2sess->cond)); + if (h2->mailcall == r2) { + h2->mailcall = NULL; + AZ(pthread_cond_signal(h2->cond)); + } + Lck_Unlock(&h2->sess->mtx); } THR_SetRequest(NULL); } diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index c80432b..2bcd1da 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -2,15 +2,30 @@ varnishtest "H2 POST" barrier b1 cond 2 +barrier b2 sock 2 +barrier b3 sock 2 + server s1 { rxreq expect req.http.content-length == 7 expect req.http.transfer-encoding == barrier b1 sync txresp -hdr "Content-Type: text/plain" -body response + + rxreq + txresp } -start -varnish v1 -vcl+backend {} -cliok "param.set feature +http2" -start +varnish v1 -vcl+backend { + import vtc; + sub vcl_recv { + if (req.url == "/a") { + vtc.barrier_sync("${b2_sock}"); + vtc.barrier_sync("${b3_sock}"); + return (fail); + } + } +} -cliok "param.set feature +http2" -start varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { @@ -41,5 +56,27 @@ client c1 { stream 0 -wait } -run +client c1 { + stream 0 { + barrier b2 sync + delay 1 + barrier b3 sync + rxwinup + } -start + stream 1 { + txreq -url "/a" -req POST -nostrend + txdata -datalen 100 + rxresp + rxwinup + expect resp.status == 503 + } -run + stream 3 { + txreq -url "/b" + rxresp + expect resp.status == 200 + } -run + stream 0 -wait +} -run + logexpect l1 -wait From nils.goroll at uplex.de Fri Mar 9 15:42:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 15:42:09 +0000 (UTC) Subject: [master] e1f2e3d simplify vmod_shard_backend() LAZY handling Message-ID: <20180309154209.2D905B6BDA@lists.varnish-cache.org> commit e1f2e3d6377ec61922d90f197b2723ea28228c3f Author: Nils Goroll Date: Fri Mar 9 14:13:21 2018 +0100 simplify vmod_shard_backend() LAZY handling diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 1ecd9f3..b672249 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -606,27 +606,20 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, switch (resolve) { case LAZY: - if ((ctx->method & VCL_MET_TASK_B) == 0) { - if ((args & ~arg_resolve) != 0) { - VRT_fail(ctx, - "shard .backend resolve=LAZY " - "with other parameters can " - "only be used in backend " - "context"); - return (NULL); - } + if ((args & ~arg_resolve) == 0) { AN(vshard->dir); return (vshard->dir); } - assert(ctx->method & VCL_MET_TASK_B); - - if ((args & ~arg_resolve) == 0) { - /* no other parameters - shortcut */ - AN(vshard->dir); - return (vshard->dir); + if ((ctx->method & VCL_MET_TASK_B) == 0) { + VRT_fail(ctx, "shard .backend resolve=LAZY with other " + "parameters can only be used in backend " + "context"); + return (NULL); } + assert(ctx->method & VCL_MET_TASK_B); + pp = shard_param_task(ctx, vshard, vshard->param); if (pp == NULL) return (NULL); From nils.goroll at uplex.de Fri Mar 9 15:42:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 15:42:09 +0000 (UTC) Subject: [master] 1514de5 shard director: change default to resolve=LAZY in vcl_init{} Message-ID: <20180309154209.46CFFB6BDD@lists.varnish-cache.org> commit 1514de54fac052c59d3f65b639394aad5eb66cbc Author: Nils Goroll Date: Fri Mar 9 16:41:09 2018 +0100 shard director: change default to resolve=LAZY in vcl_init{} diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc index c62e77e..9633561 100644 --- a/bin/varnishtest/tests/d00021.vtc +++ b/bin/varnishtest/tests/d00021.vtc @@ -50,7 +50,7 @@ varnish v1 -vcl+backend { std.log("reconfigure failed"); } new ll = directors.round_robin(); - ll.add_backend(l.backend(resolve=LAZY)); + ll.add_backend(l.backend()); } sub vcl_recv { diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc index 202c485..497b3c7 100644 --- a/bin/varnishtest/tests/d00030.vtc +++ b/bin/varnishtest/tests/d00030.vtc @@ -181,6 +181,6 @@ varnish v1 -errvcl {resolve=NOW can not be used in vcl_init} { sub vcl_init { new shard = directors.shard(); new rr = directors.round_robin(); - rr.add_backend(shard.backend()); + rr.add_backend(shard.backend(resolve=NOW)); } } diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index c2106a3..d240119 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -535,6 +535,8 @@ is _not_ the order given when backends are added. * `resolve` + default: `LAZY` in ``vcl_init{}``, `NOW` otherwise + * ``NOW``: look up a backend and return it. Can not be used in ``vcl_init{}``. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index b672249..1a0504b 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -602,7 +602,13 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); assert((args & ~_arg_mask) == 0); - resolve = (args & arg_resolve) ? parse_resolve_e(a->resolve) : NOW; + + if (args & arg_resolve) + resolve = parse_resolve_e(a->resolve); + else if (ctx->method & VCL_MET_TASK_H) + resolve = LAZY; + else + resolve = NOW; switch (resolve) { case LAZY: From daghf at varnish-software.com Fri Mar 9 15:50:07 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 9 Mar 2018 15:50:07 +0000 (UTC) Subject: [master] 744a5e0 Make h2sess deletion more explicit Message-ID: <20180309155007.D8402B6E77@lists.varnish-cache.org> commit 744a5e0f52b90aa0142151fe1b86367e797d9c15 Author: Dag Haavi Finstad Date: Fri Mar 9 16:47:15 2018 +0100 Make h2sess deletion more explicit Also add a few extra asserts, in an effort to chase down #2591 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 8b61020..e1f02eb 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -165,8 +165,6 @@ h2_del_req(struct worker *wrk, const struct h2_req *r2) { struct h2_sess *h2; struct sess *sp; - struct req *req; - int r; CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); AZ(r2->scheduled); @@ -176,22 +174,13 @@ h2_del_req(struct worker *wrk, const struct h2_req *r2) sp = h2->sess; Lck_Lock(&sp->mtx); assert(h2->refcnt > 0); - r = --h2->refcnt; + --h2->refcnt; /* XXX: PRIORITY reshuffle */ VTAILQ_REMOVE(&h2->streams, r2, list); Lck_Unlock(&sp->mtx); AZ(r2->req->ws->r); Req_Cleanup(sp, wrk, r2->req); Req_Release(r2->req); - if (r) - return; - /* All streams gone, including stream #0, clean up */ - VHT_Fini(h2->dectbl); - req = h2->srq; - AZ(req->ws->r); - Req_Cleanup(sp, wrk, req); - Req_Release(req); - SES_Delete(sp, SC_RX_JUNK, NAN); } void diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index f2d83b4..917f880 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -139,6 +139,25 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq) return (h2); } +static void +h2_del_sess(struct worker *wrk, struct h2_sess *h2, enum sess_close reason) +{ + struct sess *sp; + struct req *req; + + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + AZ(h2->refcnt); + assert(VTAILQ_EMPTY(&h2->streams)); + + VHT_Fini(h2->dectbl); + req = h2->srq; + AZ(req->ws->r); + sp = h2->sess; + Req_Cleanup(sp, wrk, req); + Req_Release(req); + SES_Delete(sp, reason, NAN); +} + /**********************************************************************/ enum htc_status_e v_matchproto_(htc_complete_f) @@ -320,7 +339,9 @@ h2_new_session(struct worker *wrk, void *arg) h2->req0 = h2_new_req(wrk, h2, 0, NULL); if (req->err_code == H2_OU_MARKER && !h2_ou_session(wrk, h2, req)) { + assert(h2->refcnt == 1); h2_del_req(wrk, h2->req0); + h2_del_sess(wrk, h2, SC_RX_JUNK); return; } assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc)); @@ -387,7 +408,10 @@ h2_new_session(struct worker *wrk, void *arg) Lck_Unlock(&h2->sess->mtx); } h2->cond = NULL; + assert(h2->refcnt == 1); h2_del_req(wrk, h2->req0); + /* TODO: proper sess close reason */ + h2_del_sess(wrk, h2, SC_RX_JUNK); } static void v_matchproto_(vtr_reembark_f) From nils.goroll at uplex.de Fri Mar 9 16:18:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 16:18:07 +0000 (UTC) Subject: [master] ee25c1b fix argstruct for PRIV_* Message-ID: <20180309161807.953996167A@lists.varnish-cache.org> commit ee25c1b559f505817cd754204bc6ed43040f2789 Author: Nils Goroll Date: Fri Mar 9 17:06:16 2018 +0100 fix argstruct for PRIV_* PRIV_* arguments are unnamed, so vmodtool names their argstruct members arg%d diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 90cc00c..f67b141 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -468,6 +468,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, const char *sa; char ssa[64]; char ssa2[64]; + int n; CAST_OBJ_NOTNULL(vv, priv, VJSN_VAL_MAGIC); assert(vv->type == VJSN_ARRAY); @@ -574,7 +575,9 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, e1 = vcc_mk_expr(rfmt, "%s(ctx%s,\v+(\n", cfunc, extra); else e1 = vcc_mk_expr(rfmt, "%s(ctx%s\v+", cfunc, extra); + n = 0; VTAILQ_FOREACH_SAFE(fa, &head, list, fa2) { + n++; if (fa->optional) VSB_printf(tl->curproc->prologue, " %s.valid_%s = %d;\n", sa, fa->name, fa->avail); @@ -583,7 +586,12 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, if (fa->result == NULL && fa->val != NULL) fa->result = vcc_mk_expr(fa->type, "%s", fa->val); if (fa->result != NULL && sa != NULL) { - bprintf(ssa2, "\v1%s.%s = \v2,\n", sa, fa->name); + if (fa->name && *fa->name != '\0') + bprintf(ssa2, "\v1%s.%s = \v2,\n", + sa, fa->name); + else + bprintf(ssa2, "\v1%s.arg%d = \v2,\n", + sa, n); e1 = vcc_expr_edit(tl, e1->fmt, ssa2, e1, fa->result); } else if (fa->result != NULL) { e1 = vcc_expr_edit(tl, e1->fmt, "\v1,\n\v2", From nils.goroll at uplex.de Fri Mar 9 16:18:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 16:18:07 +0000 (UTC) Subject: [master] ce59124 shard: replace magic defaults with optional arguments Message-ID: <20180309161807.AC2676167D@lists.varnish-cache.org> commit ce59124fc596f1281d814aff9aca1572ed038bbf Author: Nils Goroll Date: Fri Mar 9 17:07:59 2018 +0100 shard: replace magic defaults with optional arguments diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index ccd64c2..4674a40 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -656,8 +656,7 @@ shardcfg_get_rampup(const struct sharddir *shardd, int host) // assert sharddir_rdlock_held(shardd); assert (host < shardd->n_backend); - // magic value for default - if (shardd->backend[host].rampup == 973279260) + if (isnan(shardd->backend[host].rampup)) r = shardd->rampup_duration; else r = shardd->backend[host].rampup; diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index d240119..586c9db 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -368,7 +368,7 @@ The association can be changed per backend request using the `param` argument of `func_shard.backend`_. $Method BOOL .add_backend(PRIV_TASK, BACKEND backend, - STRING ident=0, DURATION rampup=973279260) + [STRING ident], [DURATION rampup]) Add a backend `backend` to the director. @@ -380,13 +380,13 @@ backend name. `ident` allows to add multiple instances of the same backend. `rampup`: Optionally specify a specific rampup time for this -backend. The magic default value of `973279260s` instructs the shard -director to use the default rampup time (see :ref:`func_shard.set_rampup`). +backend. Otherwise, the per-director rampup time is used (see +:ref:`func_shard.set_rampup`). NOTE: Backend changes need to be finalized with `shard.reconfigure()` and are only supported on one shard director at a time. -$Method BOOL .remove_backend(PRIV_TASK, BACKEND backend=0, STRING ident=0) +$Method BOOL .remove_backend(PRIV_TASK, [BACKEND backend=0], [STRING ident=0]) Remove backend(s) from the director. Either `backend` or `ident` must be specified. `ident` removes a specific instance. If `backend` is diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 1a0504b..21a73eb 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -297,26 +297,29 @@ vmod_shard_associate(VRT_CTX, VCL_BOOL v_matchproto_(td_directors_shard_add_backend) vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_priv *priv, - VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) + struct vmod_shard_add_backend_arg *args) { CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); - if (be == NULL) { + if (args->backend == NULL) { shard_err0(ctx, vshard->shardd, ".backend_add() NULL backend given"); - return 0; + return (0); } - return shardcfg_add_backend(ctx, priv, vshard->shardd, - be, ident, rampup); + return shardcfg_add_backend(ctx, args->arg1, + vshard->shardd, args->backend, + args->valid_ident ? args->ident : NULL, + args->valid_rampup ? args->rampup : nan("")); } VCL_BOOL v_matchproto_(td_directors_shard_remove_backend) vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_priv *priv, - VCL_BACKEND be, VCL_STRING ident) + struct vmod_shard_remove_backend_arg *args) { + VCL_BACKEND be = args->valid_backend ? args->backend : NULL; + VCL_STRING ident = args->ident ? args->ident : NULL; + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (be == NULL && ident == NULL) { @@ -326,7 +329,7 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, return 0; } - return shardcfg_remove_backend(ctx, priv, vshard->shardd, + return shardcfg_remove_backend(ctx, args->arg1, vshard->shardd, be, ident); } From nils.goroll at uplex.de Fri Mar 9 17:54:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Mar 2018 17:54:07 +0000 (UTC) Subject: [master] c277f10 fix the vsc_new / vsc_destroy Message-ID: <20180309175407.C3C1F631EE@lists.varnish-cache.org> commit c277f10804680069dad5813a20f54d3271e6d17a Author: Nils Goroll Date: Fri Mar 9 18:48:24 2018 +0100 fix the vsc_new / vsc_destroy 73ba50b89ad9d33f62453667c8e8e93074f0f5cf was incomplete and I confused vsc_seg and vsc staring at this trivial code Ref: #2576 diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 0534611..3ee5a0e 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -57,8 +57,8 @@ struct priv_vcl { static VCL_DURATION vcl_release_delay = 0.0; static pthread_mutex_t vsc_mtx = PTHREAD_MUTEX_INITIALIZER; -static struct vsc_seg *vsc_seg; -static struct VSC_debug *vsc; +static struct vsc_seg *vsc_seg = NULL; +static struct VSC_debug *vsc = NULL; VCL_STRING v_matchproto_(td_debug_author) xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) @@ -364,9 +364,12 @@ xyzzy_vsc_new(VRT_CTX) { (void)ctx; AZ(pthread_mutex_lock(&vsc_mtx)); - if (vsc == NULL) + if (vsc == NULL) { + AZ(vsc_seg); vsc = VSC_debug_New(NULL, &vsc_seg, ""); + } AN(vsc); + AN(vsc_seg); AZ(pthread_mutex_unlock(&vsc_mtx)); } @@ -375,8 +378,11 @@ xyzzy_vsc_destroy(VRT_CTX) { (void)ctx; AZ(pthread_mutex_lock(&vsc_mtx)); - if (vsc != NULL) + if (vsc != NULL) { + AN(vsc_seg); VSC_debug_Destroy(&vsc_seg); - AZ(vsc); + } + AZ(vsc_seg); + vsc = NULL; AZ(pthread_mutex_unlock(&vsc_mtx)); } From phk at FreeBSD.org Sat Mar 10 22:21:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Mar 2018 22:21:09 +0000 (UTC) Subject: [master] 1a39084 Remove unnecessary #include Message-ID: <20180310222109.532B6B1FD9@lists.varnish-cache.org> commit 1a39084919f87172e9b02eae3f0af9c271243cd4 Author: Poul-Henning Kamp Date: Fri Mar 9 10:17:56 2018 +0000 Remove unnecessary #include diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 3407c70..db42d31 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -63,7 +63,6 @@ #include "vapi/vsm.h" #include "vas.h" #include "vcli.h" -#include "vnum.h" #include "vtcp.h" #define RL_EXIT(status) \ From geoff at uplex.de Sun Mar 11 09:25:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Sun, 11 Mar 2018 09:25:10 +0000 (UTC) Subject: [master] 496fa02 Document sess.xid in "Upgrading to 6.0". Message-ID: <20180311092510.B1D7A925A5@lists.varnish-cache.org> commit 496fa02188cd63babf390377e6e3ef22f627a8c6 Author: Geoff Simmons Date: Sun Mar 11 10:22:34 2018 +0100 Document sess.xid in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 3931993..dd8faf1 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -73,6 +73,24 @@ Changes to VCL XXX: ... intro paragraph +VCL variables +~~~~~~~~~~~~~ + +``sess.xid`` +------------ + +This is the unique ID assigned by Varnish to the current session, +which stands for the "conversation" with a single client connection +that comprises one or more request/response transactions. It is the +same XID shown in the log for session transactions (with +``-g session`` grouping). ``sess.xid`` is read-only and is available +as of VCL 4.1. + +XXX: VCL vars subhead 2 +----------------------- + +XXX: ... + Unix domain sockets and VCL ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -262,19 +280,6 @@ to a ``*.ip`` variable whose value is set to ``0.0.0.0`` because the listener is UDS. :ref:`func_set_ip_tos` is silently ignored when the listener is UDS. -VCL variables -~~~~~~~~~~~~~ - -XXX: VCL vars subhead 1 ------------------------ - -XXX: ... - -XXX: VCL vars subhead 2 ------------------------ - -XXX: ... - XXX VCL subhead 2 ~~~~~~~~~~~~~~~~~ From geoff at uplex.de Sun Mar 11 10:06:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Sun, 11 Mar 2018 10:06:07 +0000 (UTC) Subject: [master] d86d862 Minor "Upgrading" edit for legibility. Message-ID: <20180311100607.A0D50931F8@lists.varnish-cache.org> commit d86d862a6d208979bc1627cc1089d28489cd7e2e Author: Geoff Simmons Date: Sun Mar 11 10:58:00 2018 +0100 Minor "Upgrading" edit for legibility. diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index dd8faf1..978e62e 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -275,10 +275,10 @@ PROXY protocol. VMOD std -------- -The VMOD std function :ref:`func_port` always returns 0 when applied -to a ``*.ip`` variable whose value is set to ``0.0.0.0`` because the -listener is UDS. :ref:`func_set_ip_tos` is silently ignored when the -listener is UDS. +:ref:`std.port(IP) ` always returns 0 when applied to a +``*.ip`` variable whose value is set to ``0.0.0.0`` because the +listener is UDS. :ref:`std.set_ip_tos(INT) ` is +silently ignored when the listener is UDS. XXX VCL subhead 2 ~~~~~~~~~~~~~~~~~ From geoff at uplex.de Sun Mar 11 10:06:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Sun, 11 Mar 2018 10:06:07 +0000 (UTC) Subject: [master] cd031cb Document the use of "0.0.0.0:0" in the logs in "Upgrading to 6.0". Message-ID: <20180311100607.B66B6931FB@lists.varnish-cache.org> commit cd031cb6ee404274ab3576a6bc5d17b4b422f8a7 Author: Geoff Simmons Date: Sun Mar 11 11:04:53 2018 +0100 Document the use of "0.0.0.0:0" in the logs in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 978e62e..8c354a4 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -294,7 +294,12 @@ Other changes name of the listener address over which the request was received, see :ref:`vsl(7)`. - * XXX ... + * ``0.0.0.0`` and port ``0`` appear in the logs where an IP and port + otherwise appear, when the connection in question was addressed as + a Unix domain socket. This affects ``ReqStart`` and ``SessOpen``. + If you have more than one UDS listener, they can be distinguished + with the "listener name" field -- the third field for both + ``ReqStart`` and ``SessOpen``. * ``varnishtest(1)``: From phk at FreeBSD.org Mon Mar 12 08:34:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 08:34:07 +0000 (UTC) Subject: [master] 3f24251 Make $ lines terminate on the first '\n[^ \t]'. Message-ID: <20180312083407.13EC7B2606@lists.varnish-cache.org> commit 3f242510215f4b1cc896308903720f9a98270f28 Author: Poul-Henning Kamp Date: Mon Mar 12 08:32:27 2018 +0000 Make $ lines terminate on the first '\n[^ \t]'. Add arg names to synopsis diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e866c80..e1770bb 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -422,9 +422,9 @@ class prototype(object): t = i.vcl() if t in privs: continue + if i.nm is not None: + t += " " + i.nm if not short: - if i.nm is not None: - t += " " + i.nm if i.defval is not None: t += "=" + i.defval if i.opt: @@ -843,18 +843,13 @@ class vcc(object): s = a.split("\n$") self.copyright = s.pop(0).strip() while len(s): - ss = s.pop(0) - i = ss.find("\n\n") - if i > -1: - i += 1 - else: - i = len(ss) - inputline = ss[:i] - c = ss[:i].split() + ss = re.split('\n([^\t ])', s.pop(0), maxsplit=1) + c = ss[0].split() + d = "".join(ss[1:]) m = dispatch.get(c[0]) if m is None: err("Unknown stanze $%s" % ss[:i]) - m([c[0], " ".join(c[1:])], ss[i:].split('\n'), self) + m([c[0], " ".join(c[1:])], d.split('\n'), self) inputline = None def rst_copyright(self, fo): diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index c27e866..5e1ac0c 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -28,7 +28,6 @@ $Module debug 3 Development, test and debug $ABI strict $Prefix xyzzy - DESCRIPTION =========== @@ -118,8 +117,7 @@ Encrypt the HTTP header with quad-ROT13 encryption, $Function STRING argtest( STRING one, REAL two =2, STRING three= "3", STRING comma=",", INT four = 4, - [ STRING opt] -) + [ STRING opt]) $Function INT vre_limit() From geoff at uplex.de Mon Mar 12 09:45:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:45:10 +0000 (UTC) Subject: [master] 3eb59f1 Add PFD_LocalName() and PFD_RemoteName() to the FD pool interface. Message-ID: <20180312094510.1F0C7B3BF0@lists.varnish-cache.org> commit 3eb59f10b7d3380c65c7d215e6b11c8b059d98eb Author: Geoff Simmons Date: Thu Mar 8 16:18:12 2018 +0100 Add PFD_LocalName() and PFD_RemoteName() to the FD pool interface. This makes logging the local and peer address in BackendOpen independent of the socket type. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 9c0231b..11a35c9 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -136,8 +136,8 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (bp->proxy_header != 0) VPX_Send_Proxy(*fdp, bp->proxy_header, bo->sp); - VTCP_myname(*fdp, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - VTCP_hisname(*fdp, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); + PFD_LocalName(pfd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); + PFD_RemoteName(pfd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", *fdp, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 73b7f4f..c63ad64 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -62,31 +62,20 @@ struct pfd { pthread_cond_t *cond; }; -unsigned -PFD_State(const struct pfd *p) -{ - CHECK_OBJ_NOTNULL(p, PFD_MAGIC); - return (p->state); -} - -int * -PFD_Fd(struct pfd *p) -{ - CHECK_OBJ_NOTNULL(p, PFD_MAGIC); - return (&(p->fd)); -} - /*-------------------------------------------------------------------- */ typedef int cp_open_f(const struct conn_pool *, double tmo, const void **privp); typedef void cp_close_f(struct pfd *); typedef int cp_cmp_f(const struct conn_pool *, const void *priv); +typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); struct cp_methods { cp_open_f *open; cp_close_f *close; cp_cmp_f *cmp; + cp_name_f *local_name; + cp_name_f *remote_name; }; struct conn_pool { @@ -125,6 +114,41 @@ static VTAILQ_HEAD(, conn_pool) conn_pools = VTAILQ_HEAD_INITIALIZER(conn_pools); /*-------------------------------------------------------------------- + */ + +unsigned +PFD_State(const struct pfd *p) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + return (p->state); +} + +int * +PFD_Fd(struct pfd *p) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + return (&(p->fd)); +} + +void +PFD_LocalName(const struct pfd *p, char *abuf, unsigned alen, char *pbuf, + unsigned plen) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + CHECK_OBJ_NOTNULL(p->conn_pool, CONN_POOL_MAGIC); + p->conn_pool->methods->local_name(p, abuf, alen, pbuf, plen); +} + +void +PFD_RemoteName(const struct pfd *p, char *abuf, unsigned alen, char *pbuf, + unsigned plen) +{ + CHECK_OBJ_NOTNULL(p, PFD_MAGIC); + CHECK_OBJ_NOTNULL(p->conn_pool, CONN_POOL_MAGIC); + p->conn_pool->methods->remote_name(p, abuf, alen, pbuf, plen); +} + +/*-------------------------------------------------------------------- * Waiter-handler */ @@ -533,10 +557,28 @@ vtp_cmp(const struct conn_pool *cp, const void *priv) return (0); } +static void v_matchproto_(cp_name_f) +vtp_local_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf, + unsigned plen) +{ + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + VTCP_myname(pfd->fd, addr, alen, pbuf, plen); +} + +static void v_matchproto_(cp_name_f) +vtp_remote_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf, + unsigned plen) +{ + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + VTCP_hisname(pfd->fd, addr, alen, pbuf, plen); +} + static const struct cp_methods vtp_methods = { .open = vtp_open, .close = vtp_close, .cmp = vtp_cmp, + .local_name = vtp_local_name, + .remote_name = vtp_remote_name, }; diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index f5e25c5..6c792c1 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -42,6 +42,8 @@ struct pfd; unsigned PFD_State(const struct pfd *); int *PFD_Fd(struct pfd *); +void PFD_LocalName(const struct pfd *, char *, unsigned, char *, unsigned); +void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); /*--------------------------------------------------------------------- From geoff at uplex.de Mon Mar 12 09:45:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:45:10 +0000 (UTC) Subject: [master] 5c0b833 Support Unix domain sockets as addresses in backend definitions. Message-ID: <20180312094510.723B1B3BF7@lists.varnish-cache.org> commit 5c0b833a0170f0f91631f852abdf6c7bf2aa7cbd Author: Geoff Simmons Date: Mon Feb 19 02:39:53 2018 +0100 Support Unix domain sockets as addresses in backend definitions. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 11a35c9..74b29d6 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -212,6 +212,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, int i, extrachance = 1; struct backend *bp; struct pfd *pfd; + char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -234,8 +235,9 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, if (PFD_State(pfd) != PFD_STATE_STOLEN) extrachance = 0; + PFD_RemoteName(pfd, abuf, sizeof abuf, pbuf, sizeof pbuf); i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, - &bo->acct.bereq_bodybytes, 0); + &bo->acct.bereq_bodybytes, 0, abuf, pbuf); if (PFD_State(pfd) != PFD_STATE_USED) { if (VTP_Wait(wrk, pfd, VTIM_real() + @@ -300,6 +302,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) struct backend *bp; struct v1p_acct v1a; struct pfd *pfd; + char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -320,7 +323,9 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) retval = SC_TX_ERROR; } else { CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1); + PFD_RemoteName(pfd, abuf, sizeof abuf, pbuf, sizeof pbuf); + i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1, abuf, + pbuf); VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk)); if (i == 0) V1P_Process(req, *PFD_Fd(pfd), &v1a); @@ -437,7 +442,12 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); - assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL); + if (vrt->path == NULL) + assert(vrt->ipv4_suckaddr != NULL + || vrt->ipv6_suckaddr != NULL); + else + assert(vrt->ipv4_suckaddr == NULL + && vrt->ipv6_suckaddr == NULL); vcl = ctx->vcl; AN(vcl); @@ -480,7 +490,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, VTAILQ_INSERT_TAIL(&backends, be, list); VSC_C_main->n_backend++; be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, - vbe_proto_ident); + vrt->path, vbe_proto_ident); Lck_Unlock(&backends_mtx); if (vbp != NULL) { diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index b5522dc..478ad18 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -283,7 +283,9 @@ vbp_poke(struct vbp_target *vt) } i = VSA_Get_Proto(sa); - if (i == AF_INET) + if (VSA_Compare(sa, bogo_ip) == 0) + vt->good_unix |= 1; + else if (i == AF_INET) vt->good_ipv4 |= 1; else if (i == AF_INET6) vt->good_ipv6 |= 1; diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index c63ad64..e9fa812 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -38,6 +38,7 @@ #include "vsa.h" #include "vtcp.h" +#include "vus.h" #include "vtim.h" #include "waiter/waiter.h" @@ -106,6 +107,7 @@ struct tcp_pool { struct suckaddr *ip4; struct suckaddr *ip6; + char *uds; struct conn_pool cp[1]; }; @@ -489,6 +491,20 @@ VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo) /*-------------------------------------------------------------------- */ +struct vtp_cs { + unsigned magic; +#define VTP_CS_MAGIC 0xc1e40447 + const struct suckaddr *ip4; + const struct suckaddr *ip6; + const char *uds; +}; + +static inline int +tmo2msec(double tmo) +{ + return ( (int)floor(tmo * 1000.0) ); +} + static int v_matchproto_(cp_open_f) vtp_open(const struct conn_pool *cp, double tmo, const void **privp) { @@ -499,7 +515,7 @@ vtp_open(const struct conn_pool *cp, double tmo, const void **privp) CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); - msec = (int)floor(tmo * 1000.0); + msec = tmo2msec(tmo); if (cache_param->prefer_ipv6) { *privp = tp->ip6; s = VTCP_connect(tp->ip6, msec); @@ -525,13 +541,6 @@ vtp_close(struct pfd *pfd) VTCP_close(&pfd->fd); } -struct vtp_cs { - unsigned magic; -#define VTP_CS_MAGIC 0xc1e40447 - const struct suckaddr *ip4; - const struct suckaddr *ip6; -}; - static int v_matchproto_(cp_cmp_f) vtp_cmp(const struct conn_pool *cp, const void *priv) { @@ -581,23 +590,78 @@ static const struct cp_methods vtp_methods = { .remote_name = vtp_remote_name, }; +/*-------------------------------------------------------------------- + */ + +static int v_matchproto_(cp_open_f) +vus_open(const struct conn_pool *cp, double tmo, const void **privp) +{ + int s; + int msec; + struct tcp_pool *tp; + + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); + AN(tp->uds); + + msec = tmo2msec(tmo); + *privp = bogo_ip; + s = VUS_connect(tp->uds, msec); + return (s); +} + +static int v_matchproto_(cp_cmp_f) +vus_cmp(const struct conn_pool *cp, const void *priv) +{ + const struct vtp_cs *vcs; + const struct tcp_pool *tp; + + CAST_OBJ_NOTNULL(vcs, priv, VTP_CS_MAGIC); + CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); + if (tp->uds != NULL && vcs->uds != NULL) + return (strcmp(tp->uds, vcs->uds)); + return (1); +} + +static void v_matchproto_(cp_name_f) +vus_name(const struct pfd *pfd, char *addr, unsigned alen, char *pbuf, + unsigned plen) +{ + (void) pfd; + assert(alen > strlen("0.0.0.0")); + assert(plen > 1); + strcpy(addr, "0.0.0.0"); + strcpy(pbuf, "0"); +} + +static const struct cp_methods vus_methods = { + .open = vus_open, + .close = vtp_close, + .cmp = vus_cmp, + .local_name = vus_name, + .remote_name = vus_name, +}; /*-------------------------------------------------------------------- - * Reference a TCP pool given by {ip4, ip6} pair. Create if it - * doesn't exist already. + * Reference a TCP pool given by {ip4, ip6} pair or a UDS. Create if + * it doesn't exist already. */ struct tcp_pool * -VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) +VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, + const void *id) { struct tcp_pool *tp; struct conn_pool *cp; struct vtp_cs vcs; + const struct cp_methods *methods; - assert(ip4 != NULL || ip6 != NULL); + assert((uds != NULL && ip4 == NULL && ip6 == NULL) + || (uds == NULL && (ip4 != NULL || ip6 != NULL))); INIT_OBJ(&vcs, VTP_CS_MAGIC); vcs.ip4 = ip4; vcs.ip6 = ip6; + vcs.uds = uds; cp = VCP_Ref(id, &vcs); if (cp != NULL) @@ -605,11 +669,18 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) ALLOC_OBJ(tp, TCP_POOL_MAGIC); AN(tp); - if (ip4 != NULL) - tp->ip4 = VSA_Clone(ip4); - if (ip6 != NULL) - tp->ip6 = VSA_Clone(ip6); - return(VCP_New(tp->cp, id, tp, &vtp_methods)); + if (uds != NULL) { + methods = &vus_methods; + tp->uds = strdup(uds); + } + else { + methods = &vtp_methods; + if (ip4 != NULL) + tp->ip4 = VSA_Clone(ip4); + if (ip6 != NULL) + tp->ip6 = VSA_Clone(ip6); + } + return(VCP_New(tp->cp, id, tp, methods)); } /*-------------------------------------------------------------------- @@ -638,6 +709,7 @@ VTP_Rel(struct tcp_pool **tpp) free(tp->ip4); free(tp->ip6); + free(tp->uds); FREE_OBJ(tp); } @@ -650,6 +722,8 @@ int VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) { + if (tp->uds != NULL) + return (vus_open(tp->cp, tmo, privp)); return (vtp_open(tp->cp, tmo, privp)); } diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 6c792c1..d554a36 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -51,11 +51,12 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); */ struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, - const void *id); + const char *uds, const void *id); /* - * Get a reference to a TCP pool. Either ip4 or ip6 arg must be - * non-NULL. If recycling is to be used, the id pointer distinguishes - * the pool per protocol. + * Get a reference to a TCP pool. Either one or both of ip4 or + * ip6 arg must be non-NULL, or uds must be non-NULL. If recycling + * is to be used, the id pointer distinguishes the pool per + * protocol. */ void VTP_AddRef(struct tcp_pool *); diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index f3b3ed2..7dd0c62 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -31,7 +31,7 @@ struct VSC_vbe; /* cache_http1_fetch.c [V1F] */ int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr_hdrbytes, - uint64_t *ctr_bodybytes, int onlycached); + uint64_t *ctr_bodybytes, int onlycached, char *addr, char *port); int V1F_FetchRespHdr(struct busyobj *); int V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index a3b7b4f..1a77021 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -70,7 +70,7 @@ vbf_iter_req_body(void *priv, int flush, const void *ptr, ssize_t l) int V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, - uint64_t *ctr_bodybytes, int onlycached) + uint64_t *ctr_bodybytes, int onlycached, char *abuf, char *pbuf) { struct http *hp; int j; @@ -78,8 +78,6 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, uint64_t bytes, hdrbytes; struct http_conn *htc; int do_chunked = 0; - char abuf[VTCP_ADDRBUFSIZE]; - char pbuf[VTCP_PORTBUFSIZE]; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -98,7 +96,6 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, do_chunked = 1; } - VTCP_hisname(*htc->rfd, abuf, sizeof abuf, pbuf, sizeof pbuf); VSLb(bo->vsl, SLT_BackendStart, "%s %s", abuf, pbuf); (void)VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc index 09e3d20..0acb9da 100644 --- a/bin/varnishtest/tests/b00053.vtc +++ b/bin/varnishtest/tests/b00053.vtc @@ -1,6 +1,6 @@ varnishtest "Does anything get through Unix domain sockets at all ?" -server s1 { +server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp -body "012345\n" } -start diff --git a/bin/varnishtest/tests/b00057.vtc b/bin/varnishtest/tests/b00057.vtc index 74c73cc..9f75edb 100644 --- a/bin/varnishtest/tests/b00057.vtc +++ b/bin/varnishtest/tests/b00057.vtc @@ -1,7 +1,7 @@ varnishtest "Test orderly connection closure of a UDS listen socket" -server s1 { +server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp -nolen -hdr "Transfer-encoding: chunked" delay .2 diff --git a/bin/varnishtest/tests/b00059.vtc b/bin/varnishtest/tests/b00059.vtc index 00ce94f..c529a5f 100644 --- a/bin/varnishtest/tests/b00059.vtc +++ b/bin/varnishtest/tests/b00059.vtc @@ -1,6 +1,6 @@ varnishtest "Run a lot of transactions through Unix domain sockets" -server s0 { +server s0 -listen "${tmpdir}/s1.sock" { loop 10 { rxreq txresp -body "foo1" diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc index 9e2750a..cdbb2d4 100644 --- a/bin/varnishtest/tests/b00060.vtc +++ b/bin/varnishtest/tests/b00060.vtc @@ -1,12 +1,13 @@ varnishtest "VSL tags affected by the use of UDS addresses" -varnish v1 -arg "-a foo=${tmpdir}/foo.sock -a bar=${tmpdir}/bar.sock" -vcl { - backend b { .host = "${bad_ip}"; } - - sub vcl_recv { return(synth(200)); } +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp } -start -client c1 -connect "${tmpdir}/foo.sock" { +varnish v1 -arg "-a foo=${tmpdir}/v1.sock" -vcl+backend {} -start + +client c1 -connect "${tmpdir}/v1.sock" { txreq rxresp } -run @@ -16,45 +17,12 @@ logexpect l1 -v v1 -d 1 -g session { expect 0 = SessOpen "^0.0.0.0 0 foo 0.0.0.0 0" } -run -logexpect l2 -v v1 -d 1 -g vxid { +logexpect l2 -v v1 -d 1 -g vxid -c { expect 0 1001 Begin - expect * = ReqStart "^0.0.0.0 0 foo$" -} -run - -logexpect l1 -v v1 -d 0 -g session { - expect 0 * Begin - expect 0 = SessOpen "^0.0.0.0 0 bar 0.0.0.0 0" -} -start - -logexpect l2 -v v1 -d 0 -g vxid { - expect 0 * Begin - expect * = ReqStart "^0.0.0.0 0 bar" -} -start - -client c1 -connect "${tmpdir}/bar.sock" { - txreq - rxresp + expect * = ReqStart "^0.0.0.0 0$" } -run -logexpect l1 -wait -logexpect l2 -wait - -varnish v1 -stop - -# For completeness, also test the endpoint name field in ReqStart when -# Varnish listens at an IP address. -varnish v2 -vcl { - backend b { .host = "${bad_ip}"; } - - sub vcl_recv { return(synth(200)); } -} -start - -client c2 -connect ${v2_sock} { - txreq - rxresp -} -run - -logexpect l3 -v v2 -d 1 -g vxid { - expect 0 1001 Begin - expect * = ReqStart "^${v2_addr} [0-9]+ a0$" +logexpect l2 -v v1 -d 1 -g vxid -b { + expect 0 1002 Begin + expect * = BackendOpen "${s1_sock} - - -$" } -run diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc index 5f55cdf..761ba0d 100644 --- a/bin/varnishtest/tests/c00087.vtc +++ b/bin/varnishtest/tests/c00087.vtc @@ -1,6 +1,6 @@ varnishtest "VCL *.ip vars as bogo-IPs when -a is UDS, and test ACL matches" -server s1 { +server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp } -start diff --git a/bin/varnishtest/tests/c00088.vtc b/bin/varnishtest/tests/c00088.vtc new file mode 100644 index 0000000..b39171f --- /dev/null +++ b/bin/varnishtest/tests/c00088.vtc @@ -0,0 +1,44 @@ +varnishtest "Dropping polling of a backend that listens at UDS" + +server s1 -listen "${tmpdir}/s1.sock" -repeat 40 { + rxreq + txresp +} -start + +varnish v1 -vcl { + probe default { + .window = 8; + .initial = 7; + .threshold = 8; + .interval = 0.1s; + } + backend s1 { + .path = "${s1_sock}"; + } +} -start + +delay 1 + +varnish v1 -vcl+backend { } -cliok "vcl.use vcl2" -cliok "vcl.discard vcl1" + +delay 1 + +varnish v1 -cliok "vcl.list" +varnish v1 -cliok "backend.list -p" + +server s1 -break { + rxreq + expect req.url == /foo + txresp -bodylen 4 +} -start + +delay 1 + +client c1 { + txreq -url /foo + rxresp + txreq -url /foo + rxresp + expect resp.status == 200 + expect resp.bodylen == 4 +} -run diff --git a/bin/varnishtest/tests/c00089.vtc b/bin/varnishtest/tests/c00089.vtc new file mode 100644 index 0000000..960741b --- /dev/null +++ b/bin/varnishtest/tests/c00089.vtc @@ -0,0 +1,33 @@ +varnishtest "Backend close retry with a UDS" + +server s1 -listen "${tmpdir}/s1.sock" -repeat 1 { + rxreq + txresp -bodylen 5 + + rxreq + accept + + rxreq + txresp -bodylen 6 + +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return(pass); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 5 + + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 6 +} -run + +varnish v1 -expect backend_retry == 1 diff --git a/bin/varnishtest/tests/c00090.vtc b/bin/varnishtest/tests/c00090.vtc new file mode 100644 index 0000000..189748b --- /dev/null +++ b/bin/varnishtest/tests/c00090.vtc @@ -0,0 +1,65 @@ +varnishtest "Forcing health of backends listening at UDS" + +server s1 -listen "${tmpdir}/s1.sock" -repeat 3 { + rxreq + txresp +} -start + +varnish v1 -vcl { + backend s1 { + .path = "${s1_sock}"; + .probe = { + .window = 8; + .initial = 7; + .threshold = 8; + .interval = 10s; + } + } + + sub vcl_recv { + return(pass); + } + +} -start + +delay 1 + +varnish v1 -cliok "vcl.list" +varnish v1 -cliok "backend.list -p" +varnish v1 -cliok "backend.set_health s1 auto" +varnish v1 -cliok "backend.list -p" + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cliok "backend.list" +varnish v1 -cliok "backend.set_health s1 sick" +varnish v1 -cliok "backend.list" + +client c1 { + txreq + rxresp + expect resp.status == 503 +} -run + +varnish v1 -cliok "backend.list" +varnish v1 -cliok "backend.set_health s1 healthy" +varnish v1 -cliok "backend.list" + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -clierr 106 "backend.set_health s1 foo" +varnish v1 -clierr 106 "backend.set_health s2 foo" +varnish v1 -clierr 106 "backend.set_health s2 auto" +varnish v1 -cliok "vcl.list" +varnish v1 -cliok "backend.list *" +varnish v1 -cliok "backend.list *.foo" +varnish v1 -cliok "backend.list vcl1.*" + diff --git a/bin/varnishtest/tests/c00091.vtc b/bin/varnishtest/tests/c00091.vtc new file mode 100644 index 0000000..59c63d1 --- /dev/null +++ b/bin/varnishtest/tests/c00091.vtc @@ -0,0 +1,50 @@ +varnishtest "vcl_backend_response{} retry with a UDS backend" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -hdr "foo: 1" + accept + rxreq + txresp -hdr "foo: 2" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { return (pass); } + sub vcl_backend_response { + set beresp.http.bar = bereq.retries; + if (beresp.http.foo != bereq.http.stop) { + return (retry); + } + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq -hdr "stop: 2" + rxresp + expect resp.http.foo == 2 +} -run + +delay .1 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -hdr "foo: 1" + accept + rxreq + txresp -hdr "foo: 2" + accept + rxreq + txresp -hdr "foo: 3" +} -start + +varnish v1 -cliok "param.set max_retries 2" + +client c1 { + txreq -hdr "stop: 3" + rxresp + expect resp.http.foo == 3 +} -run + +# XXX: Add a test which exceeds max_retries and gets 503 back diff --git a/bin/varnishtest/tests/c00092.vtc b/bin/varnishtest/tests/c00092.vtc new file mode 100644 index 0000000..df85d19 --- /dev/null +++ b/bin/varnishtest/tests/c00092.vtc @@ -0,0 +1,32 @@ +varnishtest "Check aborted backend body with a backend listening at UDS" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -nolen -hdr "Transfer-encoding: chunked" + chunked {} + barrier b1 sync + chunked {} + barrier b2 sync +} -start + +varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { + +} -start + + +client c1 { + txreq + rxresphdrs + expect resp.status == 200 + rxchunk + barrier b1 sync + rxchunk + barrier b2 sync + expect_close +} -run + + + diff --git a/bin/varnishtest/tests/c00093.vtc b/bin/varnishtest/tests/c00093.vtc new file mode 100644 index 0000000..625dc3d --- /dev/null +++ b/bin/varnishtest/tests/c00093.vtc @@ -0,0 +1,39 @@ +varnishtest "Test resp.is_streaming with a UDS backend" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -nolen -hdr "Content-Length: 10" + delay 1 + send "1234567890" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/synth") { + return(synth(200, "OK")); + } + } + sub vcl_synth { + set resp.http.streaming = resp.is_streaming; + } + sub vcl_deliver { + set resp.http.streaming = resp.is_streaming; + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.streaming == "true" + + delay 0.1 + + txreq + rxresp + expect resp.http.streaming == "false" + + txreq -url /synth + rxresp + expect resp.http.streaming == "false" +} -run + diff --git a/bin/varnishtest/tests/c00094.vtc b/bin/varnishtest/tests/c00094.vtc new file mode 100644 index 0000000..546efb0 --- /dev/null +++ b/bin/varnishtest/tests/c00094.vtc @@ -0,0 +1,51 @@ +varnishtest "Test Backend Polling with a backend listening at a UDS" + +barrier b1 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + # Probes + loop 8 { + rxreq + expect req.url == "/" + txresp -hdr "Bar: foo" -body "foobar" + accept + } + + loop 3 { + rxreq + expect req.url == "/" + txresp -status 404 -hdr "Bar: foo" -body "foobar" + accept + } + loop 2 { + rxreq + expect req.url == "/" + txresp -proto "FROBOZ" -status 200 -hdr "Bar: foo" -body "foobar" + accept + } + loop 2 { + rxreq + expect req.url == "/" + send "HTTP/1.1 200 \r\n" + accept + } + + barrier b1 sync +} -start + +varnish v1 -vcl { + + backend foo { + .path = "${s1_sock}"; + .probe = { + .timeout = 1 s; + .interval = 0.1 s; + } + } + +} -start + +barrier b1 sync + +varnish v1 -cliexpect "^CLI RX| -+U+ Good UNIX" "backend.list -p" +varnish v1 -cliexpect "^CLI RX| -+H{10}-{5}H{2}-{0,5} Happy" "backend.list -p" diff --git a/bin/varnishtest/tests/d00031.vtc b/bin/varnishtest/tests/d00031.vtc new file mode 100644 index 0000000..fa11ee3 --- /dev/null +++ b/bin/varnishtest/tests/d00031.vtc @@ -0,0 +1,86 @@ +varnishtest "Test round robin director with UDS backends" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -body "1" +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + txresp -body "22" +} -start + +server s3 -listen "${tmpdir}/s3.sock" { + rxreq + txresp -body "333" +} -start + +server s4 -listen "${tmpdir}/s4.sock" { + rxreq + txresp -body "4444" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new rr = directors.round_robin(); + rr.add_backend(s1); + rr.add_backend(s2); + rr.add_backend(s3); + rr.add_backend(s4); + } + + sub vcl_recv { + if (req.method == "DELETE") { + rr.remove_backend(s1); + rr.remove_backend(s2); + rr.remove_backend(s3); + return(synth(204)); + } + } + + sub vcl_backend_fetch { + set bereq.backend = rr.backend(); + } +} -start + +client c1 { + timeout 3 + txreq -url "/foo1" + rxresp + expect resp.bodylen == 1 + txreq -url "/foo2" + rxresp + expect resp.bodylen == 2 + txreq -url "/foo3" + rxresp + expect resp.bodylen == 3 + txreq -url "/foo4" + rxresp + expect resp.bodylen == 4 +} -run + +server s1 -start +server s2 -start + +client c2 { + timeout 3 + txreq -url "/foo11" + rxresp + expect resp.bodylen == 1 + txreq -url "/foo22" + rxresp + expect resp.bodylen == 2 +} -run + +server s4 -start + +client c3 { + txreq -req "DELETE" + rxresp + expect resp.status == 204 + txreq -url "/foo31" + rxresp + expect resp.bodylen == 4 +} -run diff --git a/bin/varnishtest/tests/m00046.vtc b/bin/varnishtest/tests/m00046.vtc index a5de275..dca4fba 100644 --- a/bin/varnishtest/tests/m00046.vtc +++ b/bin/varnishtest/tests/m00046.vtc @@ -21,7 +21,7 @@ client c1 -connect "${tmpdir}/v2.sock" { expect resp.http.v1_addr == "0.0.0.0" } -run -varnish v2 -errvcl {IP constant '"${v1_addr}"'} { +varnish v2 -errvcl {Cannot convert to an IP address: '"${v1_addr}"'} { import std; sub vcl_recv { diff --git a/bin/varnishtest/tests/t02013.vtc b/bin/varnishtest/tests/t02013.vtc index 47a226c..3711111 100644 --- a/bin/varnishtest/tests/t02013.vtc +++ b/bin/varnishtest/tests/t02013.vtc @@ -1,6 +1,6 @@ varnishtest "Direct H2 start over Unix domain sockets" -server s1 { +server s1 -listen "${tmpdir}/s1.sock" { rxreq expect req.http.host == foo.bar txresp \ diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index f4a8c88..d93d4d5 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -87,7 +87,7 @@ varnish v1 -errvcl "Unknown field:" { } } -varnish v1 -errvcl "Mandatory field 'host' missing." { +varnish v1 -errvcl "Expected .host or .path." { backend b1 { .port = "NONE"; } @@ -109,3 +109,30 @@ varnish v1 -errvcl "Unused backend b1, defined:" { backend b1 { .host = "127.0.0.1"; } backend default { .host = "127.0.0.1"; } } + +varnish v1 -errvcl "Address redefinition at:" { + backend b1 { + .host = "127.0.0.1"; + .path = "/path/to/uds"; + } +} + +varnish v1 -errvcl "Must be an absolute path:" { + backend b1 { + .path = "server.sock"; + } +} + +shell { rm -f ${tmpdir}/foo } + +varnish v1 -errvcl "Cannot stat:" { + backend b1 { + .path = "${tmpdir}/foo"; + } +} + +varnish v1 -errvcl "Not a socket:" { + backend b1 { + .path = "${tmpdir}"; + } +} diff --git a/bin/varnishtest/tests/v00055.vtc b/bin/varnishtest/tests/v00055.vtc new file mode 100644 index 0000000..4584f1b --- /dev/null +++ b/bin/varnishtest/tests/v00055.vtc @@ -0,0 +1,41 @@ +varnishtest "Check backend connection limit with UDS backends" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + barrier b1 sync + barrier b2 sync + txresp +} -start + +varnish v1 -vcl { + + backend default { + .path = "${s1_sock}"; + .max_connections = 1; + } + sub vcl_recv { + return(pass); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -start + + +client c2 { + barrier b1 sync + txreq + rxresp + expect resp.status == 503 +} -run + +barrier b2 sync +client c1 -wait + +varnish v1 -expect backend_busy == 1 diff --git a/bin/varnishtest/tests/v00056.vtc b/bin/varnishtest/tests/v00056.vtc new file mode 100644 index 0000000..7d3d507 --- /dev/null +++ b/bin/varnishtest/tests/v00056.vtc @@ -0,0 +1,70 @@ +varnishtest "Check req.backend.healthy with UDS backends" + +barrier b1 cond 2 +barrier b2 cond 2 +barrier b3 cond 2 +barrier b4 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + barrier b1 sync + expect req.url == "/" + txresp -body "slash" + accept + rxreq + barrier b2 sync + barrier b3 sync + expect req.url == "/" + txresp -body "slash" + accept + barrier b4 sync +} -start + +varnish v1 -vcl { + + import std; + + probe foo { + .url = "/"; + .timeout = 1s; + .interval = 1s; + .window = 3; + .threshold = 2; + .initial = 0; + } + + backend default { + .path = "${s1_sock}"; + .max_connections = 1; + .probe = foo; + } + + sub vcl_recv { + if (std.healthy(default)) { + return(synth(200,"Backend healthy")); + } else { + return(synth(500,"Backend sick")); + } + } +} -start + +varnish v1 -cliok "backend.list -p" + +client c1 { + txreq + rxresp + expect resp.status == 500 + + barrier b1 sync + + barrier b2 sync + txreq + rxresp + expect resp.status == 500 + + barrier b3 sync + barrier b4 sync + txreq + rxresp + expect resp.status == 200 +} -run diff --git a/bin/varnishtest/tests/v00057.vtc b/bin/varnishtest/tests/v00057.vtc new file mode 100644 index 0000000..58d8de3 --- /dev/null +++ b/bin/varnishtest/tests/v00057.vtc @@ -0,0 +1,41 @@ +varnishtest "Test backend .hosthdr with UDS backends" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + expect req.url == "/foo" + expect req.http.host == "snafu" + txresp -body "foo1" + + rxreq + expect req.url == "/bar" + expect req.http.host == "0.0.0.0" + txresp -body "foo1" +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq -url "/foo" -hdr "Host: snafu" + rxresp + txreq -url "/bar" + rxresp +} -run + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + expect req.url == "/barf" + expect req.http.host == "FOObar" + txresp -body "foo1" +} -start + +varnish v1 -vcl { + backend b1 { + .path = "${s2_sock}"; + .host_header = "FOObar"; + } +} + +client c1 { + txreq -url "/barf" + rxresp +} -run diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 9c084ba..9496046 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -392,9 +392,15 @@ cmd_server_genvcl(struct vsb *vsb) AZ(pthread_mutex_lock(&server_mtx)); VTAILQ_FOREACH(s, &servers, list) { - VSB_printf(vsb, - "backend %s { .host = \"%s\"; .port = \"%s\"; }\n", - s->name, s->aaddr, s->aport); + if (*s->listen != '/') + VSB_printf(vsb, + "backend %s { .host = \"%s\"; " + ".port = \"%s\"; }\n", + s->name, s->aaddr, s->aport); + else + VSB_printf(vsb, + "backend %s { .path = \"%s\"; }\n", + s->name, s->listen); } AZ(pthread_mutex_unlock(&server_mtx)); } diff --git a/include/tbl/backend_poll.h b/include/tbl/backend_poll.h index a6fa339..5c1e207 100644 --- a/include/tbl/backend_poll.h +++ b/include/tbl/backend_poll.h @@ -31,6 +31,7 @@ BITMAP(good_ipv4, '4', "Good IPv4", 0) BITMAP(good_ipv6, '6', "Good IPv6", 0) +BITMAP(good_unix, 'U', "Good UNIX", 0) BITMAP( err_xmit, 'x', "Error Xmit", 0) BITMAP(good_xmit, 'X', "Good Xmit", 0) BITMAP( err_recv, 'r', "Error Recv", 0) diff --git a/include/vrt.h b/include/vrt.h index c29f8d4..5a25e6b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,6 +53,7 @@ * * * 6.2 (scheduled for: 2018-03-15) + * path field added to struct vrt_backend * VRT_Healthy() added * VRT_VSC_Alloc() added * VRT_VSC_Destroy() added @@ -253,6 +254,7 @@ extern const void * const vrt_magic_string_unset; rigid char *ipv4_addr; \ rigid char *ipv6_addr; \ rigid char *port; \ + rigid char *path; \ rigid char *hosthdr; \ double connect_timeout; \ double first_byte_timeout; \ @@ -266,6 +268,7 @@ extern const void * const vrt_magic_string_unset; DA(ipv4_addr); \ DA(ipv6_addr); \ DA(port); \ + DA(path); \ DA(hosthdr); \ DN(connect_timeout); \ DN(first_byte_timeout); \ diff --git a/include/vus.h b/include/vus.h index 3747d92..ddaf331 100644 --- a/include/vus.h +++ b/include/vus.h @@ -33,3 +33,4 @@ typedef int vus_resolved_f(void *priv, const struct sockaddr_un *); int VUS_resolver(const char *path, vus_resolved_f *func, void *priv, const char **err); int VUS_bind(const struct sockaddr_un *uds, const char **errp); +int VUS_connect(const char *path, int msec); diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index 8471d15..69a223d 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -32,10 +32,12 @@ #include #include #include +#include #include "vdef.h" #include "vas.h" #include "vus.h" +#include "vtcp.h" int VUS_resolver(const char *path, vus_resolved_f *func, void *priv, @@ -95,3 +97,60 @@ VUS_bind(const struct sockaddr_un *uds, const char **errp) } return (sd); } + +int +VUS_connect(const char *path, int msec) +{ + int s, i; + struct pollfd fds[1]; + struct sockaddr_un uds; + socklen_t sl = (socklen_t) sizeof(uds); + + if (path == NULL) + return (-1); + /* Attempt the connect */ + assert(strlen(path) + 1 <= sizeof(uds.sun_path)); + uds.sun_family = PF_UNIX; + strcpy(uds.sun_path, path); + AN(sl); + + s = socket(PF_UNIX, SOCK_STREAM, 0); + if (s < 0) + return (s); + + /* Set the socket non-blocking */ + if (msec != 0) + (void)VTCP_nonblocking(s); + + i = connect(s, (const struct sockaddr *)&uds, sl); + if (i == 0) + return (s); + if (errno != EINPROGRESS) { + closefd(&s); + return (-1); + } + + if (msec < 0) { + /* + * Caller is responsible for waiting and + * calling VTCP_connected + */ + return (s); + } + + assert(msec > 0); + /* Exercise our patience, polling for write */ + fds[0].fd = s; + fds[0].events = POLLWRNORM; + fds[0].revents = 0; + i = poll(fds, 1, msec); + + if (i == 0) { + /* Timeout, close and give up */ + closefd(&s); + errno = ETIMEDOUT; + return (-1); + } + + return (VTCP_connected(s)); +} diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index db5ce71..640481d 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -81,20 +81,19 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, Fb(tl, 0, "\t.ipv6_addr = \"%s\",\n", ipv6a); } Fb(tl, 0, "\t.port = \"%s\",\n", pa); + Fb(tl, 0, "\t.path = (void *) 0,\n"); } /*-------------------------------------------------------------------- - * Parse a backend probe specification + * Disallow mutually exclusive field definitions */ static void -vcc_ProbeRedef(struct vcc *tl, struct token **t_did, +vcc_Redef(struct vcc *tl, const char *redef, struct token **t_did, struct token *t_field) { - /* .url and .request are mutually exclusive */ - if (*t_did != NULL) { - VSB_printf(tl->sb, "Probe request redefinition at:\n"); + VSB_printf(tl->sb, "%s redefinition at:\n", redef); vcc_ErrWhere(tl, t_field); VSB_printf(tl->sb, "Previous definition:\n"); vcc_ErrWhere(tl, *t_did); @@ -103,6 +102,10 @@ vcc_ProbeRedef(struct vcc *tl, struct token **t_did, *t_did = t_field; } +/*-------------------------------------------------------------------- + * Parse a backend probe specification + */ + static void vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) { @@ -152,7 +155,7 @@ vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) vcc_IsField(tl, &t_field, fs); ERRCHK(tl); if (vcc_IdIs(t_field, "url")) { - vcc_ProbeRedef(tl, &t_did, t_field); + vcc_Redef(tl, "Probe request", &t_did, t_field); ERRCHK(tl); ExpectErr(tl, CSTR); Fh(tl, 0, "\t.url = "); @@ -160,7 +163,7 @@ vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) Fh(tl, 0, ",\n"); vcc_NextToken(tl); } else if (vcc_IdIs(t_field, "request")) { - vcc_ProbeRedef(tl, &t_did, t_field); + vcc_Redef(tl, "Probe request", &t_did, t_field); ERRCHK(tl); ExpectErr(tl, CSTR); Fh(tl, 0, "\t.request =\n"); @@ -294,8 +297,10 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) struct token *t_val; struct token *t_host = NULL; struct token *t_port = NULL; + struct token *t_path = NULL; struct token *t_hosthdr = NULL; struct symbol *pb; + struct token *t_did = NULL; struct fld_spec *fs; struct inifin *ifp; struct vsb *vsb; @@ -304,8 +309,9 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) double t; fs = vcc_FldSpec(tl, - "!host", + "?host", "?port", + "?path", "?host_header", "?connect_timeout", "?first_byte_timeout", @@ -345,6 +351,8 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) vcc_IsField(tl, &t_field, fs); ERRCHK(tl); if (vcc_IdIs(t_field, "host")) { + vcc_Redef(tl, "Address", &t_did, t_field); + ERRCHK(tl); ExpectErr(tl, CSTR); assert(tl->t->dec != NULL); t_host = tl->t; @@ -356,6 +364,14 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) t_port = tl->t; vcc_NextToken(tl); SkipToken(tl, ';'); + } else if (vcc_IdIs(t_field, "path")) { + vcc_Redef(tl, "Address", &t_did, t_field); + ERRCHK(tl); + ExpectErr(tl, CSTR); + assert(tl->t->dec != NULL); + t_path = tl->t; + vcc_NextToken(tl); + SkipToken(tl, ';'); } else if (vcc_IdIs(t_field, "host_header")) { ExpectErr(tl, CSTR); assert(tl->t->dec != NULL); @@ -430,9 +446,19 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) vcc_FieldsOk(tl, fs); ERRCHK(tl); - /* Check that the hostname makes sense */ - assert(t_host != NULL); - Emit_Sockaddr(tl, t_host, t_port); + if (t_host == NULL && t_path == NULL) { + VSB_printf(tl->sb, "Expected .host or .path.\n"); + vcc_ErrWhere(tl, t_be); + return; + } + + assert(t_host != NULL || t_path != NULL); + if (t_host != NULL) + /* Check that the hostname makes sense */ + Emit_Sockaddr(tl, t_host, t_port); + else + /* Check that the path can be a legal UDS */ + Emit_UDS_Path(tl, t_path, "Backend path"); ERRCHK(tl); ExpectErr(tl, '}'); @@ -440,11 +466,14 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) /* We have parsed it all, emit the ident string */ /* Emit the hosthdr field, fall back to .host if not specified */ + /* If .path is specified, set "0.0.0.0". */ Fb(tl, 0, "\t.hosthdr = "); if (t_hosthdr != NULL) EncToken(tl->fb, t_hosthdr); - else + else if (t_host != NULL) EncToken(tl->fb, t_host); + else + Fb(tl, 0, "\"0.0.0.0\""); Fb(tl, 0, ",\n"); /* Close the struct */ diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 2d9478b..f530095 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -327,6 +327,8 @@ void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport, const char **ipv4, const char **ipv4_ascii, const char **ipv6, const char **ipv6_ascii, const char **p_ascii, int maxips, const struct token *t_err, const char *errid); +void Emit_UDS_Path(struct vcc *tl, const struct token *t_path, + const char *errid); double vcc_TimeUnit(struct vcc *); void vcc_ByteVal(struct vcc *, double *); void vcc_NumVal(struct vcc *, double *, int *); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index f67b141..17209dc 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -729,6 +729,19 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) case CSTR: assert(fmt != VOID); if (fmt == IP) { + if (*tl->t->dec == '/') { + /* + * On some platforms (e.g. FreeBSD), + * getaddrinfo(3) may resolve a path to a + * sockaddr_un if it happens to exist and + * is a socket. So don't let that happen. + */ + VSB_printf(tl->sb, + "Cannot convert to an IP address: "); + vcc_ErrToken(tl, tl->t); + vcc_ErrWhere(tl, tl->t); + return; + } Resolve_Sockaddr(tl, tl->t->dec, "80", &ip, NULL, &ip, NULL, NULL, 1, tl->t, "IP constant"); diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index a2ae14d..987eb98 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -33,6 +33,10 @@ #include #include #include +#include +#include +#include +#include #include "vcc_compile.h" @@ -161,6 +165,7 @@ rs_callback(void *priv, const struct suckaddr *vsa) assert(VSA_Sane(vsa)); v = VSA_Get_Proto(vsa); + assert(v != AF_UNIX); VTCP_name(vsa, a, sizeof a, p, sizeof p); VSB_printf(rss->vsb, "\t%s:%s\n", a, p); if (v == AF_INET) { @@ -252,6 +257,42 @@ Resolve_Sockaddr(struct vcc *tl, FREE_OBJ(rss); } +/* + * For UDS, we do not create a VSA. Check if it's an absolute path, can + * be accessed, and is a socket. If so, just emit the path field and set + * the IP suckaddrs to NULL. + */ +void +Emit_UDS_Path(struct vcc *tl, const struct token *t_path, const char *errid) +{ + struct stat st; + + AN(t_path); + AN(t_path->dec); + + if (*t_path->dec != '/') { + VSB_printf(tl->sb, + "%s: Must be an absolute path:\n", errid); + vcc_ErrWhere(tl, t_path); + return; + } + errno = 0; + if (stat(t_path->dec, &st) != 0) { + VSB_printf(tl->sb, "%s: Cannot stat: %s\n", errid, + strerror(errno)); + vcc_ErrWhere(tl, t_path); + return; + } + if (! S_ISSOCK(st.st_mode)) { + VSB_printf(tl->sb, "%s: Not a socket:\n", errid); + vcc_ErrWhere(tl, t_path); + return; + } + Fb(tl, 0, "\t.path = \"%s\",\n", t_path->dec); + Fb(tl, 0, "\t.ipv4_suckaddr = (void *) 0,\n"); + Fb(tl, 0, "\t.ipv6_suckaddr = (void *) 0,\n"); +} + /*-------------------------------------------------------------------- * Recognize and convert units of time, return seconds. */ From geoff at uplex.de Mon Mar 12 09:45:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:45:10 +0000 (UTC) Subject: [master] 0f5b435 Support Unix domain sockets for the -b arg. Message-ID: <20180312094510.90754B3BFC@lists.varnish-cache.org> commit 0f5b4350bc4ab59f38c36a94d6395f11d63aed1f Author: Geoff Simmons Date: Mon Feb 19 02:48:30 2018 +0100 Support Unix domain sockets for the -b arg. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index d6a4141..6e33f8d 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -593,7 +593,12 @@ main(int argc, char * const *argv) AN(vsb); VSB_printf(vsb, "vcl 4.0;\n"); VSB_printf(vsb, "backend default {\n"); - VSB_printf(vsb, " .host = \"%s\";\n", optarg); + if (*optarg != '/') + VSB_printf(vsb, " .host = \"%s\";\n", + optarg); + else + VSB_printf(vsb, " .path = \"%s\";\n", + optarg); VSB_printf(vsb, "}\n"); AZ(VSB_finish(vsb)); fa->src = strdup(VSB_data(vsb)); diff --git a/bin/varnishtest/tests/b00061.vtc b/bin/varnishtest/tests/b00061.vtc new file mode 100644 index 0000000..d06f3c6 --- /dev/null +++ b/bin/varnishtest/tests/b00061.vtc @@ -0,0 +1,15 @@ +varnishtest "-b arg with a Unix domain socket" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp -hdr "s1: got it" +} -start + +varnish v1 -arg "-b ${s1_sock}" -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.s1 == "got it" +} -run From geoff at uplex.de Mon Mar 12 09:45:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:45:10 +0000 (UTC) Subject: [master] 2b874fe Test dynamic creation of backends listening at Unix domain sockets. Message-ID: <20180312094510.B0211B3C00@lists.varnish-cache.org> commit 2b874fe7e19fcddc9f69a29187191a4b1cb6d557 Author: Geoff Simmons Date: Mon Feb 19 03:13:35 2018 +0100 Test dynamic creation of backends listening at Unix domain sockets. diff --git a/bin/varnishtest/tests/d00032.vtc b/bin/varnishtest/tests/d00032.vtc new file mode 100644 index 0000000..5e30767 --- /dev/null +++ b/bin/varnishtest/tests/d00032.vtc @@ -0,0 +1,70 @@ +varnishtest "Test dynamic backends listening at Unix domain sockets" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp +} -start + +varnish v1 -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -errvcl {path must be an absolute path} { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds(""); + } +} + +varnish v1 -errvcl {path must be an absolute path} { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("s1.sock"); + } +} + +shell { rm -f ${tmpdir}/foo } + +varnish v1 -errvcl {Cannot stat path} { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${tmpdir}/foo"); + } +} + +varnish v1 -errvcl {is not a socket} { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${tmpdir}"); + } +} diff --git a/bin/varnishtest/tests/d00033.vtc b/bin/varnishtest/tests/d00033.vtc new file mode 100644 index 0000000..1d837ab --- /dev/null +++ b/bin/varnishtest/tests/d00033.vtc @@ -0,0 +1,47 @@ +varnishtest "Test dynamic UDS backends hot swap" + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + expect req.url == "/foo" + txresp +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + expect req.url == "/bar" + txresp +} -start + +varnish v1 -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.refresh(req.http.X-Path); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + + txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run diff --git a/bin/varnishtest/tests/d00034.vtc b/bin/varnishtest/tests/d00034.vtc new file mode 100644 index 0000000..314673f --- /dev/null +++ b/bin/varnishtest/tests/d00034.vtc @@ -0,0 +1,58 @@ +varnishtest "Test dynamic UDS backends hot swap while being used" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + expect req.url == "/foo" + barrier b1 sync + barrier b2 sync + txresp +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + expect req.url == "/bar" + barrier b2 sync + txresp +} -start + +varnish v1 -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.refresh(req.http.X-Path); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +client c2 { + barrier b1 sync + txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +client c1 -wait diff --git a/bin/varnishtest/tests/d00035.vtc b/bin/varnishtest/tests/d00035.vtc new file mode 100644 index 0000000..e47c670 --- /dev/null +++ b/bin/varnishtest/tests/d00035.vtc @@ -0,0 +1,59 @@ +varnishtest "Test dynamic UDS backends hot swap during a pipe" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + expect req.url == "/foo" + barrier b1 sync + barrier b2 sync + txresp +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + expect req.url == "/bar" + barrier b2 sync + txresp +} -start + +varnish v1 -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.refresh(req.http.X-Path); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + return (pipe); + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +client c2 { + barrier b1 sync + txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" + rxresp + expect resp.status == 200 + + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +client c1 -wait diff --git a/bin/varnishtest/tests/d00036.vtc b/bin/varnishtest/tests/d00036.vtc new file mode 100644 index 0000000..3b739c7 --- /dev/null +++ b/bin/varnishtest/tests/d00036.vtc @@ -0,0 +1,62 @@ +varnishtest "Test dynamic UDS backend hot swap after it was picked by a bereq" + +barrier b1 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + txresp +} -start + +varnish v1 -vcl { + import std; + import debug; + import vtc; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.refresh(req.http.X-Path); + return (synth(200)); + } + } + + sub vcl_backend_fetch { + set bereq.backend = s1.backend(); + # hot swap should happen while we sleep + vtc.sleep(2s); + if (std.healthy(bereq.backend)) { + return(abandon); + } else { + set bereq.backend = s1.backend(); + } + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + barrier b1 sync + rxresp + expect resp.status == 200 +} + +client c2 { + barrier b1 sync + delay 0.1 + txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" + rxresp + expect resp.status == 200 +} + +client c1 -start +client c2 -run +client c1 -wait diff --git a/bin/varnishtest/tests/d00037.vtc b/bin/varnishtest/tests/d00037.vtc new file mode 100644 index 0000000..51c6e8f --- /dev/null +++ b/bin/varnishtest/tests/d00037.vtc @@ -0,0 +1,76 @@ +varnishtest "Test a dynamic UDS backend discard during a request" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + expect req.url == "/foo" + barrier b1 sync + barrier b2 sync + txresp +} -start + +varnish v1 -arg "-p thread_pools=1" -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + set req.backend_hint = s1.backend(); + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + expect req.url == "/bar" + txresp +} -start + +barrier b1 sync + +varnish v1 -vcl { + import debug; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s2 = debug.dyn_uds("${s2_sock}"); + } + + sub vcl_recv { + set req.backend_hint = s2.backend(); + } +} + +varnish v1 -cli "vcl.discard vcl1" +barrier b2 sync + +client c1 -wait +delay 2 + +varnish v1 -expect MAIN.n_backend == 4 + +varnish v1 -expect n_vcl_avail == 1 +varnish v1 -expect n_vcl_discard == 1 + +client c1 { + txreq -url "/bar" + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cli "vcl.list" +varnish v1 -expect n_vcl_avail == 1 diff --git a/bin/varnishtest/tests/d00038.vtc b/bin/varnishtest/tests/d00038.vtc new file mode 100644 index 0000000..1b8b704 --- /dev/null +++ b/bin/varnishtest/tests/d00038.vtc @@ -0,0 +1,61 @@ +varnishtest "Test a dynamic UDS backend hot swap after it was hinted to a req" + +barrier b1 cond 2 + +server s1 -listen "${tmpdir}/s1.sock" { +} -start + +server s2 -listen "${tmpdir}/s2.sock" { + rxreq + txresp +} -start + +varnish v1 -vcl { + import std; + import debug; + import vtc; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_init { + new s1 = debug.dyn_uds("${s1_sock}"); + } + + sub vcl_recv { + if (req.method == "SWAP") { + s1.refresh(req.http.X-Path); + return (synth(200)); + } + set req.backend_hint = s1.backend(); + # hot swap should happen while we sleep + vtc.sleep(2s); + if (std.healthy(req.backend_hint)) { + return(synth(800)); + } else { + set req.backend_hint = s1.backend(); + } + } +} -start + +varnish v1 -expect MAIN.n_backend == 2 + +client c1 { + txreq + barrier b1 sync + rxresp + expect resp.status == 200 +} + +client c2 { + barrier b1 sync + delay 0.1 + txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" + rxresp + expect resp.status == 200 +} + +client c1 -start +client c2 -run +client c1 -wait + +varnish v1 -cli backend.list diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 5e1ac0c..20f75ec 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -141,6 +141,19 @@ $Method VOID .refresh(STRING addr, STRING port) Dynamically refresh & (always!) replace the backend by a new one. +$Object dyn_uds(STRING path) + +Dynamically create a single-backend director listening at a Unix +domain socket, path must not be empty. + +$Method BACKEND .backend() + +Return the dynamic UDS backend. + +$Method VOID .refresh(STRING path) + +Dynamically refresh & (always!) replace the backend by a new UDS backend. + $Function VOID vcl_release_delay(DURATION) Hold a reference to the VCL when it goes cold for the given delay. diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index ebd7330..17b0da2 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -32,6 +32,10 @@ #include #include #include +#include +#include +#include +#include #include "cache/cache.h" #include "cache/cache_director.h" @@ -47,6 +51,14 @@ struct xyzzy_debug_dyn { struct director *dir; }; +struct xyzzy_debug_dyn_uds { + unsigned magic; +#define VMOD_DEBUG_UDS_MAGIC 0x6c7370e6 + pthread_mutex_t mtx; + char *vcl_name; + struct director *dir; +}; + static void dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, VCL_STRING addr, VCL_STRING port) @@ -167,3 +179,105 @@ xyzzy_dyn_refresh(VRT_CTX, struct xyzzy_debug_dyn *dyn, CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); dyn_dir_init(ctx, dyn, addr, port); } + +static int +dyn_uds_init(VRT_CTX, struct xyzzy_debug_dyn_uds *uds, VCL_STRING path) +{ + struct director *dir, *dir2; + struct vrt_backend vrt; + struct stat st; + + if (path == NULL) { + VRT_fail(ctx, "path is NULL"); + return (-1); + } + if (*path != '/') { + VRT_fail(ctx, "path must be an absolute path: %s", path); + return (-1); + } + errno = 0; + if (stat(path, &st) != 0) { + VRT_fail(ctx, "Cannot stat path %s: %s", path, strerror(errno)); + return (-1); + } + if (! S_ISSOCK(st.st_mode)) { + VRT_fail(ctx, "%s is not a socket", path); + return (-1); + } + + INIT_OBJ(&vrt, VRT_BACKEND_MAGIC); + vrt.path = path; + vrt.vcl_name = uds->vcl_name; + vrt.hosthdr = "localhost"; + vrt.ipv4_suckaddr = NULL; + vrt.ipv6_suckaddr = NULL; + + if ((dir = VRT_new_backend(ctx, &vrt)) == NULL) + return (-1); + + AZ(pthread_mutex_lock(&uds->mtx)); + dir2 = uds->dir; + uds->dir = dir; + AZ(pthread_mutex_unlock(&uds->mtx)); + + if (dir2 != NULL) + VRT_delete_backend(ctx, &dir2); + return (0); +} + +VCL_VOID v_matchproto_(td_debug_dyn_uds__init) +xyzzy_dyn_uds__init(VRT_CTX, struct xyzzy_debug_dyn_uds **udsp, + const char *vcl_name, VCL_STRING path) +{ + struct xyzzy_debug_dyn_uds *uds; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(udsp); + AZ(*udsp); + AN(vcl_name); + + ALLOC_OBJ(uds, VMOD_DEBUG_UDS_MAGIC); + AN(uds); + REPLACE(uds->vcl_name, vcl_name); + AZ(pthread_mutex_init(&uds->mtx, NULL)); + + if (dyn_uds_init(ctx, uds, path) != 0) { + free(uds->vcl_name); + AZ(pthread_mutex_destroy(&uds->mtx)); + FREE_OBJ(uds); + return; + } + *udsp = uds; +} + +VCL_VOID v_matchproto_(td_debug_dyn_uds__fini) +xyzzy_dyn_uds__fini(struct xyzzy_debug_dyn_uds **udsp) +{ + struct xyzzy_debug_dyn_uds *uds; + + if (udsp == NULL || *udsp == NULL) + return; + CHECK_OBJ(*udsp, VMOD_DEBUG_UDS_MAGIC); + uds = *udsp; + free(uds->vcl_name); + AZ(pthread_mutex_destroy(&uds->mtx)); + FREE_OBJ(uds); + udsp = NULL; +} + +VCL_BACKEND v_matchproto_(td_debug_dyn_uds_backend) +xyzzy_dyn_uds_backend(VRT_CTX, struct xyzzy_debug_dyn_uds *uds) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(uds, VMOD_DEBUG_UDS_MAGIC); + AN(uds->dir); + return (uds->dir); +} + +VCL_VOID v_matchproto_(td_debug_dyn_uds_refresh) +xyzzy_dyn_uds_refresh(VRT_CTX, struct xyzzy_debug_dyn_uds *uds, VCL_STRING path) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(uds, VMOD_DEBUG_UDS_MAGIC); + (void) dyn_uds_init(ctx, uds, path); +} From geoff at uplex.de Mon Mar 12 09:45:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:45:10 +0000 (UTC) Subject: [master] bd74e05 Verify outputs of varnishncsa -b when UDS addresses are in use. Message-ID: <20180312094510.C68FCB3C04@lists.varnish-cache.org> commit bd74e05ac0bbe2b7c673274ab4d35162b835b71f Author: Geoff Simmons Date: Wed Feb 21 15:29:06 2018 +0100 Verify outputs of varnishncsa -b when UDS addresses are in use. diff --git a/bin/varnishtest/tests/u00013.vtc b/bin/varnishtest/tests/u00013.vtc index 1d9b47d..bdfe8d1 100644 --- a/bin/varnishtest/tests/u00013.vtc +++ b/bin/varnishtest/tests/u00013.vtc @@ -1,9 +1,12 @@ varnishtest "varnishncsa outputs when UDS addresses are in use" -# The -c %h formatter gets its value from ReqStart, which now may be -# 0.0.0.0 for a UDS address. +# The %h formatter gets its value from ReqStart or BackendStart, +# which now may be a UDS address. -server s1 { +# For UDS backends without a .hosthdr setting, the Host header is +# set to "localhost", which may appear in %r output. + +server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp } -start @@ -22,3 +25,11 @@ shell -expect "0.0.0.0" { shell -expect "http://localhost/" { varnishncsa -n ${v1_name} -d -c -F "%r" } + +shell -expect "0.0.0.0" { + varnishncsa -n ${v1_name} -d -b -F "%h" +} + +shell -expect "http://0.0.0.0/" { + varnishncsa -n ${v1_name} -d -b -F "%r" +} From geoff at uplex.de Mon Mar 12 09:55:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:55:10 +0000 (UTC) Subject: [master] b38c3d3 Implement VCL var local.endpoint (VCL >= 4.1). Message-ID: <20180312095510.48167B401A@lists.varnish-cache.org> commit b38c3d32bc060c213e380ca553dfa927822e863d Author: Geoff Simmons Date: Wed Mar 7 17:15:49 2018 +0100 Implement VCL var local.endpoint (VCL >= 4.1). diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 2d08ffa..f543157 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -748,6 +748,26 @@ GIP(server) /*--------------------------------------------------------------------*/ VCL_STRING +VRT_r_local_endpoint(VRT_CTX) +{ + struct sess *sp; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + if (VALID_OBJ(ctx->req, REQ_MAGIC)) + sp = ctx->req->sp; + else { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + sp = ctx->bo->sp; + } + + CHECK_OBJ_NOTNULL(sp->listen_sock, LISTEN_SOCK_MAGIC); + AN(sp->listen_sock->endpoint); + return (sp->listen_sock->endpoint); +} + +/*--------------------------------------------------------------------*/ + +VCL_STRING VRT_r_server_identity(VRT_CTX) { diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc index 761ba0d..3f2a3e3 100644 --- a/bin/varnishtest/tests/c00087.vtc +++ b/bin/varnishtest/tests/c00087.vtc @@ -5,7 +5,7 @@ server s1 -listen "${tmpdir}/s1.sock" { txresp } -start -varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { +varnish v1 -syntax 4.1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { acl acl1 { "${localhost}"; } @@ -16,6 +16,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { set beresp.http.b-local = local.ip; set beresp.http.b-remote = remote.ip; set beresp.http.b-compare = local.ip == remote.ip; + set beresp.http.b-endpoint = local.endpoint; } sub vcl_deliver { @@ -28,6 +29,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { set resp.http.server_acl = server.ip ~ acl1; set resp.http.local_acl = local.ip ~ acl1; set resp.http.remote_acl = remote.ip ~ acl1; + set resp.http.c-endpoint = local.endpoint; } } -start @@ -40,11 +42,13 @@ client c1 -connect "${tmpdir}/v1.sock" { expect resp.http.c-local == "0.0.0.0" expect resp.http.c-remote == "0.0.0.0" expect resp.http.c-compare == "true" + expect resp.http.c-endpoint == "${tmpdir}/v1.sock" expect resp.http.b-client == "0.0.0.0" expect resp.http.b-server == "0.0.0.0" expect resp.http.b-local == "0.0.0.0" expect resp.http.b-remote == "0.0.0.0" expect resp.http.b-compare == "true" + expect resp.http.b-endpoint == "${tmpdir}/v1.sock" expect resp.http.client_acl == "false" expect resp.http.server_acl == "false" expect resp.http.local_acl == "false" diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 9a20422..9639053 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -106,13 +106,27 @@ varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'sess.xid' (Only available whe } } +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.endpoint' (Only available when 4.1 <= VCL syntax)} { + sub vcl_recv { + set req.http.Endpoint = local.endpoint; + } +} + +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.endpoint' (Only available when 4.1 <= VCL syntax)} { + sub vcl_backend_fetch { + set bereq.http.Endpoint = local.endpoint; + } +} + varnish v1 -syntax 4.1 -vcl+backend { sub vcl_backend_response { set beresp.http.B-Sess-XID = sess.xid; + set beresp.http.B-Endpoint = local.endpoint; } sub vcl_deliver { set resp.http.C-Sess-XID = sess.xid; + set resp.http.C-Endpoint = local.endpoint; } } @@ -123,4 +137,6 @@ client c1 { expect resp.http.C-Sess-XID ~ "^[0-9]+$" expect resp.http.B-Sess-XID ~ "^[0-9]+$" expect resp.http.C-Sess-XID == resp.http.B-Sess-XID + expect resp.http.C-Endpoint == "${v1_addr}:${v1_port}" + expect resp.http.B-Endpoint == "${v1_addr}:${v1_port}" } -run diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 8f20afc..c66386c 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -52,22 +52,34 @@ With PROXY protocol:: CLIENT ------------ PROXY ------------ VARNISHD -local.endpoint +local.ip + + Type: IP + + Readable from: client, backend + + The IP address (and port number) of the local end of the + TCP connection, for instance `192.168.1.1:81` + + If the connection is a UNIX domain socket, the value + will be `0.0.0.0:0` + +local.endpoint ``VCL >= 4.1`` Type: STRING - Readable from: client + Readable from: client, backend The address of the '-a' socket the session was accepted on. If the argument was `-a foo=:81` this would be ":81" -local.socket +local.socket ``VCL >= 4.1`` Type: STRING - Readable from: client + Readable from: client, backend The name of the '-a' socket the session was accepted on. @@ -76,18 +88,6 @@ local.socket Note that all '-a' gets a default name on the form `a%d` if no name is provided. -local.ip - - Type: IP - - Readable from: client, backend - - The IP address (and port number) of the local end of the - TCP connection, for instance `192.168.1.1:81` - - If the connection is a UNIX domain socket, the value - will be `0.0.0.0:0` - remote.ip Type: IP From geoff at uplex.de Mon Mar 12 09:55:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 09:55:10 +0000 (UTC) Subject: [master] 29c0c57 Implement VCL var local.socket (VCL >= 4.1). Message-ID: <20180312095510.712EDB401E@lists.varnish-cache.org> commit 29c0c5711049ac2d4cde84ad0eecabd1cbc25a3e Author: Geoff Simmons Date: Wed Mar 7 18:04:31 2018 +0100 Implement VCL var local.socket (VCL >= 4.1). diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index f543157..86df664 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -745,26 +745,33 @@ GIP(client) GIP(server) #undef GIP -/*--------------------------------------------------------------------*/ - -VCL_STRING -VRT_r_local_endpoint(VRT_CTX) -{ - struct sess *sp; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (VALID_OBJ(ctx->req, REQ_MAGIC)) - sp = ctx->req->sp; - else { - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - sp = ctx->bo->sp; - } +/*-------------------------------------------------------------------- + * local.[endpoint|socket] + */ - CHECK_OBJ_NOTNULL(sp->listen_sock, LISTEN_SOCK_MAGIC); - AN(sp->listen_sock->endpoint); - return (sp->listen_sock->endpoint); +#define LOC(var,fld) \ +VCL_STRING \ +VRT_r_local_##var(VRT_CTX) \ +{ \ + struct sess *sp; \ + \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + if (VALID_OBJ(ctx->req, REQ_MAGIC)) \ + sp = ctx->req->sp; \ + else { \ + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ + sp = ctx->bo->sp; \ + } \ + \ + CHECK_OBJ_NOTNULL(sp->listen_sock, LISTEN_SOCK_MAGIC); \ + AN(sp->listen_sock->fld); \ + return (sp->listen_sock->fld); \ } +LOC(endpoint, endpoint) +LOC(socket, name) +#undef LOC + /*--------------------------------------------------------------------*/ VCL_STRING diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc index 3f2a3e3..443cea5 100644 --- a/bin/varnishtest/tests/c00087.vtc +++ b/bin/varnishtest/tests/c00087.vtc @@ -5,7 +5,7 @@ server s1 -listen "${tmpdir}/s1.sock" { txresp } -start -varnish v1 -syntax 4.1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { +varnish v1 -syntax 4.1 -arg "-a foo=${tmpdir}/v1.sock" -vcl+backend { acl acl1 { "${localhost}"; } @@ -17,6 +17,7 @@ varnish v1 -syntax 4.1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { set beresp.http.b-remote = remote.ip; set beresp.http.b-compare = local.ip == remote.ip; set beresp.http.b-endpoint = local.endpoint; + set beresp.http.b-socket = local.socket; } sub vcl_deliver { @@ -30,6 +31,7 @@ varnish v1 -syntax 4.1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { set resp.http.local_acl = local.ip ~ acl1; set resp.http.remote_acl = remote.ip ~ acl1; set resp.http.c-endpoint = local.endpoint; + set resp.http.c-socket = local.socket; } } -start @@ -43,12 +45,14 @@ client c1 -connect "${tmpdir}/v1.sock" { expect resp.http.c-remote == "0.0.0.0" expect resp.http.c-compare == "true" expect resp.http.c-endpoint == "${tmpdir}/v1.sock" + expect resp.http.c-socket == "foo" expect resp.http.b-client == "0.0.0.0" expect resp.http.b-server == "0.0.0.0" expect resp.http.b-local == "0.0.0.0" expect resp.http.b-remote == "0.0.0.0" expect resp.http.b-compare == "true" expect resp.http.b-endpoint == "${tmpdir}/v1.sock" + expect resp.http.b-socket == "foo" expect resp.http.client_acl == "false" expect resp.http.server_acl == "false" expect resp.http.local_acl == "false" diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 9639053..699137b 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -118,15 +118,29 @@ varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.endpoint' (Only availab } } +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.socket' (Only available when 4.1 <= VCL syntax)} { + sub vcl_recv { + set req.http.Socket = local.socket; + } +} + +varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.socket' (Only available when 4.1 <= VCL syntax)} { + sub vcl_backend_fetch { + set bereq.http.Socket = local.socket; + } +} + varnish v1 -syntax 4.1 -vcl+backend { sub vcl_backend_response { set beresp.http.B-Sess-XID = sess.xid; set beresp.http.B-Endpoint = local.endpoint; + set beresp.http.B-Socket = local.socket; } sub vcl_deliver { set resp.http.C-Sess-XID = sess.xid; set resp.http.C-Endpoint = local.endpoint; + set resp.http.C-Socket = local.socket; } } @@ -139,4 +153,52 @@ client c1 { expect resp.http.C-Sess-XID == resp.http.B-Sess-XID expect resp.http.C-Endpoint == "${v1_addr}:${v1_port}" expect resp.http.B-Endpoint == "${v1_addr}:${v1_port}" + expect resp.http.C-Socket == "a0" + expect resp.http.B-Socket == "a0" +} -run + +varnish v1 -stop + +server s1 { + rxreq + txresp + rxreq + txresp +} -start + +varnish v2 -arg "-a foo=${tmpdir}/foo.sock -a bar=${tmpdir}/bar.sock" \ + -syntax 4.1 -vcl+backend { + + sub vcl_backend_response { + set beresp.http.B-Endpoint = local.endpoint; + set beresp.http.B-Socket = local.socket; + } + + sub vcl_deliver { + set resp.http.C-Endpoint = local.endpoint; + set resp.http.C-Socket = local.socket; + } +} -start + +client c2 -connect "${tmpdir}/foo.sock" { + txreq + rxresp + expect resp.status == 200 + expect resp.http.C-Endpoint == "${tmpdir}/foo.sock" + expect resp.http.B-Endpoint == "${tmpdir}/foo.sock" + expect resp.http.C-Socket == "foo" + expect resp.http.B-Socket == "foo" +} -run + +# The backend endpoint/socket may be either of the two possibilities, +# because the busyobj may point to the session started for the first +# fetch. +client c2 -connect "${tmpdir}/bar.sock" { + txreq + rxresp + expect resp.status == 200 + expect resp.http.C-Endpoint == "${tmpdir}/bar.sock" + expect resp.http.B-Endpoint ~ "^${tmpdir}/(bar|foo).sock$" + expect resp.http.C-Socket == "bar" + expect resp.http.B-Socket ~ "^(bar|foo)$" } -run From geoff at uplex.de Mon Mar 12 10:06:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 10:06:08 +0000 (UTC) Subject: [master] 8337f6f Add the $Synopsis directive to vmod.vcc sources. Message-ID: <20180312100608.B5131B43B2@lists.varnish-cache.org> commit 8337f6ffc66c68c0b5f8265eb58e36112d06a507 Author: Geoff Simmons Date: Fri Mar 9 14:04:22 2018 +0100 Add the $Synopsis directive to vmod.vcc sources. Vaild values are 'auto' or 'manual', default 'auto'. With 'auto' you get the auto-generated SYNOPSIS in the VMOD docs. With 'manual', the SYNOPSIS is just skipped, so the VMOD author has to write it. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e1770bb..569758b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -602,15 +602,16 @@ class s_module(stanza): fo.write("\n") fo.write(":Manual section: " + self.vcc.mansection + "\n") - fo.write("\n") - write_rst_hdr(fo, "SYNOPSIS", "=") - fo.write("\n") - fo.write("\n::\n\n") - fo.write(' import %s [from "path"] ;\n' % self.vcc.modname) - fo.write(" \n") - for c in self.vcc.contents: - c.synopsis(fo, man) - fo.write("\n") + if self.vcc.auto_synopsis: + fo.write("\n") + write_rst_hdr(fo, "SYNOPSIS", "=") + fo.write("\n") + fo.write("\n::\n\n") + fo.write(' import %s [from "path"] ;\n' % self.vcc.modname) + fo.write(" \n") + for c in self.vcc.contents: + c.synopsis(fo, man) + fo.write("\n") def rsttail(self, fo, man): @@ -652,6 +653,15 @@ class s_prefix(stanza): self.vcc.contents.append(self) +class s_synopsis(stanza): + def parse(self): + if self.line[1] not in ('auto', 'manual'): + err("Valid Synopsis values are 'auto' or 'manual', got '%s'\n" % + self.line[1]) + self.vcc.auto_synopsis = self.line[1] == 'auto' + self.vcc.contents.append(self) + + class s_event(stanza): def parse(self): self.event_func = self.line[1] @@ -814,6 +824,7 @@ dispatch = { "Function": s_function, "Object": s_object, "Method": s_method, + "Synopsis": s_synopsis, } @@ -828,6 +839,7 @@ class vcc(object): self.copyright = "" self.enums = {} self.strict_abi = True + self.auto_synopsis = True def openfile(self, fn): self.commit_files.append(fn) From phk at FreeBSD.org Mon Mar 12 10:23:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 10:23:08 +0000 (UTC) Subject: [master] 4ae5e18 Improve a diagnostic Message-ID: <20180312102308.25B4DB4921@lists.varnish-cache.org> commit 4ae5e1816aac66a7b7dd8b619fec0b402e015bbe Author: Poul-Henning Kamp Date: Mon Mar 12 08:55:43 2018 +0000 Improve a diagnostic diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index e2e252e..1699ff7 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -260,9 +260,9 @@ vcc_ParseImport(struct vcc *tl) vmd->vrt_minor > VRT_MINOR_VERSION)) { VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tVMOD version %u.%u\n", + VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", vmd->vrt_major, vmd->vrt_minor); - VSB_printf(tl->sb, "\tvarnishd version %u.%u\n", + VSB_printf(tl->sb, "\tvarnishd provices ABI version %u.%u\n", VRT_MAJOR_VERSION, VRT_MINOR_VERSION); vcc_ErrWhere(tl, mod); return; From phk at FreeBSD.org Mon Mar 12 10:23:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 10:23:08 +0000 (UTC) Subject: [master] fee476a Move another batch of stuff from cache.h to cache_varnishd.h Message-ID: <20180312102308.3E7A8B4924@lists.varnish-cache.org> commit fee476acc085e6563225d35fd8e19f1b10f04139 Author: Poul-Henning Kamp Date: Mon Mar 12 09:35:36 2018 +0000 Move another batch of stuff from cache.h to cache_varnishd.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index eb1c43d..9490129 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -90,12 +90,11 @@ struct backend; struct ban; struct ban_proto; struct cli; -struct cli_proto; +struct http_conn; struct mempool; struct objcore; struct objhead; struct pool; -struct poolparam; struct sess; struct transport; struct worker; @@ -125,6 +124,7 @@ enum fetch_step { }; /*--------------------------------------------------------------------*/ + struct lock { void *priv; }; // Opaque /*-------------------------------------------------------------------- @@ -166,34 +166,6 @@ struct http { uint8_t conds; /* If-* headers present */ }; -/*-------------------------------------------------------------------- - * HTTP Protocol connection structure - * - * This is the protocol independent object for a HTTP connection, used - * both for backend and client sides. - * - */ - -struct http_conn { - unsigned magic; -#define HTTP_CONN_MAGIC 0x3e19edd1 - - int *rfd; - enum sess_close doclose; - enum body_status body_status; - struct ws *ws; - char *rxbuf_b; - char *rxbuf_e; - char *pipeline_b; - char *pipeline_e; - ssize_t content_length; - void *priv; - - /* Timeouts */ - double first_byte_timeout; - double between_bytes_timeout; -}; - /*--------------------------------------------------------------------*/ struct acct_req { @@ -211,9 +183,9 @@ struct acct_bereq { /*--------------------------------------------------------------------*/ struct vsl_log { - uint32_t *wlb, *wlp, *wle; - unsigned wlr; - unsigned wid; + uint32_t *wlb, *wlp, *wle; + unsigned wlr; + unsigned wid; }; /*--------------------------------------------------------------------*/ @@ -512,7 +484,7 @@ struct req { double t_prev; /* Previous timestamp logged */ double t_req; /* Headers complete */ - struct http_conn htc[1]; + struct http_conn *htc; struct vfp_ctx *vfc; const char *client_identity; @@ -591,9 +563,6 @@ struct sess { /* Prototypes etc ----------------------------------------------------*/ -/* Cross file typedefs */ - -typedef enum htc_status_e htc_complete_f(struct http_conn *); /* cache_ban.c */ @@ -608,47 +577,6 @@ void BAN_Abandon(struct ban_proto *b); extern pthread_t cli_thread; #define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0) -/* cache_expire.c */ - -/* - * The set of variables which control object expiry are inconveniently - * 24 bytes long (double+3*float) and this causes alignment waste if - * we put then in a struct. - * These three macros operate on the struct we don't use. - */ - -#define EXP_ZERO(xx) \ - do { \ - (xx)->t_origin = 0.0; \ - (xx)->ttl = 0.0; \ - (xx)->grace = 0.0; \ - (xx)->keep = 0.0; \ - } while (0) - -#define EXP_COPY(to,fm) \ - do { \ - (to)->t_origin = (fm)->t_origin; \ - (to)->ttl = (fm)->ttl; \ - (to)->grace = (fm)->grace; \ - (to)->keep = (fm)->keep; \ - } while (0) - -#define EXP_WHEN(to) \ - ((to)->t_origin + (to)->ttl + (to)->grace + (to)->keep) - -/* cache_exp.c */ -void EXP_Rearm(struct objcore *, double now, double ttl, double grace, - double keep); - -/* cache_fetch.c */ -enum vbf_fetch_mode_e { - VBF_NORMAL = 0, - VBF_PASS = 1, - VBF_BACKGROUND = 2, -}; -void VBF_Fetch(struct worker *wrk, struct req *req, - struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e); - /* cache_http.c */ unsigned HTTP_estimate(unsigned nhttp); void HTTP_Copy(struct http *to, const struct http * const fm); @@ -696,14 +624,6 @@ int HTTP_IterHdrPack(struct worker *, struct objcore *, const char **); const char *HTTP_GetHdrPack(struct worker *, struct objcore *, const char *hdr); enum sess_close http_DoConnection(struct http *hp); -/* cache_http1_proto.c */ - -htc_complete_f HTTP1_Complete; -uint16_t HTTP1_DissectRequest(struct http_conn *, struct http *); -uint16_t HTTP1_DissectResponse(struct http_conn *, struct http *resp, - const struct http *req); -unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); - #define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ #define HTTPH_R_FETCH (1 << 1) /* Request (c->b) for fetch */ #define HTTPH_A_INS (1 << 2) /* Response (b->o) for insert */ @@ -719,7 +639,6 @@ extern const char H__Reason[]; /* cache_main.c */ #define VXID(u) ((u) & VSL_IDENTMASK) uint32_t VXID_Get(struct worker *, uint32_t marker); -extern volatile struct params * cache_param; extern pthread_key_t witness_key; /* cache_lck.c */ @@ -752,91 +671,28 @@ void Lck_DestroyClass(struct vsc_seg **); #define LOCK(nam) extern struct VSC_lck *lck_##nam; #include "tbl/locks.h" -/* cache_mempool.c */ -void MPL_AssertSane(const void *item); -struct mempool * MPL_New(const char *name, volatile struct poolparam *pp, - volatile unsigned *cur_size); -void MPL_Destroy(struct mempool **mpp); -void *MPL_Get(struct mempool *mpl, unsigned *size); -void MPL_Free(struct mempool *mpl, void *item); - /* cache_obj.c */ -struct objcore * ObjNew(const struct worker *); -void ObjDestroy(const struct worker *, struct objcore **); -typedef int objiterate_f(void *priv, int flush, const void *ptr, ssize_t len); -int ObjIterate(struct worker *, struct objcore *, - void *priv, objiterate_f *func, int final); -int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); -void ObjExtend(struct worker *, struct objcore *, ssize_t l); -uint64_t ObjWaitExtend(const struct worker *, const struct objcore *, - uint64_t l); -void ObjSetState(struct worker *, const struct objcore *, - enum boc_state_e next); -void ObjWaitState(const struct objcore *, enum boc_state_e want); -void ObjTrimStore(struct worker *, struct objcore *); -void ObjTouch(struct worker *, struct objcore *, double now); -unsigned ObjGetXID(struct worker *, struct objcore *); -uint64_t ObjGetLen(struct worker *, struct objcore *); -void ObjFreeObj(struct worker *, struct objcore *); -void ObjSlim(struct worker *, struct objcore *); + int ObjHasAttr(struct worker *, struct objcore *, enum obj_attr); const void *ObjGetAttr(struct worker *, struct objcore *, enum obj_attr, ssize_t *len); -void *ObjSetAttr(struct worker *, struct objcore *, enum obj_attr, - ssize_t len, const void *); -int ObjCopyAttr(struct worker *, struct objcore *, struct objcore *, - enum obj_attr attr); -void ObjBocDone(struct worker *, struct objcore *, struct boc **); -int ObjSetDouble(struct worker *, struct objcore *, enum obj_attr, double); -int ObjSetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t); -int ObjSetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t); +typedef int objiterate_f(void *priv, int flush, const void *ptr, ssize_t len); +int ObjIterate(struct worker *, struct objcore *, + void *priv, objiterate_f *func, int final); + +unsigned ObjGetXID(struct worker *, struct objcore *); +uint64_t ObjGetLen(struct worker *, struct objcore *); int ObjGetDouble(struct worker *, struct objcore *, enum obj_attr, double *); int ObjGetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t *); int ObjGetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t *); - int ObjCheckFlag(struct worker *, struct objcore *, enum obj_flags of); -void ObjSetFlag(struct worker *, struct objcore *, enum obj_flags of, int val); - -#define OEV_INSERT (1U<<1) -#define OEV_BANCHG (1U<<2) -#define OEV_TTLCHG (1U<<3) -#define OEV_EXPIRE (1U<<4) - -#define OEV_MASK (OEV_INSERT|OEV_BANCHG|OEV_TTLCHG|OEV_EXPIRE) - -typedef void obj_event_f(struct worker *, void *priv, struct objcore *, - unsigned); - -uintptr_t ObjSubscribeEvents(obj_event_f *, void *, unsigned mask); -void ObjUnsubscribeEvents(uintptr_t *); -void ObjSendEvent(struct worker *, struct objcore *oc, unsigned event); - -/* cache_panic.c */ -const char *body_status_2str(enum body_status e); -const char *sess_close_2str(enum sess_close sc, int want_desc); - -/* cache_req_body.c */ -int VRB_Ignore(struct req *); -ssize_t VRB_Cache(struct req *, ssize_t maxsize); -ssize_t VRB_Iterate(struct req *, objiterate_f *func, void *priv); -void VRB_Free(struct req *); - /* cache_session.c [SES] */ -void HTC_RxInit(struct http_conn *htc, struct ws *ws); -void HTC_RxPipeline(struct http_conn *htc, void *); -enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, - double *t1, double *t2, double ti, double tn, int maxbytes); - #define SESS_ATTR(UP, low, typ, len) \ - int SES_Set_##low(const struct sess *sp, const typ *src); \ - int SES_Get_##low(const struct sess *sp, typ **dst); \ - void SES_Reserve_##low(struct sess *sp, typ **dst); + int SES_Get_##low(const struct sess *sp, typ **dst); #include "tbl/sess_attr.h" - -void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a); /* cache_shmlog.c */ diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 4ced78a..a6b9ee0 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -95,8 +95,6 @@ Req_New(const struct worker *wrk, struct sess *sp) req->sp = sp; req->top = req; // esi overrides - INIT_OBJ(req->htc, HTTP_CONN_MAGIC); - e = (char*)req + sz; p = (char*)(req + 1); p = (void*)PRNDUP(p); @@ -129,11 +127,18 @@ Req_New(const struct worker *wrk, struct sess *sp) INIT_OBJ(req->vfc, VFP_CTX_MAGIC); p = (void*)PRNDUP(p + sizeof(*req->vfc)); + req->htc = (void*)p; + p = (void*)PRNDUP(p + sizeof(*req->htc)); + req->vdc = (void*)p; INIT_OBJ(req->vdc, VDP_CTX_MAGIC); VTAILQ_INIT(&req->vdc->vdp); p = (void*)PRNDUP(p + sizeof(*req->vdc)); + req->htc = (void*)p; + INIT_OBJ(req->htc, HTTP_CONN_MAGIC); + p = (void*)PRNDUP(p + sizeof(*req->htc)); + assert(p < e); WS_Init(req->ws, "req", p, e - p); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index bad6380..31a0694 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -46,6 +46,42 @@ /* -------------------------------------------------------------------*/ struct vfp; +struct cli_proto; +struct poolparam; + +/*-------------------------------------------------------------------- + * HTTP Protocol connection structure + * + * This is the protocol independent object for a HTTP connection, used + * both for backend and client sides. + * + */ + +struct http_conn { + unsigned magic; +#define HTTP_CONN_MAGIC 0x3e19edd1 + + int *rfd; + enum sess_close doclose; + enum body_status body_status; + struct ws *ws; + char *rxbuf_b; + char *rxbuf_e; + char *pipeline_b; + char *pipeline_e; + ssize_t content_length; + void *priv; + + /* Timeouts */ + double first_byte_timeout; + double between_bytes_timeout; +}; + +typedef enum htc_status_e htc_complete_f(struct http_conn *); + +/* -------------------------------------------------------------------*/ + +extern volatile struct params * cache_param; /* Prototypes etc ----------------------------------------------------*/ @@ -94,6 +130,39 @@ void EXP_Remove(struct objcore *); #define EXP_Dttl(req, oc) (oc->ttl - (req->t_req - oc->t_origin)) +/* cache_expire.c */ + +/* + * The set of variables which control object expiry are inconveniently + * 24 bytes long (double+3*float) and this causes alignment waste if + * we put then in a struct. + * These three macros operate on the struct we don't use. + */ + +#define EXP_ZERO(xx) \ + do { \ + (xx)->t_origin = 0.0; \ + (xx)->ttl = 0.0; \ + (xx)->grace = 0.0; \ + (xx)->keep = 0.0; \ + } while (0) + +#define EXP_COPY(to,fm) \ + do { \ + (to)->t_origin = (fm)->t_origin; \ + (to)->ttl = (fm)->ttl; \ + (to)->grace = (fm)->grace; \ + (to)->keep = (fm)->keep; \ + } while (0) + +#define EXP_WHEN(to) \ + ((to)->t_origin + (to)->ttl + (to)->grace + (to)->keep) + +/* cache_exp.c */ +void EXP_Rearm(struct objcore *, double now, double ttl, double grace, + double keep); + + /* From cache_main.c */ void BAN_Init(void); void BAN_Compile(void); @@ -122,6 +191,15 @@ extern const struct vdp VDP_esi; /* cache_expire.c */ void EXP_Init(void); +/* cache_fetch.c */ +enum vbf_fetch_mode_e { + VBF_NORMAL = 0, + VBF_PASS = 1, + VBF_BACKGROUND = 2, +}; +void VBF_Fetch(struct worker *wrk, struct req *req, + struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e); + /* cache_fetch_proc.c */ void VFP_Init(void); enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); @@ -140,6 +218,14 @@ extern const struct vfp VFP_esi_gzip; /* cache_http.c */ void HTTP_Init(void); +/* cache_http1_proto.c */ + +htc_complete_f HTTP1_Complete; +uint16_t HTTP1_DissectRequest(struct http_conn *, struct http *); +uint16_t HTTP1_DissectResponse(struct http_conn *, struct http *resp, + const struct http *req); +unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); + /* cache_main.c */ void THR_SetName(const char *name); const char* THR_GetName(void); @@ -152,12 +238,63 @@ void THR_Init(void); /* cache_lck.c */ void LCK_Init(void); +/* cache_mempool.c */ +void MPL_AssertSane(const void *item); +struct mempool * MPL_New(const char *name, volatile struct poolparam *pp, + volatile unsigned *cur_size); +void MPL_Destroy(struct mempool **mpp); +void *MPL_Get(struct mempool *mpl, unsigned *size); +void MPL_Free(struct mempool *mpl, void *item); + /* cache_obj.c */ void ObjInit(void); +struct objcore * ObjNew(const struct worker *); +void ObjDestroy(const struct worker *, struct objcore **); +int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); +void ObjExtend(struct worker *, struct objcore *, ssize_t l); +uint64_t ObjWaitExtend(const struct worker *, const struct objcore *, + uint64_t l); +void ObjSetState(struct worker *, const struct objcore *, + enum boc_state_e next); +void ObjWaitState(const struct objcore *, enum boc_state_e want); +void ObjTrimStore(struct worker *, struct objcore *); +void ObjTouch(struct worker *, struct objcore *, double now); +void ObjFreeObj(struct worker *, struct objcore *); +void ObjSlim(struct worker *, struct objcore *); +void *ObjSetAttr(struct worker *, struct objcore *, enum obj_attr, + ssize_t len, const void *); +int ObjCopyAttr(struct worker *, struct objcore *, struct objcore *, + enum obj_attr attr); +void ObjBocDone(struct worker *, struct objcore *, struct boc **); + +int ObjSetDouble(struct worker *, struct objcore *, enum obj_attr, double); +int ObjSetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t); +int ObjSetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t); + +void ObjSetFlag(struct worker *, struct objcore *, enum obj_flags of, int val); + +void ObjSendEvent(struct worker *, struct objcore *oc, unsigned event); + +#define OEV_INSERT (1U<<1) +#define OEV_BANCHG (1U<<2) +#define OEV_TTLCHG (1U<<3) +#define OEV_EXPIRE (1U<<4) + +#define OEV_MASK (OEV_INSERT|OEV_BANCHG|OEV_TTLCHG|OEV_EXPIRE) + +typedef void obj_event_f(struct worker *, void *priv, struct objcore *, + unsigned); + +uintptr_t ObjSubscribeEvents(obj_event_f *, void *, unsigned mask); +void ObjUnsubscribeEvents(uintptr_t *); + + /* cache_panic.c */ void PAN_Init(void); int PAN_already(struct vsb *, const void *); +const char *body_status_2str(enum body_status e); +const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ void Pool_Init(void); @@ -179,6 +316,12 @@ void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); void Req_Fail(struct req *req, enum sess_close reason); void Req_AcctLogCharge(struct VSC_main *, struct req *); +/* cache_req_body.c */ +int VRB_Ignore(struct req *); +ssize_t VRB_Cache(struct req *, ssize_t maxsize); +ssize_t VRB_Iterate(struct req *, objiterate_f *func, void *priv); +void VRB_Free(struct req *); + /* cache_req_fsm.c [CNT] */ enum req_fsm_nxt { @@ -197,6 +340,18 @@ void SES_Ref(struct sess *sp); void SES_Rel(struct sess *sp); int SES_Reschedule_Req(struct req *, enum task_prio); +void HTC_RxInit(struct http_conn *htc, struct ws *ws); +void HTC_RxPipeline(struct http_conn *htc, void *); +enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, + double *t1, double *t2, double ti, double tn, int maxbytes); + +#define SESS_ATTR(UP, low, typ, len) \ + int SES_Set_##low(const struct sess *sp, const typ *src); \ + void SES_Reserve_##low(struct sess *sp, typ **dst); +#include "tbl/sess_attr.h" +void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); + + enum htc_status_e { HTC_S_JUNK = -5, HTC_S_CLOSE = -4, @@ -289,4 +444,3 @@ void SMP_Ready(void); if (DO_DEBUG(debug_bit)) \ VSL(SLT_Debug, (id), __VA_ARGS__); \ } while (0) - diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index 7dd0c62..a6558a2 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -50,7 +50,7 @@ struct v1p_acct { uint64_t out; }; -void V1P_Process(struct req *, int fd, struct v1p_acct *); +void V1P_Process(const struct req *, int fd, struct v1p_acct *); void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *); /* cache_http1_line.c */ diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index cb7de49..6f8828a 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -84,7 +84,7 @@ V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_vbe *b) } void -V1P_Process(struct req *req, int fd, struct v1p_acct *v1a) +V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a) { struct pollfd fds[2]; int i, j; diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index d165281..a0f76fe 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -46,7 +46,7 @@ static const char vpx1_sig[] = {'P', 'R', 'O', 'X', 'Y'}; static int -vpx_proto1(const struct worker *wrk, struct req *req) +vpx_proto1(const struct worker *wrk, const struct req *req) { const char *fld[5]; int i; diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 3ee5a0e..4de4947 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -35,16 +35,13 @@ #include #include -#include "cache/cache.h" +#include "cache/cache_varnishd.h" #include "vsa.h" -#include "vsb.h" #include "vtim.h" #include "vcc_if.h" #include "VSC_debug.h" -#include "common/common_param.h" - struct priv_vcl { unsigned magic; #define PRIV_VCL_MAGIC 0x8E62FA9D From phk at FreeBSD.org Mon Mar 12 10:23:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 10:23:08 +0000 (UTC) Subject: [master] 63d19da Improve panic message in case #2592 happens again Message-ID: <20180312102308.5E6E4B4928@lists.varnish-cache.org> commit 63d19da46d2ed910647a65638e306335f35a6ea0 Author: Poul-Henning Kamp Date: Mon Mar 12 10:20:40 2018 +0000 Improve panic message in case #2592 happens again diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9490129..582f70b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -818,6 +818,6 @@ Tlen(const txt t) do { \ if ((ptr)->magic != (exp)) \ VSB_printf((vsb), \ - "MAGIC 0x%08x (Should:%s/0x%08x)\n", \ - (ptr)->magic, #exp, exp); \ + "MAGIC at %p is 0x%08x (Should be: %s/0x%08x)\n", \ + ptr, (ptr)->magic, #exp, exp); \ } while(0) diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 4c194bb..dea26a6 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -118,7 +118,7 @@ sess_close_2str(enum sess_close sc, int want_desc) /*--------------------------------------------------------------------*/ -#define N_ALREADY 64 +#define N_ALREADY 256 static const void *already_list[N_ALREADY]; static int already_idx; diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index 26bb42f..9e80eb6 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -48,6 +48,7 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, "streams {\n"); VSB_indent(vsb, 2); VTAILQ_FOREACH(r2, &h2->streams, list) { + PAN_CheckMagic(vsb, r2, H2_REQ_MAGIC); VSB_printf(vsb, "0x%08x", r2->stream); switch (r2->state) { #define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, " %-6s", sd); break; From geoff at uplex.de Mon Mar 12 10:38:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 10:38:08 +0000 (UTC) Subject: [master] 94443fe Add VMOD unix. Message-ID: <20180312103808.7FC8AB4EB2@lists.varnish-cache.org> commit 94443fed07dc76d92afa868925b68fa5c1eb3f73 Author: Geoff Simmons Date: Mon Mar 12 11:32:35 2018 +0100 Add VMOD unix. diff --git a/bin/varnishtest/tests/m00047.vtc b/bin/varnishtest/tests/m00047.vtc new file mode 100644 index 0000000..bfc8f7c --- /dev/null +++ b/bin/varnishtest/tests/m00047.vtc @@ -0,0 +1,130 @@ +varnishtest "VMOD unix" + +# This test requires some manual verification, by checking the log, +# because support for peer credentials via UDS varies by platform, see +# below. + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import unix; + + sub vcl_backend_response { + set beresp.http.b-uid = unix.uid(); + set beresp.http.b-gid = unix.gid(); + set beresp.http.b-user = unix.user(); + set beresp.http.b-group = unix.group(); + } + + sub vcl_deliver { + set resp.http.c-uid = unix.uid(); + set resp.http.c-gid = unix.gid(); + set resp.http.c-user = unix.user(); + set resp.http.c-group = unix.group(); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.b-uid == -1 + expect resp.http.b-gid == -1 + expect resp.http.b-user == "" + expect resp.http.b-group == "" + expect resp.http.c-uid == -1 + expect resp.http.c-gid == -1 + expect resp.http.c-user == "" + expect resp.http.c-group == "" +} -run + +logexpect l1 -v v1 -d 1 -b { + expect * 1002 Begin bereq + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} +} -run + +logexpect l1 -v v1 -d 1 -c { + expect * 1001 Begin req + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} +} -run + + +varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} { + import unix; + import std; + backend b { .host = "${bad_ip}"; } + + sub vcl_init { + std.log(unix.uid()); + } +} + +varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} { + import unix; + import std; + backend b { .host = "${bad_ip}"; } + + sub vcl_init { + std.log(unix.gid()); + } +} + +varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} { + import unix; + import std; + backend b { .host = "${bad_ip}"; } + + sub vcl_init { + std.log(unix.user()); + } +} + +varnish v1 -errvcl {vmod unix failure: may not be called in vcl_init or vcl_fini} { + import unix; + import std; + backend b { .host = "${bad_ip}"; } + + sub vcl_init { + std.log(unix.group()); + } +} + +varnish v1 -stop +server s1 -wait +server s1 -start + +varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend { + import unix; + + sub vcl_backend_response { + set beresp.http.b-uid = unix.uid(); + set beresp.http.b-gid = unix.gid(); + set beresp.http.b-user = unix.user(); + set beresp.http.b-group = unix.group(); + } + + sub vcl_deliver { + set resp.http.c-uid = unix.uid(); + set resp.http.c-gid = unix.gid(); + set resp.http.c-user = unix.user(); + set resp.http.c-group = unix.group(); + } +} -start + +# Check the log output for the response header values to see how this +# worked on your platform. + +client c2 -connect "${v2_addr}" { + txreq + rxresp + expect resp.status ~ "^(200|503)$" +} -run diff --git a/bin/varnishtest/vmods.h b/bin/varnishtest/vmods.h index e732468..fb8f68f 100644 --- a/bin/varnishtest/vmods.h +++ b/bin/varnishtest/vmods.h @@ -33,3 +33,4 @@ VTC_VMOD(directors) VTC_VMOD(purge) VTC_VMOD(vtc) VTC_VMOD(blob) +VTC_VMOD(unix) diff --git a/configure.ac b/configure.ac index cf14412..c0dc0f7 100644 --- a/configure.ac +++ b/configure.ac @@ -216,6 +216,8 @@ AC_CHECK_FUNCS([setppriv]) AC_CHECK_FUNCS([fallocate]) AC_CHECK_FUNCS([closefrom]) AC_CHECK_FUNCS([sigaltstack]) +AC_CHECK_FUNCS([getpeereid]) +AC_CHECK_FUNCS([getpeerucred]) save_LIBS="${LIBS}" LIBS="${PTHREAD_LIBS}" @@ -758,6 +760,7 @@ AC_CONFIG_FILES([ lib/libvmod_purge/Makefile lib/libvmod_vtc/Makefile lib/libvmod_blob/Makefile + lib/libvmod_unix/Makefile man/Makefile varnishapi.pc varnishapi-uninstalled.pc diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 490ee50..810e903 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -218,6 +218,10 @@ reference/vmod_blob.generated.rst: reference $(top_builddir)/lib/libvmod_blob/vm cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@ || true BUILT_SOURCES += reference/vmod_blob.generated.rst +reference/vmod_unix.generated.rst: reference $(top_builddir)/lib/libvmod_unix/vmod_unix.rst + cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ || true +BUILT_SOURCES += reference/vmod_unix.generated.rst + EXTRA_DIST += $(BUILT_SOURCES) MAINTAINERCLEANFILES = $(EXTRA_DIST) diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 39ccb81..9df60eb 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -25,6 +25,7 @@ The Varnish Reference Manual vmod_vtc.generated.rst vmod_purge.generated.rst vmod_blob.generated.rst + vmod_unix.generated.rst directors.rst varnish-counters.rst vsl.rst diff --git a/lib/Makefile.am b/lib/Makefile.am index 4afd3d0..411bd80 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -10,4 +10,5 @@ SUBDIRS = \ libvmod_directors \ libvmod_purge \ libvmod_vtc \ - libvmod_blob + libvmod_blob \ + libvmod_unix diff --git a/lib/libvmod_unix/Makefile.am b/lib/libvmod_unix/Makefile.am new file mode 100644 index 0000000..8dafeba --- /dev/null +++ b/lib/libvmod_unix/Makefile.am @@ -0,0 +1,8 @@ +# + +libvmod_unix_la_SOURCES = \ + vmod_unix.c \ + cred_compat.h + +# Use vmodtool.py generated automake boilerplate +include $(srcdir)/automake_boilerplate.am diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate.am new file mode 100644 index 0000000..b71674a --- /dev/null +++ b/lib/libvmod_unix/automake_boilerplate.am @@ -0,0 +1,39 @@ + +# Boilerplate generated by vmodtool.py - changes will be overwritten + +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmoddir = $(pkglibdir)/vmods +vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py +vmodtoolargs = --strict --boilerplate + +vmod_LTLIBRARIES = libvmod_unix.la + +libvmod_unix_la_CFLAGS = \ + @SAN_CFLAGS@ + +libvmod_unix_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + $(VMOD_LDFLAGS) \ + @SAN_LDFLAGS@ + +nodist_libvmod_unix_la_SOURCES = vcc_if.c vcc_if.h + +$(libvmod_unix_la_OBJECTS): vcc_if.h + +vcc_if.h vmod_unix.rst vmod_unix.man.rst: vcc_if.c + +vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc + +EXTRA_DIST = vmod.vcc automake_boilerplate.am + +CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \ + $(builddir)/vmod_unix.rst \ + $(builddir)/vmod_unix.man.rst + diff --git a/lib/libvmod_unix/cred_compat.h b/lib/libvmod_unix/cred_compat.h new file mode 100644 index 0000000..5c2340b --- /dev/null +++ b/lib/libvmod_unix/cred_compat.h @@ -0,0 +1,87 @@ +/*- + * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Geoffrey Simmons + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include +#include + +#if defined(HAVE_GETPEEREID) +#include +#endif + +#if defined(HAVE_GETPEERUCRED) +#include +#endif + +#define CREDS_FAIL -1 +#define NOT_SUPPORTED -2 + +static int +get_ids(int fd, uid_t *uid, gid_t *gid) +{ + +#if defined(SO_PEERCRED) + + struct ucred ucred; + socklen_t l = sizeof(ucred); + + errno = 0; + if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, (void *) &ucred, &l) != 0) + return (CREDS_FAIL); + *uid = ucred.uid; + *gid = ucred.gid; + return (0); + +#elif defined(HAVE_GETPEEREID) + + errno = 0; + if (getpeereid(fd, uid, gid) != 0) + return (CREDS_FAIL); + return (0); + +#elif defined(HAVE_GETPEERUCRED) + + ucred_t *ucred; + + errno = 0; + if (getpeerucred(fd, &ucred) != 0) + return (CREDS_FAIL); + *uid = ucred_geteuid(ucred); + *gid = ucred_getegid(ucred); + ucred_free(ucred); + return (0); + +#else + (void) fd; + (void) uid; + (void) gid; + return (NOT_SUPPORTED); +#endif + +} diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc new file mode 100644 index 0000000..1035df3 --- /dev/null +++ b/lib/libvmod_unix/vmod.vcc @@ -0,0 +1,110 @@ +#- +# This document is licensed under the same conditions as Varnish itself. +# See LICENSE for details. +# +# Authors: Geoffrey Simmons +# + +$Module unix 3 utilities for Unix domain sockets + +$ABI strict + +DESCRIPTION +=========== + +This VMOD provides information about the credentials of the peer +process (user and group of the process owner) that is connected to +Varnish via a Unix domain socket, if the platform supports it. + +Examples:: + + sub vcl_recv { + # Return "403 Forbidden" if the connected peer is + # not running as the user "trusteduser". + if (unix.user() != "trusteduser") { + return( synth(403) ); + } + + # Require the connected peer to run in the group + # "trustedgroup". + if (unix.group() != "trustedgroup") { + return( synth(403) ); + } + + # Require the connected peer to run under a specific numeric + # user id. + if (unix.uid() != 4711) { + return( synth(403) ); + } + + # Require the connected peer to run under a numeric group id. + if (unix.gid() != 815) { + return( synth(403) ); + } + } + +Obtaining the peer credentials is possible on a platform that supports +one of the following: + +* ``getpeereid(3)`` (such as FreeBSD and other BSD-derived systems) + +* ``getpeerucred(3)`` (SunOS and descendants) + + * Reading peer credentials on such a system may require that the + Varnish child process runs with certain privileges (such as + ``PRIV_PROC_INFO``) + +* the socket option ``SO_PEERCRED`` for ``getsockopt(2)`` (Linux) + +On most platforms, the value returned is the effective user or group +that was valid when the peer process initiated the connection. + +$Function STRING user() + +Return the user name of the peer process owner. + +$Function STRING group() + +Return the group name of the peer process owner. + +$Function INT uid() + +Return the numeric user id of the peer process owner. + +$Function INT gid() + +Return the numeric group id of the peer process owner. + +ERRORS +====== + +All functions in this VMOD are subject to the following constraints: + +* None of them may be called in ``vcl_init`` or ``vcl_fini``. If one + of them is called in ``vcl_init``, then the VCL program will fail to + load, with an error message from the VMOD. + +* If called on a platform that is not supported, then VCL failure is + invoked. An error message is written to the log (with the + ``VCL_Error`` tag), and for all VCL subroutines except for + ``vcl_synth``, control is directed immediately to ``vcl_synth``, + with the response status set to 503 and the reason string set to + "VCL failed". + + If the failure occurs during ``vcl_synth``, then ``vcl_synth`` is + aborted, and the the response line "503 VCL failed" is sent. + +* If the current listener is not a Unix domain socket, or if the + attempt to read credentials fails, then a ``VCL_Error`` message is + written to the log. The STRING functions (``vmod_user`` and + ``vmod_group``) return NULL, while the INT functions (``vmod_user`` + and ``vmod_gid``) return -1. + +SEE ALSO +======== + +* :ref:`varnishd(1)` +* :ref:`vcl(7)` +* ``getpeereid(3)`` +* ``getpeerucred(3)`` +* ``getsockopt(2)`` diff --git a/lib/libvmod_unix/vmod_unix.c b/lib/libvmod_unix/vmod_unix.c new file mode 100644 index 0000000..ee1f3e0 --- /dev/null +++ b/lib/libvmod_unix/vmod_unix.c @@ -0,0 +1,142 @@ +/*- + * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Geoffrey Simmons + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "cred_compat.h" + +#include +#include +#include + +#include "cache/cache.h" +#include "vcl.h" +#include "common/heritage.h" + +#include "vcc_if.h" + +#define FAIL(ctx, msg) \ + VRT_fail((ctx), "vmod unix failure: " msg) + +#define ERR(ctx, msg) \ + VSLb((ctx)->vsl, SLT_VCL_Error, "vmod unix error: " msg) + +#define VERR(ctx, fmt, ...) \ + VSLb((ctx)->vsl, SLT_VCL_Error, "vmod unix error: " fmt, __VA_ARGS__) + +#define FAILNOINIT(ctx) \ + FAIL((ctx), "may not be called in vcl_init or vcl_fini") + +#define ERRNOTUDS(ctx) \ + ERR((ctx), "not listening on a Unix domain socket") + +#define FAIL_SUPPORT(ctx) \ + FAIL((ctx), "not supported on this platform") + +#define ERRNOCREDS(ctx) \ + VERR((ctx), "could not read peer credentials: %s", strerror(errno)) + +#define ERRNOMEM(ctx) \ + ERR((ctx), "out of space") + +static struct sess * +get_sp(VRT_CTX) +{ + struct sess *sp; + + if (VALID_OBJ(ctx->req, REQ_MAGIC)) + sp = ctx->req->sp; + else { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + sp = ctx->bo->sp; + } + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->listen_sock, LISTEN_SOCK_MAGIC); + return (sp); +} + +#define NUM_FUNC(func) \ +VCL_INT \ +vmod_##func(VRT_CTX) \ +{ \ + struct sess *sp; \ + uid_t uid; \ + gid_t gid; \ + int ret; \ + \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + if ((ctx->method & VCL_MET_TASK_H) != 0) { \ + FAILNOINIT(ctx); \ + return (-1); \ + } \ + \ + sp = get_sp(ctx); \ + if (! sp->listen_sock->uds) { \ + ERRNOTUDS(ctx); \ + return (-1); \ + } \ + \ + ret = get_ids(sp->fd, &uid, &gid); \ + if (ret == 0) \ + return (func); \ + \ + if (ret == NOT_SUPPORTED) \ + FAIL_SUPPORT(ctx); \ + else if (ret == CREDS_FAIL) \ + ERRNOCREDS(ctx); \ + return (-1); \ +} + +NUM_FUNC(uid) +NUM_FUNC(gid) + +#define NAME_FUNC(func, type, get, id, fld) \ +VCL_STRING \ +vmod_##func(VRT_CTX) \ +{ \ + struct type *s; \ + id##_t i; \ + VCL_STRING name; \ + \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + i = (id##_t) vmod_##id(ctx); \ + if (i == -1) \ + return (NULL); \ + \ + errno = 0; \ + s = get(i); \ + if (s == NULL) { \ + ERRNOCREDS(ctx); \ + return (NULL); \ + } \ + if ((name = WS_Copy(ctx->ws, s->fld, -1)) == NULL) { \ + ERRNOMEM(ctx); \ + return (NULL); \ + } \ + return (name); \ +} + +NAME_FUNC(user, passwd, getpwuid, uid, pw_name) +NAME_FUNC(group, group, getgrgid, gid, gr_name) diff --git a/man/Makefile.am b/man/Makefile.am index 2187ff9..e7c00df 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -19,7 +19,8 @@ dist_man_MANS = \ vmod_purge.3 \ vmod_std.3 \ vmod_vtc.3 \ - vmod_blob.3 + vmod_blob.3 \ + vmod_unix.3 CLEANFILES = $(dist_man_MANS) @@ -101,4 +102,7 @@ vmod_vtc.3: $(top_builddir)/lib/libvmod_vtc/vmod_vtc.man.rst vmod_blob.3: $(top_builddir)/lib/libvmod_blob/vmod_blob.man.rst ${RST2MAN} $(RST2ANY_FLAGS) $? $@ +vmod_unix.3: $(top_builddir)/lib/libvmod_unix/vmod_unix.man.rst + ${RST2MAN} $(RST2ANY_FLAGS) $? $@ + .NOPATH: $(dist_man_MANS) From phk at FreeBSD.org Mon Mar 12 10:42:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 10:42:09 +0000 (UTC) Subject: [master] 2168fc6 Handle that varnishd is built after in-tree vmods Message-ID: <20180312104209.1843AB5007@lists.varnish-cache.org> commit 2168fc606f7d3c2e976c65f606502699bf4fce0e Author: Poul-Henning Kamp Date: Mon Mar 12 10:41:23 2018 +0000 Handle that varnishd is built after in-tree vmods diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index a135f80..452d044 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -154,6 +154,7 @@ vcldir=$(datarootdir)/$(PACKAGE)/vcl varnishd_CFLAGS = \ @PCRE_CFLAGS@ \ @SAN_CFLAGS@ \ + -DNOT_IN_A_VMOD \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ -DVARNISH_VMOD_DIR='"${pkglibdir}/vmods"' \ -DVARNISH_VCL_DIR='"${pkgsysconfdir}:${vcldir}"' diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 31a0694..e05e87c 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -41,7 +41,9 @@ #include "common/common_param.h" -#include "VSC_main.h" +#ifdef NOT_IN_A_VMOD +# include "VSC_main.h" +#endif /* -------------------------------------------------------------------*/ From daghf at varnish-software.com Mon Mar 12 10:53:07 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 12 Mar 2018 10:53:07 +0000 (UTC) Subject: [master] d5208c1 Handle failed write(2) in h2_ou_session Message-ID: <20180312105307.26DA1B5398@lists.varnish-cache.org> commit d5208c185723dc8647a5a93682a2792afcdbc910 Author: Dag Haavi Finstad Date: Mon Mar 12 11:51:10 2018 +0100 Handle failed write(2) in h2_ou_session Fixes: #2607 diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 917f880..29ba010 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -31,6 +31,7 @@ #include "cache/cache_varnishd.h" +#include #include #include "cache/cache_transport.h" @@ -235,6 +236,18 @@ h2_b64url_settings(struct h2_sess *h2, struct req *req) /**********************************************************************/ static int +h2_ou_rel(struct worker *wrk, struct req *req) +{ + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN (req->vcl); + VCL_Rel(&req->vcl); + Req_AcctLogCharge(wrk->stats, req); + Req_Release(req); + return (0); +} + +static int h2_ou_session(struct worker *wrk, struct h2_sess *h2, struct req *req) { @@ -244,15 +257,15 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2, if (h2_b64url_settings(h2, req)) { VSLb(h2->vsl, SLT_Debug, "H2: Bad HTTP-Settings"); - AN (req->vcl); - VCL_Rel(&req->vcl); - Req_AcctLogCharge(wrk->stats, req); - Req_Release(req); - return (0); + return (h2_ou_rel(wrk, req)); } sz = write(h2->sess->fd, h2_resp_101, strlen(h2_resp_101)); - assert(sz == strlen(h2_resp_101)); + if (sz != strlen(h2_resp_101)) { + VSLb(h2->vsl, SLT_Debug, "H2: Upgrade: Error writing 101" + " response: %s\n", strerror(errno)); + return (h2_ou_rel(wrk, req)); + } http_Unset(req->http, H_Upgrade); http_Unset(req->http, H_HTTP2_Settings); From phk at FreeBSD.org Mon Mar 12 11:53:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 11:53:08 +0000 (UTC) Subject: [master] ba83ba5 Bump the VRT version for release Message-ID: <20180312115308.C2A2DB661E@lists.varnish-cache.org> commit ba83ba549509fea3b8e167406484d11de72f0e41 Author: Poul-Henning Kamp Date: Mon Mar 12 11:52:09 2018 +0000 Bump the VRT version for release diff --git a/include/vrt.h b/include/vrt.h index 5a25e6b..7939459 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,7 +52,10 @@ * binary/load-time compatible, increment MAJOR version * * - * 6.2 (scheduled for: 2018-03-15) + * 7.0 (2018-03-15) + * lots of stuff moved from cache.h to cache_varnishd.h + * (ie: from "$Abi vrt" to "$Abi strict") + * VCL_INT and VCL_BYTES are always 64 bits. * path field added to struct vrt_backend * VRT_Healthy() added * VRT_VSC_Alloc() added @@ -95,9 +98,9 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 6U +#define VRT_MAJOR_VERSION 7U -#define VRT_MINOR_VERSION 2U +#define VRT_MINOR_VERSION 0U /***********************************************************************/ From nils.goroll at uplex.de Mon Mar 12 12:30:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Mar 2018 12:30:12 +0000 (UTC) Subject: [master] 681dc94 fix optional argument handling in vmod_shard_remove_backend() Message-ID: <20180312123012.CEA57610C1@lists.varnish-cache.org> commit 681dc94b52be3cac41bc58dcd0759614955f361f Author: Nils Goroll Date: Mon Mar 12 13:28:02 2018 +0100 fix optional argument handling in vmod_shard_remove_backend() Thank you to phk for spotting this! diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 21a73eb..9b42e5a 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -318,7 +318,7 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, struct vmod_shard_remove_backend_arg *args) { VCL_BACKEND be = args->valid_backend ? args->backend : NULL; - VCL_STRING ident = args->ident ? args->ident : NULL; + VCL_STRING ident = args->valid_ident ? args->ident : NULL; CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); From geoff at uplex.de Mon Mar 12 12:44:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 12:44:09 +0000 (UTC) Subject: [master] 51344da VMOD unix: hold back the SunOS/getpeerucred() case for now. Message-ID: <20180312124409.C96F26153F@lists.varnish-cache.org> commit 51344da7baf647bc23eb2e0b00e57da774a11f48 Author: Geoff Simmons Date: Mon Mar 12 13:42:40 2018 +0100 VMOD unix: hold back the SunOS/getpeerucred() case for now. To be taken up again after 6.0. diff --git a/configure.ac b/configure.ac index c0dc0f7..4323d1b 100644 --- a/configure.ac +++ b/configure.ac @@ -217,7 +217,6 @@ AC_CHECK_FUNCS([fallocate]) AC_CHECK_FUNCS([closefrom]) AC_CHECK_FUNCS([sigaltstack]) AC_CHECK_FUNCS([getpeereid]) -AC_CHECK_FUNCS([getpeerucred]) save_LIBS="${LIBS}" LIBS="${PTHREAD_LIBS}" diff --git a/lib/libvmod_unix/cred_compat.h b/lib/libvmod_unix/cred_compat.h index 5c2340b..6a79636 100644 --- a/lib/libvmod_unix/cred_compat.h +++ b/lib/libvmod_unix/cred_compat.h @@ -65,18 +65,6 @@ get_ids(int fd, uid_t *uid, gid_t *gid) return (CREDS_FAIL); return (0); -#elif defined(HAVE_GETPEERUCRED) - - ucred_t *ucred; - - errno = 0; - if (getpeerucred(fd, &ucred) != 0) - return (CREDS_FAIL); - *uid = ucred_geteuid(ucred); - *gid = ucred_getegid(ucred); - ucred_free(ucred); - return (0); - #else (void) fd; (void) uid; diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index 1035df3..4e90bad 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -48,12 +48,6 @@ one of the following: * ``getpeereid(3)`` (such as FreeBSD and other BSD-derived systems) -* ``getpeerucred(3)`` (SunOS and descendants) - - * Reading peer credentials on such a system may require that the - Varnish child process runs with certain privileges (such as - ``PRIV_PROC_INFO``) - * the socket option ``SO_PEERCRED`` for ``getsockopt(2)`` (Linux) On most platforms, the value returned is the effective user or group @@ -106,5 +100,4 @@ SEE ALSO * :ref:`varnishd(1)` * :ref:`vcl(7)` * ``getpeereid(3)`` -* ``getpeerucred(3)`` * ``getsockopt(2)`` From hermunn at varnish-software.com Mon Mar 12 12:50:10 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Mar 2018 12:50:10 +0000 (UTC) Subject: [master] a92f9aa Document grace and keep, and how to deal with problematic servers Message-ID: <20180312125010.76F2061768@lists.varnish-cache.org> commit a92f9aa986bf43526316e385440412f7daba37d6 Author: P?l Hermunn Johansen Date: Mon Mar 12 13:48:00 2018 +0100 Document grace and keep, and how to deal with problematic servers This commit comes with the realization that finding a workaround for #1799 is even harder than I thought. Unfortunately we cannot close issue #1799 for the reasons described in the documentation. Thanks to @nigoroll for valuable feedback. diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst index 4b0a908..8db2a46 100644 --- a/doc/sphinx/users-guide/vcl-grace.rst +++ b/doc/sphinx/users-guide/vcl-grace.rst @@ -1,11 +1,15 @@ .. _users-guide-handling_misbehaving_servers: -Misbehaving servers +Grace mode and keep ------------------- -A key feature of Varnish is its ability to shield you from misbehaving -web- and application servers. +Sometimes you want Varnish to serve content that is somewhat stale +instead of waiting for a fresh object from the backend. For example, +if you run a news site, serving a main page that is a few seconds old +is not a problem if this gives your site faster load times. +In Varnish this is achieved by using `grace mode`. A related idea +is `keep`, which is also explained here. Grace mode ~~~~~~~~~~ @@ -19,52 +23,199 @@ If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to -wait. To deal with this we can instruct Varnish to keep -the objects in cache beyond their TTL and to serve the waiting -requests somewhat stale content. +wait. + +Setting an object's `grace` to a positive value tells Varnish that it +should serve the object to clients for some time after the TTL has +expired, while Varnish fetches a new version of the object. The default +value is controlled by the runtime parameter ``default_grace``. + +Keep +~~~~ + +Setting an object's `keep` tells Varnish that it should keep an object +in the cache for some additional time. There are two reasons to do this: -So, in order to serve stale content we must first have some content to -serve. So to make Varnish keep all objects for 2 minutes beyond their -TTL use the following VCL:: +* To use the object to construct a conditional GET backend request (with + If-Modified-Since: and/or ?f-None-Match: headers), allowing the backend + to reply with a 304 Not Modified response, which may be more efficient + on the backend and saves re-transmitting the unchanged body. +* To be able to serve the object when grace has expired but we have a + problem with getting a fresh object from the backend. This will require + a change in ``sub vcl_hit``, as described below. + +The values are additive, so if grace is 10 seconds and keep is 1 minute, +then objects will survive in cache for 70 seconds after the TTL has +expired. + +Setting grace and keep +~~~~~~~~~~~~~~~~~~~~~~ + +We can use VCL to make Varnish keep all objects for 10 minutes beyond +their TTL with a grace period of 2 minutes:: sub vcl_backend_response { - set beresp.grace = 2m; + set beresp.grace = 2m; + set beresp.keep = 8m; } -Now Varnish will be allowed to serve objects that are up to two -minutes out of date. When it does it will also schedule a refresh of -the object. This will happen asynchronously and the moment the new -object is in it will replace the one we've already got. +The effect of grace and keep +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For most users setting the default grace and/or a suitable grace for +each object is enough. The default VCL will do the right thing and +behave as described above. However, if you want to customize how varnish +behaves by changing ``sub vcl_hit``, then you should know some of the +details on how this works. + +When ``sub vcl_recv`` ends with ``return (lookup)`` (which is the +default behavior), Varnish will look for a matching object in its +cache. Then, if it only found an object whose TTL has run out, Varnish +will consider the following: + +* Is there already an ongoing backend request for the object? +* Is the object within the `grace period`? -You can influence how this logic works by adding code in vcl_hit. The -default looks like this:: +Then, Varnish reacts using the following rules: + +* If there is no backend request for the object, one is scheduled and + ``sub vcl_hit`` is called immediately. +* If there is a backend request going on, but the object is under grace, + ``sub vcl_hit`` is called immediately. +* If there is a backend request going on, but the grace has expired, + processing is halted until the backend request has finished and a + fresh object is available. + +Note that the backend fetch happens asynchronously, and the moment the +new object is in it will replace the one we've already got. + +If you do not define your own ``sub vcl_hit``, then the default one is +used. It looks like this:: sub vcl_hit { - if (obj.ttl >= 0s) { - // A pure unadulterated hit, deliver it - return (deliver); - } - if (obj.ttl + obj.grace > 0s) { - // Object is in grace, deliver it - // Automatically triggers a background fetch - return (deliver); - } - // fetch & deliver once we get the result - return (fetch); + if (obj.ttl >= 0s) { + // A pure unadulterated hit, deliver it + return (deliver); + } + if (obj.ttl + obj.grace > 0s) { + // Object is in grace, deliver it + // Automatically triggers a background fetch + return (deliver); + } + // fetch & deliver once we get the result + return (miss); } -The grace logic is pretty obvious here. If you have enabled -:ref:`users-guide-advanced_backend_servers-health` you can check if -the backend is sick and only serve graced object then. Replace the -second if-clause with something like this:: +If you follow the code, you see that Varnish delivers graced objects +while fetching fresh copies, but if grace has expired the clients have to +wait until a new copy is available. + +Misbehaving servers +~~~~~~~~~~~~~~~~~~~ + +A key feature of Varnish is its ability to shield you from misbehaving +web- and application servers. + +If you have enabled :ref:`users-guide-advanced_backend_servers-health` +you can check if the backend is sick and modify the behavior when it +comes to grace. There are essentially two ways of doing this. You can +explicitly deliver kept object (that is not within grace) when you see +that the backend is sick, or you can explicitly `not` serve an expired +object when you know that the backend is healthy. The two methods have +slightly different characteristics, as we shall see. - if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) { - return (deliver); - } else { - return (fetch); +In both cases we assume that you avoid inserting objects into the cache +when you get certain errors from the backend, for example by using the +following:: + + sub vcl_backend_response { + if (beresp.status == 503 && bereq.is_bgfetch) { + return (abandon); + } + } + +Method 1: When the backend is healthy, use a lower grace value +============================================================== + +Imagine that you have set an object's grace to a high value that you +wish to use when the backend is sick, for example:: + + sub vcl_backend_response { + set beresp.grace = 24h; + // no keep + } + +Then you can use the following code as your ``sub vcl_hit``:: + + if (std.healthy(req.backend_hint)) { + // change the behavior for health backends: Cap grace to 10s + if (obj.ttl + obj.grace > 0s && obj.ttl + 10s > 0s) { + return (deliver); + } else { + return (miss); + } } -So, to sum up, grace mode solves two problems: - * it serves stale content to avoid request pile-up. - * it serves stale content if you allow it. +The effect of this is that, when the backend is healthy, objects with +grace above 10 seconds will have an `effective` grace of 10 seconds. +When the backend is sick, the default VCL kicks in, and the long grace +is used. + +This method has one potentially serious problem when more than one +client asks for an object that has expired its TTL. If the second of +these requests arrives after the effective grace, but before the first +request has completed, then the second request will be turned into a +`pass`. + +In practice this method works well in most cases, but if you +experience excessive `pass` behavior, this translates to a reduced +hit rate and higher load on the backend. When this happens you will +see the error message `vcl_hit{} returns miss without busy object` in +the log. + +Method 2: When the backend is sick, deliver kept objects +======================================================== + +With this method, we assume that we have used `sub backend_response` +to set `beresp.grace` to a value that is suitable for healthy backends, +and with a `beresp.keep` that corresponds to the time we want to serve +the object when the backend is sick. For example:: + + sub vcl_backend_response { + set beresp.grace = 10s; + set beresp.keep = 24h; + } + +The appropriate code for ``vcl_hit`` then becomes:: + + if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace + obj.keep > 0s)) { + return (deliver); + } + +Typically you can omit the second part of the if test due to the +expiry thread deleting objects where `grace + keep` has expired. It is +possible that the `expiry thread` can be lagging slightly behind, but +for almost all practical purposes you are probably fine with the +following:: + + if (!std.healthy(req.backend_hint)) { + return (deliver); + } + +The problem with this solution concerns requests that are waiting for +a backend fetch to finish. If the backend fetch gets to ``return +(abandon)``, then all the requests that are waiting will get to ``sub +vcl_hit`` with an `error object` created by the error handling +code/VCL. In other words, you risk that some clients will get errors +instead of the more desirable stale objects. + +Summary +~~~~~~~ + +Grace mode allows Varnish to deliver slightly stale content to clients while +getting a fresh version from the backend. The result is faster load times +with a low cost. +It is possible to change the behavior when it comes to grace and keep, for +example by changing the `effective` grace depending on the health of the +backend, but you have to be careful. From geoff at uplex.de Mon Mar 12 13:22:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 13:22:09 +0000 (UTC) Subject: [master] 6ac0126 Document UDS backend addressing in vcl(7). Message-ID: <20180312132209.ABEE76216A@lists.varnish-cache.org> commit 6ac01261e022636ca30e0f947d84e669d96e9388 Author: Geoff Simmons Date: Mon Mar 12 14:20:40 2018 +0100 Document UDS backend addressing in vcl(7). diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 976a908..9275b57 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -212,16 +212,24 @@ backend. The actual declaration is in curly brackets, in a key/value fashion.:: .attribute = "value"; } -The only mandatory attribute is ``.host``. The attributes will inherit -their defaults from the global parameters. The following attributes -are available: +One of the attributes ``.host`` or ``.path`` is mandatory (but not +both). The attributes will inherit their defaults from the global +parameters. The following attributes are available: - ``.host`` *(mandatory)* + ``.host`` The host to be used. IP address or a hostname that resolves to a - single IP address. + single IP address. This attribute is mandatory, unless ``.path`` + is declared. + + ``.path`` + The absolute path of a Unix domain socket at which a backend is + listening. The file at that path must exist and must be accessible + to Varnish at VCL load time, and it must be a socket. One of + ``.path`` or ``.host`` must be declared (but not both). ``.port`` - The port on the backend that Varnish should connect to. + The port on the backend that Varnish should connect to. Ignored if + a Unix domain socket is declared in ``.path``. ``.host_header`` A host header to add to probes and regular backend requests if they have no From geoff at uplex.de Mon Mar 12 13:54:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 13:54:08 +0000 (UTC) Subject: [master] f982299 Document UDS backends in "Upgrading to 6.0". Message-ID: <20180312135408.DB9D562BB1@lists.varnish-cache.org> commit f98229940f3bdc70ea38c193f832d1963190d5e3 Author: Geoff Simmons Date: Mon Mar 12 14:53:21 2018 +0100 Document UDS backends in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 8c354a4..c840b92 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -52,6 +52,38 @@ that must be done outside of the Varnish configuration). If you continue using only IP addresses in your ``-a`` arguments, you won't have to change them. +Unix domain sockets as backend addresses +======================================== + +A backend declaration may now have the ``.path`` field to specify a +Unix domain socket to which Varnish connects:: + + backend my_uds_backend { + .path = "/path/to/backend.sock"; + } + +One of the fields ``.host`` or ``.path`` must be specified for a +backend (but not both). + +The value of ``.path`` must be an absolute path (beginning with +``/``), and the file at that path must exist and be accessible to +Varnish at VCL load time; and it must be a socket. + +The platform-specific restrictions on UDSen mentioned above apply of +course to backends as well; but in this case your deployment of the +peer component listening at the socket file must fulfill those +conditions, otherwise Varnish may not be able to connect to the +backend. + +The path of a socket file may also be specified in the +``varnishd -b`` command-line option (see varnishd +:ref:`ref-varnishd-options`):: + + $ varnishd -b /path/to/backend.sock + +The value of ``-b`` must fulfill the same conditions as the ``.path`` +field in a backend declaration. + varnishd parameters =================== From geoff at uplex.de Mon Mar 12 14:04:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 14:04:10 +0000 (UTC) Subject: [master] 8cf5d29 Document varnishd -b /path/to/uds in varnishd(1). Message-ID: <20180312140410.45DE462F39@lists.varnish-cache.org> commit 8cf5d29739fc69efc6f146b581e6664bf6ff5994 Author: Geoff Simmons Date: Mon Mar 12 15:02:38 2018 +0100 Document varnishd -b /path/to/uds in varnishd(1). diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index a70d2df..9651e1d 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -15,7 +15,7 @@ HTTP accelerator daemon SYNOPSIS ======== -varnishd [-a [name=][address][:port][,PROTO][,user=][,group=][,mode=]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter] +varnishd [-a [name=][address][:port][,PROTO][,user=][,group=][,mode=]] [-b [host[:port]|path]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter] varnishd [-x parameter|vsl|cli|builtin|optstring] @@ -61,11 +61,17 @@ Basic options Multiple listening addresses can be specified by using different -a arguments. --b +-b <[host[:port]|path]> Use the specified host as backend server. If port is not specified, - the default is 8080. -b can be used only once, and not together with - -f. + the default is 8080. + + If the value of ``-b`` begins with ``/``, it is interpreted as the + absolute path of a Unix domain socket to which Varnish connects. In + that case, the value of ``-b`` must satisfy the conditions required + for the ``.path`` field of a backend declaration, see :ref:`vcl(7)`. + + -b can be used only once, and not together with f. -f config From geoff at uplex.de Mon Mar 12 14:13:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 14:13:09 +0000 (UTC) Subject: [master] 93ae66a Document -b /path/to/uds in varnishd usage output. Message-ID: <20180312141309.36F206327F@lists.varnish-cache.org> commit 93ae66aefb9d6795087a638feec8f201269d53a0 Author: Geoff Simmons Date: Mon Mar 12 15:12:25 2018 +0100 Document -b /path/to/uds in varnishd usage output. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 6e33f8d..4338fb0 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -100,7 +100,8 @@ usage(void) printf(FMT, "", "Proto can be \"PROXY\" or \"HTTP\" (default)"); printf(FMT, "", "user, group and mode set permissions for"); printf(FMT, "", " a Unix domain socket."); - printf(FMT, "-b address[:port]", "Backend address and port"); + printf(FMT, "-b [addr[:port]|path]", "Backend address and port"); + printf(FMT, "", " or socket file path"); printf(FMT, "", " default: \":80\""); printf(FMT, "-f vclfile", "VCL program"); printf(FMT, "", "Can be specified multiple times."); From nils.goroll at uplex.de Mon Mar 12 14:28:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Mar 2018 14:28:10 +0000 (UTC) Subject: [master] ef2b7f7 consistent use of tabs and spaces for python3 Message-ID: <20180312142810.A48A463752@lists.varnish-cache.org> commit ef2b7f74c3d41ee3b4fdd52d46e1d8f08fa81748 Author: Nils Goroll Date: Mon Mar 12 15:22:41 2018 +0100 consistent use of tabs and spaces for python3 diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 569758b..b6495c5 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -422,8 +422,8 @@ class prototype(object): t = i.vcl() if t in privs: continue - if i.nm is not None: - t += " " + i.nm + if i.nm is not None: + t += " " + i.nm if not short: if i.defval is not None: t += "=" + i.defval @@ -602,7 +602,7 @@ class s_module(stanza): fo.write("\n") fo.write(":Manual section: " + self.vcc.mansection + "\n") - if self.vcc.auto_synopsis: + if self.vcc.auto_synopsis: fo.write("\n") write_rst_hdr(fo, "SYNOPSIS", "=") fo.write("\n") @@ -610,8 +610,8 @@ class s_module(stanza): fo.write(' import %s [from "path"] ;\n' % self.vcc.modname) fo.write(" \n") for c in self.vcc.contents: - c.synopsis(fo, man) - fo.write("\n") + c.synopsis(fo, man) + fo.write("\n") def rsttail(self, fo, man): From geoff at uplex.de Mon Mar 12 15:30:13 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 15:30:13 +0000 (UTC) Subject: [master] 6c6e320 Document local.socket and .endpoint in "Upgrading to 6.0". Message-ID: <20180312153013.48F9364ABF@lists.varnish-cache.org> commit 6c6e320bda437eae324a4c45d3bf8e936144e0d1 Author: Geoff Simmons Date: Mon Mar 12 16:28:52 2018 +0100 Document local.socket and .endpoint in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index c840b92..b5ed03e 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -108,6 +108,72 @@ XXX: ... intro paragraph VCL variables ~~~~~~~~~~~~~ +``local.socket`` and ``local.endpoint`` +--------------------------------------- + +These read-only variables are available as of VCL 4.1, and provide +information about the listener address over which the current client +request was received. + +``local.socket`` is the name provided in the ``-a`` command-line +argument for the current listener, which defaults to ``a0``, ``a1`` +and so on (see varnishd :ref:`ref-varnishd-options`). + +``local.endpoint`` is the value of the ``address[:port]`` or ``path`` +field provided as the ``-a`` value for the current listener, exactly +as given on the command line. For example:: + + # When varnishd is invoked with these -a arguments ... + $ varnishd -a foo=12.34.56.78:4711 -a bar=/path/to/listen.sock + + # ... then in VCL, for requests received over the first listener: + local.socket == "foo" + local.endpoint == "12.34.56.78:4711" + + # ... and for requests received over the second listener: + local.socket == "bar" + local.endpoint == "/path/to/listen.sock" + + # With this invocation ... + $ varnishd -a :80 -a 87.65.43.21 + + # ... then for requests received over the first listener: + local.socket == "a0" + local.endpoint == ":80" + + # ... and for the second listener + local.socket == "a1" + local.endpoint == "87.65.43.21" + +So if you have more than one listener and need to tell them apart in +VCL, for example a listener for "regular" client traffic and another +one for "admin" requests that you must restrict to internal systems, +these two variables can help you do so. + +``local.socket`` and ``local.endpoint`` are available on both the +client and backend sides. But the values on the backend side are not +necessarily the same as they were on the side of the client request +that initiated the backend request. This is because of the separation +of client and backend threads -- a backend thread may be re-used that +was initiated by a client request over another listener, and +``local.socket`` and ``local.endpoint`` on that thread retain the +values for the original listener. + +So if, in your backend VCL code, you need to be sure about the +listener that was used on the client side of the same transaction, +assign ``local.socket`` and/or ``local.endpoint`` to a client request +header, and retrieve the value from a backend request header:: + + sub vcl_miss { + set req.http.X-Listener = local.socket; + } + + sub vcl_backend_fetch { + if (bereq.http.X-Listener == "a0") { + # ... + } + } + ``sess.xid`` ------------ @@ -118,11 +184,6 @@ same XID shown in the log for session transactions (with ``-g session`` grouping). ``sess.xid`` is read-only and is available as of VCL 4.1. -XXX: VCL vars subhead 2 ------------------------ - -XXX: ... - Unix domain sockets and VCL ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From geoff at uplex.de Mon Mar 12 16:42:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 16:42:10 +0000 (UTC) Subject: [master] aff42fa Explain why Host may be 0.0.0.0 for a UDS backend in "Upgrading". Message-ID: <20180312164210.04D03903E5@lists.varnish-cache.org> commit aff42fa5ed73751400b2f73bf361aedbe8f53313 Author: Geoff Simmons Date: Mon Mar 12 17:38:57 2018 +0100 Explain why Host may be 0.0.0.0 for a UDS backend in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index b5ed03e..c2eb6d6 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -365,6 +365,35 @@ protocol is not used, then ``0.0.0.0`` will be added to Again, this is probably not a concern if ``client.ip`` is set via the PROXY protocol. +UDS backends and the Host header +-------------------------------- + +By default, Varnish forwards the Host header from a client request to +the backend. If there is no Host header in the client request, and the +``.host_header`` field was set in the backend declaration, then that +value is used for the backend Host header. For backends declared with +the ``.host`` field (with a domain name or IP address), then if there +is neither a client Host header nor a ``.host_header`` declaration, +the value of ``.host`` is set as the Host header of the backend +request. + +If the backend was declared with ``.path`` for a socket path, then the +backend Host header is set to ``0.0.0.0`` under those conditions. + +To re-state that: + +* If the backend was declared with ``.path`` to connect to a Unix + domain socket, ... + +* and ``.host_header`` was not set in the backend declaration, ... + +* and there is no Host header in the client request, ... + +* then the Host header in the backend request is set to ``0.0.0.0``. + +If you want to avoid that, set a ``.host_header`` value for the +backend, or set a value for the Host header in VCL. + VMOD std -------- From geoff at uplex.de Mon Mar 12 16:42:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 16:42:10 +0000 (UTC) Subject: [master] d70ec08 Fix the test for BackendOpen with a UDS backend. Message-ID: <20180312164210.26C81903E8@lists.varnish-cache.org> commit d70ec08499e85c64bf4f480fd5263e9b856f528c Author: Geoff Simmons Date: Mon Mar 12 17:40:18 2018 +0100 Fix the test for BackendOpen with a UDS backend. diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc index cdbb2d4..be6a106 100644 --- a/bin/varnishtest/tests/b00060.vtc +++ b/bin/varnishtest/tests/b00060.vtc @@ -17,12 +17,13 @@ logexpect l1 -v v1 -d 1 -g session { expect 0 = SessOpen "^0.0.0.0 0 foo 0.0.0.0 0" } -run -logexpect l2 -v v1 -d 1 -g vxid -c { - expect 0 1001 Begin - expect * = ReqStart "^0.0.0.0 0$" +logexpect l2 -v v1 -d 1 -g vxid { + expect * * Begin ^req + expect * = ReqStart "^0.0.0.0 0 foo$" } -run -logexpect l2 -v v1 -d 1 -g vxid -b { - expect 0 1002 Begin - expect * = BackendOpen "${s1_sock} - - -$" +logexpect l3 -v v1 -d 1 -g vxid { + expect * * Begin ^bereq + expect * = BackendOpen "0.0.0.0 0 0.0.0.0 0$" + expect * = BackendStart "^0.0.0.0 0$" } -run From geoff at uplex.de Mon Mar 12 16:56:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 16:56:08 +0000 (UTC) Subject: [master] 265a3ff More docs about the effects of UDS on logging in "Upgrading". Message-ID: <20180312165608.D2DE7908E5@lists.varnish-cache.org> commit 265a3ff12551dc83c8c5bcd3596a8a6b272e18d4 Author: Geoff Simmons Date: Mon Mar 12 17:54:59 2018 +0100 More docs about the effects of UDS on logging in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index c2eb6d6..4d835e8 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -418,11 +418,17 @@ Other changes * ``0.0.0.0`` and port ``0`` appear in the logs where an IP and port otherwise appear, when the connection in question was addressed as - a Unix domain socket. This affects ``ReqStart`` and ``SessOpen``. + a Unix domain socket. This affects ``ReqStart``, ``SessOpen``, + ``BackendStart`` and ``BackendOpen``. + If you have more than one UDS listener, they can be distinguished with the "listener name" field -- the third field for both ``ReqStart`` and ``SessOpen``. + If you have more than one UDS backend, they can be distinguished + with the backend name field -- the second field in + ``BackendOpen``. + * ``varnishtest(1)``: * The ``client -connect`` and ``server -listen`` commands in vtc From geoff at uplex.de Mon Mar 12 17:14:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 17:14:08 +0000 (UTC) Subject: [master] 570a92b Update docs about UDS with varnishtest/vtc in "Upgrading to 6.0". Message-ID: <20180312171408.99E6090EB0@lists.varnish-cache.org> commit 570a92be487598747aec1bdb0c451afc6ba56af0 Author: Geoff Simmons Date: Mon Mar 12 18:12:52 2018 +0100 Update docs about UDS with varnishtest/vtc in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 4d835e8..f93b8d2 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -429,7 +429,7 @@ Other changes with the backend name field -- the second field in ``BackendOpen``. -* ``varnishtest(1)``: +* ``varnishtest(1)`` and ``vtc(7)``: * The ``client -connect`` and ``server -listen`` commands in vtc scripts now allow Unix domain sockets as addresses, recognized @@ -450,12 +450,19 @@ Other changes ``varnish -arg`` command with the appropriate settings for the ``-a`` command line argument, see :ref:`varnishd(1)`. + The ``varnish -vcl+backend`` command now works to include backend + definitions for server objects that are listening at UDS. Backend + declarations are implicitly included for such servers with the + appropriate ``.path`` setting. + A convenient location for socket files to be used in a test is the temporary directory created by ``varnishtest`` for each test, whose path is held in the macro ``${tmpdir}``. So this is a common idiom for tests that involve UDSen:: - varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl { ... } -start + server s1 -listen "${tmpdir}/s1.sock" { ... } -start + + varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { ... } -start client c1 -connect "${tmpdir}/v1.sock" { ... } -run From geoff at uplex.de Mon Mar 12 17:50:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 17:50:10 +0000 (UTC) Subject: [master] bf77721 Document the impact of UDS on varnishncsa in "Upgrading to 6.0". Message-ID: <20180312175010.A110991A75@lists.varnish-cache.org> commit bf77721c5fcc3daf77a5a79d64120b822b9fa392 Author: Geoff Simmons Date: Mon Mar 12 18:49:23 2018 +0100 Document the impact of UDS on varnishncsa in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index f93b8d2..cffab95 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -429,6 +429,32 @@ Other changes with the backend name field -- the second field in ``BackendOpen``. +* ``varnishncsa(1)`` + + * The ``%h`` formatter (remote host) gets its value from + ``ReqStart`` for client requests and ``BackendStart`` for backend + requests. The value will be ``0.0.0.0`` for client requests when + the listener is UDS, and for backend requests when the backend is + UDS. + + * The ``%r`` formatter (first line of the request) is reconstructed + in part from the Host request header. For UDS backends, Host may + be ``0.0.0.0`` for the reasons explained above (no client Host + header and no ``.host_header`` setting for the backend), so that + may appear in the output for ``%r``. You can avoid that with the + measures discussed above. + + * If you have more than one UDS listener and/or more than one UDS + backend, and you want to tell them apart in the ``varnishncsa`` + output (rather than just see ``0.0.0.0``), use the ``%{VSL}x`` + formatter to capture the listener name and the backend name. + + For the listener name, use ``%{VSL:ReqStart[3]}x`` for client logs + (the third field of ``ReqStart`` logs). + + For the backend name, use ``%{VSL:BackendOpen[2]}x`` for backend + logs. + * ``varnishtest(1)`` and ``vtc(7)``: * The ``client -connect`` and ``server -listen`` commands in vtc From geoff at uplex.de Mon Mar 12 17:54:06 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 17:54:06 +0000 (UTC) Subject: [master] 694eaa2 Fix an incorrect comment. Message-ID: <20180312175406.E296C91C2E@lists.varnish-cache.org> commit 694eaa295e703a28c4a72595acb291405af66202 Author: Geoff Simmons Date: Mon Mar 12 18:52:50 2018 +0100 Fix an incorrect comment. diff --git a/bin/varnishtest/tests/u00013.vtc b/bin/varnishtest/tests/u00013.vtc index bdfe8d1..411bba8 100644 --- a/bin/varnishtest/tests/u00013.vtc +++ b/bin/varnishtest/tests/u00013.vtc @@ -4,7 +4,7 @@ varnishtest "varnishncsa outputs when UDS addresses are in use" # which now may be a UDS address. # For UDS backends without a .hosthdr setting, the Host header is -# set to "localhost", which may appear in %r output. +# set to "0.0.0.0", which may appear in %r output. server s1 -listen "${tmpdir}/s1.sock" { rxreq From geoff at uplex.de Mon Mar 12 18:18:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 18:18:07 +0000 (UTC) Subject: [master] 4107128 Write up VMOD unix in "Upgrading to 6.0". Message-ID: <20180312181807.A48D092439@lists.varnish-cache.org> commit 41071281d91c275be3f1633dca5f20e7ec110047 Author: Geoff Simmons Date: Mon Mar 12 19:16:03 2018 +0100 Write up VMOD unix in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index cffab95..685c184 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -402,10 +402,30 @@ VMOD std listener is UDS. :ref:`std.set_ip_tos(INT) ` is silently ignored when the listener is UDS. -XXX VCL subhead 2 -~~~~~~~~~~~~~~~~~ +New VMODs +~~~~~~~~~ -XXX: ... +VMOD unix +--------- + +:ref:`vmod_unix(3)` provides functions to determine the credentials of +the peer process (user and group of the process owner) that connected +to Varnish over a listener at a Unix domain socket. You can use this, +for example, to impose tighter restrictions on who can access certain +resources:: + + import unix; + + sub vcl_recv { + # Return "403 Forbidden" if the connected peer is + # not running as the user "trusteduser". + if (unix.user() != "trusteduser") { + return( synth(403) ); + } + +This is not available on every platform. As always, check the +documentation and test the code before you attempt something like this +in production. Other changes ============= From geoff at uplex.de Mon Mar 12 18:31:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 12 Mar 2018 18:31:08 +0000 (UTC) Subject: [master] 37c13cd Write up the $Synopsis directive for vcc files in "Upgrading". Message-ID: <20180312183109.0534E92886@lists.varnish-cache.org> commit 37c13cde544b3f4ee2defc692713bf301286f77b Author: Geoff Simmons Date: Mon Mar 12 19:30:29 2018 +0100 Write up the $Synopsis directive for vcc files in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 685c184..3565eb0 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -538,7 +538,17 @@ Other changes * Changes for developers: - * XXX ... + * VMOD vcc sources may now include a directive ``$Synopsis`` whose + value may be ``auto`` or ``manual``, default ``auto``. + + When ``$Synopsis`` is ``auto``, the vmodtool generates a more + comprehensive ``SYNOPSIS`` section in the documentation than in + previous versions -- an overview of the objects, methods and + functions in your VMOD, with their type signatures. + + When ``$Synopsis`` is ``manual``, the ``SYNOPSIS`` is left out of + the generated docs altogether; so you can write the ``SYNOPSIS`` + section yourself, if you prefer. * XXX ... From phk at FreeBSD.org Mon Mar 12 20:08:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Mar 2018 20:08:08 +0000 (UTC) Subject: [master] bd844c3 Fix to match varnishd compilation flags Message-ID: <20180312200808.449C9944FC@lists.varnish-cache.org> commit bd844c3511d76b4f04a945a463c5a83047f17c26 Author: Poul-Henning Kamp Date: Mon Mar 12 20:06:51 2018 +0000 Fix to match varnishd compilation flags diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 0be99fc..8f7ecb8 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -2,7 +2,7 @@ FLOPS=' -I../../lib/libvgz - -DVARNISHD_IS_NOT_A_VMOD + -DNOT_IN_A_VMOD -DVARNISH_STATE_DIR="foo" -DVARNISH_VMOD_DIR="foo" -DVARNISH_VCL_DIR="foo" From phk at FreeBSD.org Tue Mar 13 07:51:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Mar 2018 07:51:09 +0000 (UTC) Subject: [master] f346827 White-space OCD Message-ID: <20180313075110.03A2FA6847@lists.varnish-cache.org> commit f3468275891966ac10607e02d8cf698a19af5a64 Author: Poul-Henning Kamp Date: Tue Mar 13 07:23:41 2018 +0000 White-space OCD diff --git a/bin/varnishtest/tests/m00047.vtc b/bin/varnishtest/tests/m00047.vtc index bfc8f7c..3eb3857 100644 --- a/bin/varnishtest/tests/m00047.vtc +++ b/bin/varnishtest/tests/m00047.vtc @@ -50,11 +50,11 @@ logexpect l1 -v v1 -d 1 -b { } -run logexpect l1 -v v1 -d 1 -c { - expect * 1001 Begin req - expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} - expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} - expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} - expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * 1001 Begin req + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} + expect * = VCL_Error {^vmod unix error: not listening on a Unix domain socket$} } -run diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index 4e90bad..55d7f67 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -26,13 +26,13 @@ Examples:: } # Require the connected peer to run in the group - # "trustedgroup". + # "trustedgroup". if (unix.group() != "trustedgroup") { return( synth(403) ); } # Require the connected peer to run under a specific numeric - # user id. + # user id. if (unix.uid() != 4711) { return( synth(403) ); } @@ -87,7 +87,7 @@ All functions in this VMOD are subject to the following constraints: If the failure occurs during ``vcl_synth``, then ``vcl_synth`` is aborted, and the the response line "503 VCL failed" is sent. - + * If the current listener is not a Unix domain socket, or if the attempt to read credentials fails, then a ``VCL_Error`` message is written to the log. The STRING functions (``vmod_user`` and From phk at FreeBSD.org Tue Mar 13 08:58:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Mar 2018 08:58:08 +0000 (UTC) Subject: [master] b532d13 Exercise the VSC's created from VMODs Message-ID: <20180313085808.B86B2A7BC9@lists.varnish-cache.org> commit b532d1340ed8745b183e2aff2fa9cfa94b02bd2c Author: Poul-Henning Kamp Date: Tue Mar 13 08:56:39 2018 +0000 Exercise the VSC's created from VMODs diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index fc0ecf1..7bdaa02 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -13,10 +13,12 @@ varnish v1 -vcl+backend { sub vcl_init { new objx = debug.obj(); + debug.vsc_new(); } sub vcl_recv { debug.rot52(req); + debug.vsc_count(); } sub vcl_deliver { @@ -36,6 +38,8 @@ varnish v1 -vcl+backend { } } -start +varnish v1 -expect DEBUG.count == 0 + client c1 { txreq -url "/bar" rxresp @@ -50,6 +54,8 @@ client c1 { expect resp.http.not == -1 } -run +varnish v1 -expect DEBUG.count == 1 + logexpect l1 -v v1 -g raw -d 1 { expect * 1001 VCL_call {^DELIVER} expect 0 = RespUnset {^foo: bAr} diff --git a/lib/libvmod_debug/VSC_debug.vsc b/lib/libvmod_debug/VSC_debug.vsc index aac6410..d6902ff 100644 --- a/lib/libvmod_debug/VSC_debug.vsc +++ b/lib/libvmod_debug/VSC_debug.vsc @@ -5,7 +5,7 @@ Test counters from vmod_debug -.. varnish_vsc:: foo +.. varnish_vsc:: count :type: counter :level: debug :oneliner: arbitrary counter diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 20f75ec..d6094bc 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -170,6 +170,10 @@ $Function VOID vsc_new() Add a vsc +$Function VOID vsc_count(INT val = 1) + +Update counter + $Function VOID vsc_destroy() Remove a vsc diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 4de4947..0da45d8 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -371,6 +371,14 @@ xyzzy_vsc_new(VRT_CTX) } VCL_VOID +xyzzy_vsc_count(VRT_CTX, VCL_INT cnt) +{ + (void)ctx; + AN(vsc); + vsc->count += cnt; +} + +VCL_VOID xyzzy_vsc_destroy(VRT_CTX) { (void)ctx; From phk at FreeBSD.org Tue Mar 13 09:48:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Mar 2018 09:48:08 +0000 (UTC) Subject: [master] 5a239cb More correct handling of POLLERR and POLLHUP Message-ID: <20180313094808.67417A8B36@lists.varnish-cache.org> commit 5a239cbc3a6def944712f3ad063076cb14724acf Author: Poul-Henning Kamp Date: Tue Mar 13 09:47:16 2018 +0000 More correct handling of POLLERR and POLLHUP diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index 6f8828a..224298b 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -104,9 +104,9 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a) } memset(fds, 0, sizeof fds); fds[0].fd = fd; - fds[0].events = POLLIN | POLLERR; + fds[0].events = POLLIN; fds[1].fd = req->sp->fd; - fds[1].events = POLLIN | POLLERR; + fds[1].events = POLLIN; while (fds[0].fd > -1 || fds[1].fd > -1) { fds[0].revents = 0; diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index f72bbb0..e5857e7 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1509,14 +1509,14 @@ cmd_http_expect_close(CMD_ARGS) stop_h2(hp); while (1) { fds[0].fd = hp->fd; - fds[0].events = POLLIN | POLLERR; + fds[0].events = POLLIN; fds[0].revents = 0; i = poll(fds, 1, hp->timeout); if (i < 0 && errno == EINTR) continue; if (i == 0) vtc_log(vl, hp->fatal, "Expected close: timeout"); - if (i != 1 || !(fds[0].revents & (POLLIN|POLLERR))) + if (i != 1 || !(fds[0].revents & (POLLIN|POLLERR|POLLHUP))) vtc_log(vl, hp->fatal, "Expected close: poll = %d, revents = 0x%x", i, fds[0].revents); From nils.goroll at uplex.de Tue Mar 13 11:14:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Mar 2018 11:14:08 +0000 (UTC) Subject: [master] 1f26a9b doc: be concise about about std.real2integer / std.real2time Message-ID: <20180313111408.C56D9AC56E@lists.varnish-cache.org> commit 1f26a9b5411c2d1368c1f808db75ff8e9f24c2bb Author: Nils Goroll Date: Tue Mar 13 12:13:22 2018 +0100 doc: be concise about about std.real2integer / std.real2time diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fe7fd63..34b6399 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -195,16 +195,21 @@ Example $Function INT real2integer(REAL r, INT fallback) Description - Converts the real *r* to an integer. If conversion fails, + Rounds the real *r* to the nearest integer, but round halfway + cases away from zero (see round(3)). If conversion fails, *fallback* will be returned. Example set req.http.integer = std.real2integer(1140618699.00, 0); + set req.http.posone = real2integer( 0.5, 0); # = 1.0 + set req.http.negone = real2integer(-0.5, 0); # = -1.0 $Function TIME real2time(REAL r, TIME fallback) Description - Converts the real *r* to a time. If conversion fails, - *fallback* will be returned. + Rounds the real *r* to the nearest integer (see + `func_real2integer`_) and returns the corresponding time when + interpreted as a unix epoch. If conversion fails, *fallback* + will be returned. Example set req.http.time = std.real2time(1140618699.00, now); From geoff at uplex.de Tue Mar 13 12:18:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 13 Mar 2018 12:18:08 +0000 (UTC) Subject: [master] 1323396 Clarify that VMOD unix applies to a listener address. Message-ID: <20180313121808.E2CBAAD874@lists.varnish-cache.org> commit 13233968041fe794a64c34205733eab72aabe0f0 Author: Geoff Simmons Date: Tue Mar 13 13:17:00 2018 +0100 Clarify that VMOD unix applies to a listener address. Currently not backends. diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index 55d7f67..8b72913 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -13,8 +13,9 @@ DESCRIPTION =========== This VMOD provides information about the credentials of the peer -process (user and group of the process owner) that is connected to -Varnish via a Unix domain socket, if the platform supports it. +process (user and group of the process owner) that is connected to a +Varnish listener via a Unix domain socket, if the platform supports +it. Examples:: From geoff at uplex.de Tue Mar 13 12:33:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 13 Mar 2018 12:33:08 +0000 (UTC) Subject: [master] 2edb4d2 Doc: VRT creation of a UDS backend for VMODs in "Upgrading to 6.0". Message-ID: <20180313123308.6C83CADD46@lists.varnish-cache.org> commit 2edb4d2ceee4ebbf86e2e744508778a5d1f473b6 Author: Geoff Simmons Date: Tue Mar 13 13:31:22 2018 +0100 Doc: VRT creation of a UDS backend for VMODs in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 3565eb0..62c83d0 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -538,6 +538,17 @@ Other changes * Changes for developers: + * As part of VRT version 7.0, the ``path`` field has been added to + to ``struct vrt_backend``, which a VMOD can use with + ``VRT_new_backend()`` to create a dynamic backend with a UDS + address (see ``vrt.h``). + + If ``path`` is non-NULL, then both of the IPv4 and IPv6 addresses + must be NULL. If ``path`` is NULL, then (as before) one or both of + the IP addresses must be non-NULL. The ``dyn_uds`` object in VMOD + debug (available in the source tree) illustrates how this can be + done. + * VMOD vcc sources may now include a directive ``$Synopsis`` whose value may be ``auto`` or ``manual``, default ``auto``. From dridi.boukelmoune at gmail.com Tue Mar 13 15:18:08 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Mar 2018 15:18:08 +0000 (UTC) Subject: [master] 5cbab39 First stab at packaging upgrade notes Message-ID: <20180313151808.1BFE2B0FB0@lists.varnish-cache.org> commit 5cbab39899f544d036e0f0fd180e84980082b689 Author: Dridi Boukelmoune Date: Tue Mar 13 16:16:10 2018 +0100 First stab at packaging upgrade notes Not that I'm planning another stab, but please review and let me know if anything needs clarification. diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 62c83d0..8a5e84a 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -427,6 +427,115 @@ This is not available on every platform. As always, check the documentation and test the code before you attempt something like this in production. +Packaging changes +================= + +Supported platforms +~~~~~~~~~~~~~~~~~~~ + +Official Varnish packages went through major changes for this release, +and target Debian 8 and 9, Ubuntu 16.04 LTS and (Red Hat) Enterprise +Linux 7. Ubuntu 14.04 LTS will likely reach its end of life before +Varnish 6 and the venerable Enterprise Linux 6 is getting too old and +forced time-consuming workarounds so for these reasons we dropped +community support for those platforms. + +Services +~~~~~~~~ + +As a result we ended up with ended up with systemd-only platforms for +the official packages. The old services are still available as we +archived them in the ``pkg-varnish-cache`` source tree. This was the +occasion to remove differences between Red Hat and Debian derivatives +since there's no more reasons to have them diverge: we initially +inherited packaging support from downstream package maintainers, and +they deserve many thanks for that. + +Another big difference between Red Hat and Debian derivatives was the +way we handled VCL reloads via the service manager. We introduced a +new ``varnishreload`` script that operates on top of ``varnishadm`` +to perform hot reloads of one VCL configuration or label at a time. +All you need is enough privileges to contact ``varnishd``'s command +line interface, which should not be a problem for package managers. + +Once the ``varnish`` package is installed, you can learn more:: + + varnishreload -h + +Again, many thanks to downstream maintainers and some early adopters +for their help in testing the new script. + +To stay on the topic of the command line interface, packages no longer +create a secret file for the CLI, and services omit ``-S`` and ``-T`` +options on the ``varnishd`` command. This means that out of the box, +you can only connect to the CLI locally with enough privileges to read +a secret generated randomly. This means less noise in our packages, +and you need to change the service configuration to enable remote +access to the CLI. With previous packages, you also needed to change +your configuration because the CLI would only listen to the loopback +interface anyway. + +To change how ``varnishd`` is started, please refer to the systemd +documentation. + +Virtual provides +~~~~~~~~~~~~~~~~ + +Last but not least in the packaging space, we took a first step towards +improving dependency management between official ``varnish`` packages +and VMODs built on top of them. RPMs and Deb packages now provide the +strict and VRT ABIs from ``varnishd`` and the goal is to ultimately +prevent a package installation or upgrade that would prevent a VMOD +from being loaded. + +For Deb packages:: + + Provides: + varnishd-abi-SHA1, + varnishd-vrt (= x.y) + +And for RPMs:: + + Provides: varnishd(abi)(x86-64) = SHA1 + Provides: varnishd(vrt)(x86-64) = x.y + +For VMOD authors or downstream distributors, this means that depending +on the ``$ABI`` stanza in the VMOD descriptor, they can either tie their +backend manually to the git hash Varnish was built from or to the VRT +version used at the time. + +For example, a VMOD RPM built against Varnish 6.0.0 could have:: + + Requires: varnishd(vrt)%{?_isa} >= 7.0 + Requires: varnishd(vrt)%{?_isa} < 8 + +Future plans include the ability to automate this for out-of-tree VMODs +and remove manual steps. To learn more about the history behind this +change, it was formalized via the Varnish Improvement Process: + +https://github.com/varnishcache/varnish-cache/wiki/VIP20%3A-Varnish-ABI-and-packaging + +Another thing available only to RPM packages as of 6.0.0 is virtual +provides for VMODs. + +Instead of showing shared objects that aren't even in the dynamic +linker's default path:: + + Provides: libvmod_std.so(64bit) + Provides: libvmod_directors.so(64bit) + [...] + +You get virtual VMOD provides with a version:: + + Provides: vmod(std)(x86-64) = 6.0.0-1 + Provides: vmod(directors)(x86-64) = 6.0.0-1 + [...] + +With the same mechanism it becomes possible to require a VMOD directly +and let it bring along its dependencies, like ``varnish``. As this is +currently not automated for out-of-tree VMODs, consider this a preview +of what you will be able to do once VIP 20 is completed. + Other changes ============= From dridi.boukelmoune at gmail.com Tue Mar 13 18:00:12 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Mar 2018 18:00:12 +0000 (UTC) Subject: [master] 90c4b74 Install cache/cache_varnishd.h as part of $ABI strict Message-ID: <20180313180012.4D707B40BA@lists.varnish-cache.org> commit 90c4b747ced9053353c13d827ef0a0df5ab4445f Author: Dridi Boukelmoune Date: Tue Mar 13 18:41:21 2018 +0100 Install cache/cache_varnishd.h as part of $ABI strict This removes the need for non-VRT VMODs to require a Varnish source tree to build, especially since varnish.m4 only supports Varnish installations today and would leave VMOD authors on their own in order to build from a source tree. Since $ABI defaults to strict, any VMOD author actively choosing vrt compliance is responsible for ensuring it only uses blessed symbols. In our official packages, they land in the varnish-dev deb and the varnish-devel RPM: $ rpm --query --package --list varnish-trunk-1.el7.x86_64.rpm | > grep cache_varnishd.h $ rpm --query --package --list varnish-devel-trunk-1.el7.x86_64.rpm | > grep cache_varnishd.h /usr/include/varnish/cache/cache_varnishd.h You will have to trust me for the Debs as it's a bit cumbersome for me to build them locally. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 452d044..a033001 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -124,9 +124,7 @@ noinst_HEADERS = \ cache/cache_pool.h \ cache/cache_tcp_pool.h \ cache/cache_transport.h \ - cache/cache_varnishd.h \ cache/cache_vgz.h \ - common/common_param.h \ common/heritage.h \ common/vsmw.h \ hash/hash_slinger.h \ @@ -147,6 +145,8 @@ nobase_pkginclude_HEADERS = \ cache/cache_backend.h \ cache/cache_director.h \ cache/cache_filter.h \ + cache/cache_varnishd.h \ + common/common_param.h \ waiter/waiter.h vcldir=$(datarootdir)/$(PACKAGE)/vcl From nils.goroll at uplex.de Tue Mar 13 18:36:02 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Mar 2018 19:36:02 +0100 Subject: [master] 90c4b74 Install cache/cache_varnishd.h as part of $ABI strict In-Reply-To: <20180313180012.4D707B40BA@lists.varnish-cache.org> References: <20180313180012.4D707B40BA@lists.varnish-cache.org> Message-ID: <04217589-9db9-1e03-cc28-3697fcd6b406@uplex.de> -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 I wonder if this is what phk had in mind when he moved the ObjIter symbold to cache_varnishd.h ... That is not to say that I wouldn't find this solution the most practical, but at this point my impression is that we got conflicting goals. Dridi, if I understand your intention correctly, you are suggesting that we include headers in the dist, which, during packaging, we do not put into the varnish "bin" package, but rather only "-dev". If this becomes the new way to got, then I would suggest that we mark in tree which includes to give this special packaging handling such that we do not need to duplicate that information for each packager. Nils - -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -----BEGIN PGP SIGNATURE----- iQEzBAEBCAAdFiEEicdlqw+9Qa90PBmNHc2PV6OGi9cFAlqoGhAACgkQHc2PV6OG i9dKrggAkgDsMrZZjwe1fA8gzfiCx2M4EHfaTbOUQbb4YN9JbMKZUNUdAsNr+M81 KafP5U6qk0/vvhW5UjbFaFEDFsdWLW6/j48nEA6VH0yaHDrENkMY5gbqKFxfG07F AszqtuKczF4Pye1nURuwbmxZGy23XNkKlvN6qQWoprMi9/7jCNeF/zCShkelpEvr eX1ovgfEE6joEySq+6HgYO32vgJ2E1dQ3XWNVsbEZ6QDFAFn2aurRbKiIjA+DDuf SLyPbYFF4CSLjoMp5B90KIu2uALaI5tBKQ6xhqkmaW2F9jmMkeNrWdlRMaMdJcsT vXyVivPnGibDHWZvEZZsewCsvfhoyw== =JXIl -----END PGP SIGNATURE----- From dridi at varni.sh Tue Mar 13 18:52:41 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 13 Mar 2018 19:52:41 +0100 Subject: [master] 90c4b74 Install cache/cache_varnishd.h as part of $ABI strict In-Reply-To: <04217589-9db9-1e03-cc28-3697fcd6b406@uplex.de> References: <20180313180012.4D707B40BA@lists.varnish-cache.org> <04217589-9db9-1e03-cc28-3697fcd6b406@uplex.de> Message-ID: > Dridi, if I understand your intention correctly, you are suggesting that we > include headers in the dist, The header is unconditionally part of the dist, it rather moved from noinst to includedir. This is for make install instead. > which, during packaging, we do not put into the > varnish "bin" package, but rather only "-dev". Yes/no, all headers are part of the -dev RPMs/Debs as per the packaging policies of our supported platforms. > If this becomes the new way to got, then I would suggest that we mark in tree > which includes to give this special packaging handling such that we do not > need to duplicate that information for each packager. Since $ABI vrt is a conscious choice from the VMOD author, I don't see the point of adding one more layer of indirection. Dridi From geoff at uplex.de Tue Mar 13 19:03:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 13 Mar 2018 19:03:09 +0000 (UTC) Subject: [master] c138645 VMOD unix doc fixes. Message-ID: <20180313190309.4F0D4B5468@lists.varnish-cache.org> commit c138645f92698670bae157f61c893f8ca2679b06 Author: Geoff Simmons Date: Tue Mar 13 20:01:41 2018 +0100 VMOD unix doc fixes. diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index 8b72913..d698a91 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -19,6 +19,8 @@ it. Examples:: + import unix; + sub vcl_recv { # Return "403 Forbidden" if the connected peer is # not running as the user "trusteduser". @@ -92,7 +94,7 @@ All functions in this VMOD are subject to the following constraints: * If the current listener is not a Unix domain socket, or if the attempt to read credentials fails, then a ``VCL_Error`` message is written to the log. The STRING functions (``vmod_user`` and - ``vmod_group``) return NULL, while the INT functions (``vmod_user`` + ``vmod_group``) return NULL, while the INT functions (``vmod_uid`` and ``vmod_gid``) return -1. SEE ALSO From nils.goroll at uplex.de Tue Mar 13 19:09:30 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Mar 2018 20:09:30 +0100 Subject: [master] 90c4b74 Install cache/cache_varnishd.h as part of $ABI strict In-Reply-To: References: <20180313180012.4D707B40BA@lists.varnish-cache.org> <04217589-9db9-1e03-cc28-3697fcd6b406@uplex.de> Message-ID: -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 On 13/03/18 19:52, Dridi Boukelmoune wrote: >> Dridi, if I understand your intention correctly, you are suggesting that >> we include headers in the dist, > The header is unconditionally part of the dist, it rather moved from noinst > to includedir. This is for make install instead. Apologies for my confusion - dist was wrong. Yes, you have moved that header from not being installed to being installed. And my impression was that phk explicitly intended cache_varnishd to be private to varnishd except for "I know what I am doing"-vmods which would use SRCDIR. Nils (really confused) - -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -----BEGIN PGP SIGNATURE----- iQEzBAEBCAAdFiEEicdlqw+9Qa90PBmNHc2PV6OGi9cFAlqoIeAACgkQHc2PV6OG i9ePBAf+J9Ptg08h1ntPVeD6yHM64+5hbo45+UtZQRg2JqEQriUI/QQXQzYfELSG IYVB6Ts9IZZJfU1oS+T3d5bM9fnVCp+6w+jo8TNI7O4wccGxyp6mwbdfsMDG4L1z IHNK7bed5fZlglD83zgt4ZJCvbXzmsOpzftAW2tjzwUOXaqg1MUpaScMfmEYw3/6 Wn6Qm5S0m+XsNAXAvpmLK+2M2CdEgV9tD+IA74EckJXFqmpixmBIYkzYIBJ6JEuy Za7n0aFByIHytJ/vfooY/kO5Q1U7cBKz4roLcYf1s9yyWeXoAgkWpBKXOIqbk4nG fQGfpSz973tJZoWWzwwxx0bhXM50+A== =qUbz -----END PGP SIGNATURE----- From dridi at varni.sh Tue Mar 13 19:20:14 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 13 Mar 2018 20:20:14 +0100 Subject: [master] 90c4b74 Install cache/cache_varnishd.h as part of $ABI strict In-Reply-To: References: <20180313180012.4D707B40BA@lists.varnish-cache.org> <04217589-9db9-1e03-cc28-3697fcd6b406@uplex.de> Message-ID: > Apologies for my confusion - dist was wrong. Yes, you have moved that header > from not being installed to being installed. And my impression was that phk > explicitly intended cache_varnishd to be private to varnishd except for "I > know what I am doing"-vmods which would use SRCDIR. Again, if you wish not to rebuild a VMOD when upgrading Varnish except when a major VRT bump occurs, you are in "I know what I am doing" territory since the defaults don't do that and you as a VMOD author have to consider whether your code is VRT-safe. The $ABI stanza is a major step forward in reliability, I don't want to move back to the Varnish 3 days of doing things. While it was fine back then, Varnish 4 made it clear that we could do a lot better and I believe we have gradually done that. Please read VIP 20 again, there's a history showing where we come from and how much we iterated since the VMOD inception. And we still have room for further improvements, so let's not shoot our community in the feet and have them go back to pain points of VMOD building that we managed to erase. For example, the Fedora packaging policy goes against having more than one source for a given set of packages, so Ingvar would have to rethink the packaging of varnish and have it install a source tree somewhere in order to update to Varnish 6, and only then could he finally update the varnish-modules package. Dridi From geoff at uplex.de Tue Mar 13 19:27:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 13 Mar 2018 19:27:10 +0000 (UTC) Subject: [master] 9d58364 Document 4.0->4.1 VCL variable changes in "Upgrading to 6.0". Message-ID: <20180313192710.2EF72B5BA9@lists.varnish-cache.org> commit 9d58364eedbff7c03abb0ff25e3391539e276d94 Author: Geoff Simmons Date: Tue Mar 13 20:25:06 2018 +0100 Document 4.0->4.1 VCL variable changes in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 8a5e84a..23e05e7 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -184,6 +184,24 @@ same XID shown in the log for session transactions (with ``-g session`` grouping). ``sess.xid`` is read-only and is available as of VCL 4.1. +Variable changes in VCL 4.0 and 4.1 +----------------------------------- + +The ``*.proto`` variables (``req.proto``, ``resp.proto``, +``bereq.proto`` and ``beresp.proto``) are read-only as of VCL 4.1, but +are still writable in VCL 4.0. + +``req.esi`` is available in VCL 4.0, but no longer in 4.1. In its +place, ``resp.do_esi`` has been introduced in VCL 4.1. Set +``resp.do_esi`` to false in ``vcl_deliver`` if you want to selectively +disable ESI processing for a client response (even though +``beresp.do_esi`` was true during fetch). + +``beresp.backend.ip`` and ``beresp.storage_hint`` are discontinued as +of VCL 4.1, but are still available in 4.0. Note that +``beresp.storage_hint`` has been deprecated since Varnish 5.1; you +should use ``beresp.storage`` instead. + Unix domain sockets and VCL ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From geoff at uplex.de Tue Mar 13 19:29:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 13 Mar 2018 19:29:08 +0000 (UTC) Subject: [master] a56ea1d Clarification in "Upgrading to 6.0". Message-ID: <20180313192908.9CA9AB5CBA@lists.varnish-cache.org> commit a56ea1d9ee43bac1b25ee1b5005c6aaf7f36d9e1 Author: Geoff Simmons Date: Tue Mar 13 20:28:26 2018 +0100 Clarification in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 23e05e7..f801aa8 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -220,10 +220,10 @@ the component forwarding to Varnish via UDS uses the PROXY protocol, which sets ``client.ip`` and ``server.ip`` to the addresses sent in the PROXY header. -If you don't use UDSen, then nothing about VCL changes. UDS support -requires version 4.1, so if you are keeping your VCL level at 4.0 (and -hence are staying with IP addresses), then none of the following is of -concern. +If you don't use UDSen, then nothing about VCL changes with respect to +network addressing. UDS support requires version 4.1, so if you are +keeping your VCL level at 4.0 (and hence are staying with IP +addresses), then none of the following is of concern. ``client.ip``, ``server.ip``, ``local.ip`` and ``remote.ip`` ------------------------------------------------------------ From nils.goroll at uplex.de Wed Mar 14 09:19:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Mar 2018 09:19:12 +0000 (UTC) Subject: [master] 34fa018 document #2603 Message-ID: <20180314091912.B30369AE08@lists.varnish-cache.org> commit 34fa01836e59369b6884dd122c2e8004f3054546 Author: Nils Goroll Date: Wed Mar 14 10:18:47 2018 +0100 document #2603 diff --git a/doc/changes.rst b/doc/changes.rst index e7cab57..30fb551 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -129,6 +129,16 @@ VCL and bundled VMODs words, all parameters to the shard director ``.backend()`` method now need to be named. +* Integers in VCL are now 64 bits wide across all platforms + (implemented as ``int64_t`` C type) , but due to implementation + specifics of the VCL compiler (VCC), integer literals' precision is + limited to that of a VCL real (``double`` C type, roughly 53 bits). + + In effect, larger integers are not represented accurately (they get + rounded) and may even have their sign changed or trigger a C + compiler warning / error. + + Logging / statistics -------------------- From daghf at varnish-software.com Wed Mar 14 10:29:10 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 14 Mar 2018 10:29:10 +0000 (UTC) Subject: [master] 7f84533 Move cleaning/closing h2 streams to its own function Message-ID: <20180314102910.7002DA0338@lists.varnish-cache.org> commit 7f84533f9e8830c6a8e245343f886099ada6bee7 Author: Dag Haavi Finstad Date: Tue Mar 13 17:48:05 2018 +0100 Move cleaning/closing h2 streams to its own function Stream threads may signal a cleanup sweep by setting h2->do_sweep diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 51b175b..a87e78a 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -150,6 +150,7 @@ struct h2_sess { int refcnt; uint32_t highest_stream; int bogosity; + int do_sweep; struct h2_req *req0; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index e1f02eb..14d381f 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -512,6 +512,7 @@ h2_do_req(struct worker *wrk, void *priv) Lck_Lock(&h2->sess->mtx); r2->scheduled = 0; r2->state = H2_S_CLOSED; + r2->h2sess->do_sweep = 1; if (h2->mailcall == r2) { h2->mailcall = NULL; AZ(pthread_cond_signal(h2->cond)); @@ -879,9 +880,7 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, } VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { - if (r2->state == H2_S_CLOSED && !r2->scheduled) - h2_del_req(wrk, r2); - else if (r2->stream == h2->rxf_stream) + if (r2->stream == h2->rxf_stream) break; } @@ -944,6 +943,47 @@ h2_stream_tmo(struct h2_sess *h2, const struct h2_req *r2) return (r); } +/* + * This is the janitorial task of cleaning up any closed streams, and + * checking if the session is timed out. + */ +static int +h2_sweep(struct worker *wrk, struct h2_sess *h2) +{ + int tmo = 0; + struct h2_req *r2, *r22; + + ASSERT_RXTHR(h2); + + h2->do_sweep = 0; + VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { + if (r2 == h2->req0) { + assert (r2->state == H2_S_IDLE); + continue; + } + switch (r2->state) { + case H2_S_CLOSED: + if (!r2->scheduled) + h2_del_req(wrk, r2); + break; + case H2_S_CLOS_REM: + case H2_S_CLOS_LOC: + case H2_S_OPEN: + if (h2_stream_tmo(h2, r2)) { + tmo = 1; + continue; + } + break; + default: + break; + } + } + if (tmo) + return (0); + return (h2->refcnt > 1); +} + + /*********************************************************************** * Called in loop from h2_new_session() */ @@ -965,9 +1005,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) enum htc_status_e hs; h2_frame h2f; h2_error h2e; - struct h2_req *r2, *r22; char b[8]; - int abandon = 0; ASSERT_RXTHR(h2); (void)VTCP_blocking(*h2->htc->rfd); @@ -980,26 +1018,9 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) case HTC_S_COMPLETE: break; case HTC_S_TIMEOUT: - VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { - if (abandon) - break; - switch (r2->state) { - case H2_S_CLOSED: - if (!r2->scheduled) - h2_del_req(wrk, r2); - break; - case H2_S_OPEN: - case H2_S_CLOS_REM: - case H2_S_CLOS_LOC: - if (h2_stream_tmo(h2, r2)) { - abandon = 1; - continue; - } - return (1); - default: - break; - } - } + if (h2_sweep(wrk, h2)) + return (1); + /* FALLTHROUGH */ default: Lck_Lock(&h2->sess->mtx); @@ -1009,6 +1030,9 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) return (0); } + if (h2->do_sweep) + (void)h2_sweep(wrk, h2); + h2->rxf_len = vbe32dec(h2->htc->rxbuf_b) >> 8; h2->rxf_type = h2->htc->rxbuf_b[3]; h2->rxf_flags = h2->htc->rxbuf_b[4]; From daghf at varnish-software.com Wed Mar 14 10:29:10 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 14 Mar 2018 10:29:10 +0000 (UTC) Subject: [master] b191057 Let h2_tx_rst take the stream id as an argument Message-ID: <20180314102910.82ACFA033B@lists.varnish-cache.org> commit b1910571538484c03bbc1447f2f01da30caff5cd Author: Dag Haavi Finstad Date: Tue Mar 13 18:02:19 2018 +0100 Let h2_tx_rst take the stream id as an argument diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 14d381f..9aa487c 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -829,7 +829,8 @@ h2_frame_complete(struct http_conn *htc) /**********************************************************************/ static h2_error -h2_tx_rst(struct worker *wrk, struct h2_sess *h2, h2_error h2e) +h2_tx_rst(struct worker *wrk, struct h2_sess *h2, + uint32_t stream, h2_error h2e) { h2_error ret; char b[4]; @@ -837,13 +838,13 @@ h2_tx_rst(struct worker *wrk, struct h2_sess *h2, h2_error h2e) CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); Lck_Lock(&h2->sess->mtx); - VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", h2->rxf_stream, h2e->txt); + VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", stream, h2e->txt); Lck_Unlock(&h2->sess->mtx); vbe32enc(b, h2e->val); H2_Send_Get(wrk, h2, h2->req0); ret = H2_Send_Frame(wrk, h2, H2_F_RST_STREAM, - 0, sizeof b, h2->rxf_stream, b); + 0, sizeof b, stream, b); H2_Send_Rel(h2, h2->req0); return (ret); @@ -892,7 +893,8 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, "H2: stream %u: Hit maximum number of " "concurrent streams", h2->rxf_stream); // rfc7540,l,1200,1205 - return (h2_tx_rst(wrk, h2, H2SE_REFUSED_STREAM)); + return (h2_tx_rst(wrk, h2, h2->rxf_stream, + H2SE_REFUSED_STREAM)); } h2->highest_stream = h2->rxf_stream; r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL); @@ -909,7 +911,7 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, if (h2->rxf_stream == 0 || h2e->connection) return (h2e); // Connection errors one level up - return (h2_tx_rst(wrk, h2, h2e)); + return (h2_tx_rst(wrk, h2, h2->rxf_stream, h2e)); } static int From daghf at varnish-software.com Wed Mar 14 10:29:10 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 14 Mar 2018 10:29:10 +0000 (UTC) Subject: [master] 70d49d5 h2: Fix reembark failure handling Message-ID: <20180314102910.A22B1A033F@lists.varnish-cache.org> commit 70d49d5997503fcc91a693723993fbe06806b56b Author: Dag Haavi Finstad Date: Wed Mar 14 11:11:38 2018 +0100 h2: Fix reembark failure handling Properly get rid of streams that failed to reschedule after being waitlisted. Fixes: #2563 Fixes: #2592 diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index a87e78a..cfe8598 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -232,6 +232,7 @@ void h2_kill_req(struct worker *, const struct h2_sess *, int h2_rxframe(struct worker *, struct h2_sess *); h2_error h2_set_setting(struct h2_sess *, const uint8_t *); void h2_req_body(struct req*); +void h2_cleanup_waiting(struct worker *wrk, struct h2_req *r2); task_func_t h2_do_req; #ifdef TRANSPORT_MAGIC vtr_req_fail_f h2_req_fail; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 9aa487c..d4ae479 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -37,6 +37,7 @@ #include "cache/cache_transport.h" #include "cache/cache_filter.h" #include "http2/cache_http2.h" +#include "cache/cache_objhead.h" #include "vend.h" #include "vtcp.h" @@ -946,8 +947,8 @@ h2_stream_tmo(struct h2_sess *h2, const struct h2_req *r2) } /* - * This is the janitorial task of cleaning up any closed streams, and - * checking if the session is timed out. + * This is the janitorial task of cleaning up any closed & refused + * streams, and checking if the session is timed out. */ static int h2_sweep(struct worker *wrk, struct h2_sess *h2) @@ -969,6 +970,13 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2) h2_del_req(wrk, r2); break; case H2_S_CLOS_REM: + if (!r2->scheduled) { + h2_tx_rst(wrk, h2, r2->stream, + H2SE_REFUSED_STREAM); + h2_del_req(wrk, r2); + continue; + } + /* FALLTHROUGH */ case H2_S_CLOS_LOC: case H2_S_OPEN: if (h2_stream_tmo(h2, r2)) { @@ -1088,3 +1096,21 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) } return (h2e ? 0 : 1); } + +void +h2_cleanup_waiting(struct worker *wrk, struct h2_req *r2) +{ + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(r2->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(r2->h2sess, H2_SESS_MAGIC); + + AN(r2->req->ws->r); + WS_Release(r2->req->ws, 0); + AN(r2->req->hash_objhead); + (void)HSH_DerefObjHead(wrk, &r2->req->hash_objhead); + AZ(r2->req->hash_objhead); + assert(r2->state == H2_S_CLOS_REM); + AN(r2->scheduled); + r2->scheduled = 0; + r2->h2sess->do_sweep = 1; +} diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 29ba010..50cdd9a 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -431,6 +431,7 @@ static void v_matchproto_(vtr_reembark_f) h2_reembark(struct worker *wrk, struct req *req) { struct sess *sp; + struct h2_req *r2; sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -443,12 +444,9 @@ h2_reembark(struct worker *wrk, struct req *req) /* Couldn't schedule, ditch */ wrk->stats->busy_wakeup--; wrk->stats->busy_killed++; - AN (req->vcl); - VCL_Rel(&req->vcl); - Req_AcctLogCharge(wrk->stats, req); - Req_Release(req); - DSL(DBG_WAITINGLIST, req->vsl->wid, "kill from waiting list"); - usleep(10000); + CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); + VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list"); + h2_cleanup_waiting(wrk, r2); } diff --git a/bin/varnishtest/tests/r02563.vtc b/bin/varnishtest/tests/r02563.vtc new file mode 100644 index 0000000..bfe3514 --- /dev/null +++ b/bin/varnishtest/tests/r02563.vtc @@ -0,0 +1,64 @@ +varnishtest "#2563: Panic after reembark failure" + +barrier b1 cond 2 +barrier b2 cond 2 + +server s1 { + rxreq + expect req.url == "/foo" + expect req.http.client == "c1" + send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n" + delay .2 + barrier b1 sync + delay .2 + send "line1\n" + delay .2 + barrier b2 sync + send "line2\n" +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.do_stream = false; + } +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +failresched" +varnish v1 -cliok "param.set debug +waitinglist" +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq -url "/foo" -hdr "client: c1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 12 + expect resp.http.x-varnish == "1001" +} -start + +barrier b1 sync + +client c2 { + stream 1 { + txreq -url "/foo" + delay .2 + barrier b2 sync + rxrst + expect rst.err == REFUSED_STREAM + } -start + + stream 3 { + delay 1 + txreq -url "/foo" + rxresp + } -run + + stream 1 -wait +} -run + +client c1 -wait + +varnish v1 -vsl_catchup +varnish v1 -expect busy_sleep >= 1 +varnish v1 -expect busy_wakeup == 0 +varnish v1 -expect busy_killed == 1 From phk at FreeBSD.org Wed Mar 14 11:01:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 11:01:11 +0000 (UTC) Subject: [master] b0e5182 Make the -b VCL-snippet use syntax 4.1 Message-ID: <20180314110111.3E239A0F35@lists.varnish-cache.org> commit b0e5182967cca22e45bcf91ae926b6c5543e4565 Author: Poul-Henning Kamp Date: Wed Mar 14 10:46:36 2018 +0000 Make the -b VCL-snippet use syntax 4.1 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 4338fb0..05b4c61 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -592,7 +592,7 @@ main(int argc, char * const *argv) REPLACE(fa->farg, "<-b argument>"); vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "vcl 4.0;\n"); + VSB_printf(vsb, "vcl 4.1;\n"); VSB_printf(vsb, "backend default {\n"); if (*optarg != '/') VSB_printf(vsb, " .host = \"%s\";\n", From phk at FreeBSD.org Wed Mar 14 11:01:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 11:01:11 +0000 (UTC) Subject: [master] 3ac9ecb Only allow UDS backends in syntax >= 4.1 Message-ID: <20180314110111.5ADB3A0F38@lists.varnish-cache.org> commit 3ac9ecb76e6985ef031336b6f8a162bca91fef41 Author: Poul-Henning Kamp Date: Wed Mar 14 10:55:22 2018 +0000 Only allow UDS backends in syntax >= 4.1 diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 640481d..3f21cdb 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -365,6 +365,15 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) vcc_NextToken(tl); SkipToken(tl, ';'); } else if (vcc_IdIs(t_field, "path")) { + if (tl->syntax < VCL_41) { + VSB_printf(tl->sb, + "Unix socket backends only supported" + " in VCL4.1 and higher.\n"); + vcc_ErrToken(tl, tl->t); + VSB_printf(tl->sb, " at "); + vcc_ErrWhere(tl, tl->t); + return; + } vcc_Redef(tl, "Address", &t_did, t_field); ERRCHK(tl); ExpectErr(tl, CSTR); From phk at FreeBSD.org Wed Mar 14 11:01:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 11:01:11 +0000 (UTC) Subject: [master] 6187f45 UDS listen sockets restrict VCL to >= 4.1 Message-ID: <20180314110111.74DEDA0F3D@lists.varnish-cache.org> commit 6187f45a3b598c6761c7b7849bcab28f6d6310bb Author: Poul-Henning Kamp Date: Wed Mar 14 10:55:41 2018 +0000 UDS listen sockets restrict VCL to >= 4.1 I had hoped for a more explanatory error message, but that is too involved at this point. diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index bffd461..796a11b 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -268,6 +268,9 @@ MAC_Arg(const char *spec) ARGV_ERR("Unix domain socket addresses must be absolute paths " "in -a (%s)\n", la->endpoint); + if (*la->endpoint == '/' && heritage.min_vcl < 41) + heritage.min_vcl = 41; + for (int i = 2; av[i] != NULL; i++) { char *eq, *val; int len; From phk at FreeBSD.org Wed Mar 14 11:01:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 11:01:11 +0000 (UTC) Subject: [master] 0de7f7f Test the vcl4.1 restrictions on UDS Message-ID: <20180314110111.90DEBA0F41@lists.varnish-cache.org> commit 0de7f7f6e4feae3df57ea05909cb05e0fff35bde Author: Poul-Henning Kamp Date: Wed Mar 14 10:56:31 2018 +0000 Test the vcl4.1 restrictions on UDS diff --git a/bin/varnishtest/tests/a00019.vtc b/bin/varnishtest/tests/a00019.vtc index 864ab21..a6d1317 100644 --- a/bin/varnishtest/tests/a00019.vtc +++ b/bin/varnishtest/tests/a00019.vtc @@ -2,6 +2,14 @@ varnishtest "vtc v_* macros when the listen address is UDS" varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start +varnish v1 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} { + backend default { .host = "${bad_ip}"; } +} + +varnish v1 -syntax 4.0 -errvcl \ + {Unix socket backends only supported in VCL4.1 and higher.} \ + {backend default { .path = "${tmpdir}/v1.sock"; }} + varnish v2 -vcl { backend default { .host = "${bad_ip}"; } From geoff at uplex.de Wed Mar 14 11:38:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 11:38:09 +0000 (UTC) Subject: [master] 4e0da85 Blurb about UDS in "What's New in 6.0". Message-ID: <20180314113809.8DB0DA2DBA@lists.varnish-cache.org> commit 4e0da85ee43c579a21fbfb683520a1afb472525f Author: Geoff Simmons Date: Wed Mar 14 12:37:05 2018 +0100 Blurb about UDS in "What's New in 6.0". diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index 73ea543..9b860fe 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -22,10 +22,57 @@ XXX subsubhead 1.2 XXX: ... -XXX subhead 2 -~~~~~~~~~~~~~ +.. _whatsnew_new_uds: -XXX ... +Unix domain sockets as listen and backend addresses +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using VCL 4.1, the ``varnishd -a`` command-line argument +allows you to specify Unix domain sockets as listener addresses +(beginning with ``/``, see varnishd :ref:`ref-varnishd-options`):: + + varnishd -a /path/to/listen.sock,PROXY,user=vcache,group=varnish,mode=660 + +The ``user``, ``group`` and ``mode`` sub-arguments set the permissions +of the newly created socket file. + +A backend declaration in VCL 4.1 may now include the ``.path`` field +(instead of ``.host``) for the absolute path of a Unix domain socket +to which Varnish connects:: + + backend uds { + .path = "/path/to/backend.sock"; + } + +This of course can only be used to communicate with other processes on +the same host, if they also support socket file addresses. Until now, +connections with other process co-located with Varnish were only +possible over locally available IP addresses, such as loopback. Unix sockets +may have some advantages for such a configuration: + +* Network traffic over Unix sockets does not have the overhead of the + TCP stack. You may see a significant improvement in throughput + compared to using the loopback address. + +* The file permission system can be used to impose restrictions on the + processes that can connect to Varnish, and the processes to which + Varnish can connect. + +* Using paths as addresses may be easier to configure than searching + for open port numbers on loopback, especially for automated + configurations. + +The obvious use cases are SSL offloaders that decrypt traffic from the +network and pass requests on to Varnish, and SSL "onloaders" that +encrypt backend traffic from Varnish and forward requests over +untrusted networks. But this should be useful for any configuration in +which Varnish talks to processes on the same host. + +The distribution has a new :ref:`VMOD unix ` that you +may be able to use in VCL to get information about the credentials of +a process (user and group of the process owner) that is connected to a +Varnish listener over a Unix socket. This is not supported on every +platform, so check the VMOD docs to see if it will work for you. XXX subsubhead 2.1 ------------------ From daghf at varnish-software.com Wed Mar 14 11:48:09 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 14 Mar 2018 11:48:09 +0000 (UTC) Subject: [master] 0bd7b61 Tidy up: No need for _SAFE Message-ID: <20180314114809.91721A30B7@lists.varnish-cache.org> commit 0bd7b61bab655d64b734ef6918a9b08e65146806 Author: Dag Haavi Finstad Date: Wed Mar 14 12:46:38 2018 +0100 Tidy up: No need for _SAFE diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index d4ae479..39d97e6 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -857,7 +857,7 @@ static h2_error h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f) { - struct h2_req *r2 = NULL, *r22; + struct h2_req *r2 = NULL; h2_error h2e; ASSERT_RXTHR(h2); @@ -881,10 +881,9 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, return (H2CE_PROTOCOL_ERROR); } - VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { + VTAILQ_FOREACH(r2, &h2->streams, list) if (r2->stream == h2->rxf_stream) break; - } if (r2 == NULL && h2f->act_sidle == 0) { if (h2->rxf_stream <= h2->highest_stream) From geoff at uplex.de Wed Mar 14 12:52:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 12:52:09 +0000 (UTC) Subject: [master] 4f8c20d Document the VCL >= 4.1 restriction for -a/-b /uds in varnishd(1). Message-ID: <20180314125209.2C057A4363@lists.varnish-cache.org> commit 4f8c20db5e96009543be6a1c0a1244145f69c791 Author: Geoff Simmons Date: Wed Mar 14 13:50:28 2018 +0100 Document the VCL >= 4.1 restriction for -a/-b /uds in varnishd(1). diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 9651e1d..b6d383a 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -51,7 +51,9 @@ Basic options user, group and mode sub-arguments may be used to specify the permissions of the socket file -- use names for user and group, and a 3-digit octal value for mode. These sub-arguments are not - permitted if an IP address is specified. + permitted if an IP address is specified. When Unix domain socket + listeners are in use, all VCL configurations must have version >= + 4.1. Name is referenced in logs. If name is not specified, "a0", "a1", etc. is used. An additional protocol type can be set for the @@ -70,6 +72,8 @@ Basic options absolute path of a Unix domain socket to which Varnish connects. In that case, the value of ``-b`` must satisfy the conditions required for the ``.path`` field of a backend declaration, see :ref:`vcl(7)`. + Backends with Unix socket addresses may only be used with VCL + versions >= 4.1. -b can be used only once, and not together with f. From geoff at uplex.de Wed Mar 14 12:58:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 12:58:09 +0000 (UTC) Subject: [master] a648591 Document backend.path restriction to VCL >= 4.1 in vcl(7). Message-ID: <20180314125809.76421A4546@lists.varnish-cache.org> commit a6485919dd06f858054d0e19f3c9be42e034561a Author: Geoff Simmons Date: Wed Mar 14 13:56:44 2018 +0100 Document backend.path restriction to VCL >= 4.1 in vcl(7). diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 9275b57..d2eec3d 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -221,11 +221,12 @@ parameters. The following attributes are available: single IP address. This attribute is mandatory, unless ``.path`` is declared. - ``.path`` + ``.path`` (``VCL >= 4.1``) The absolute path of a Unix domain socket at which a backend is listening. The file at that path must exist and must be accessible to Varnish at VCL load time, and it must be a socket. One of - ``.path`` or ``.host`` must be declared (but not both). + ``.path`` or ``.host`` must be declared (but not both). ``.path`` + may only be used in VCL since version 4.1. ``.port`` The port on the backend that Varnish should connect to. Ignored if From martin at varnish-software.com Wed Mar 14 13:05:09 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 14 Mar 2018 13:05:09 +0000 (UTC) Subject: [master] d6a127b Fix memory leak of vary string on stevedore alloc fail Message-ID: <20180314130509.7EEF4A47A1@lists.varnish-cache.org> commit d6a127b09ef6a241c20016e567a65a64a8af5ed4 Author: Martin Blix Grydeland Date: Wed Mar 14 13:52:36 2018 +0100 Fix memory leak of vary string on stevedore alloc fail If the stevedore failed the object creation, we would leak the temporary VSB holding the computed vary string. This patch frees it. Problem exists in 4.1 and later. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index f64cfb4..69dcfcc 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -129,8 +129,12 @@ vbf_beresp2obj(struct busyobj *bo) if (bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; - if (!vbf_allocobj(bo, l)) + if (!vbf_allocobj(bo, l)) { + if (vary != NULL) + VSB_destroy(&vary); + AZ(vary); return (-1); + } if (vary != NULL) { AN(ObjSetAttr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl, From geoff at uplex.de Wed Mar 14 13:13:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 13:13:07 +0000 (UTC) Subject: [master] c4f7d10 Elaborate on the UDS restriction to 4.1 in "Upgrading to 6.0". Message-ID: <20180314131307.49F92A4A4C@lists.varnish-cache.org> commit c4f7d10748b9764567ab41d0a5b95a5ffa818953 Author: Geoff Simmons Date: Wed Mar 14 14:11:14 2018 +0100 Elaborate on the UDS restriction to 4.1 in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index f801aa8..aa551cc 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -49,8 +49,13 @@ access the socket file. So you can reliably restrict access by restricting permissions on the directory containing the socket (but that must be done outside of the Varnish configuration). +When UDS listeners are in use, VCL >= 4.1 will be required for all VCL +programs loaded by Varnish. If you attempt to load a VCL source with +``vcl 4.0;``, the load will fail with a message that the version is +not supported. + If you continue using only IP addresses in your ``-a`` arguments, you -won't have to change them. +won't have to change them, and you can continue using VCL 4.0. Unix domain sockets as backend addresses ======================================== @@ -84,6 +89,10 @@ The path of a socket file may also be specified in the The value of ``-b`` must fulfill the same conditions as the ``.path`` field in a backend declaration. +Backends with the ``.path`` specification require VCL 4.1, as do paths +with the ``-b`` argument. If you don't use UDS backends, you can +continue using VCL 4.0. + varnishd parameters =================== From geoff at uplex.de Wed Mar 14 16:10:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 16:10:09 +0000 (UTC) Subject: [master] e89586b Add proxy/cache_proxy.h to varnishd_SOURCES. Message-ID: <20180314161009.CD170A7E72@lists.varnish-cache.org> commit e89586b69f919d4f836db10a33ad2136cff1340c Author: Geoff Simmons Date: Wed Mar 14 16:52:54 2018 +0100 Add proxy/cache_proxy.h to varnishd_SOURCES. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index a033001..739b16c 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -90,6 +90,7 @@ varnishd_SOURCES = \ mgt/mgt_vcc.c \ mgt/mgt_vcl.c \ proxy/cache_proxy_proto.c \ + proxy/cache_proxy.h \ storage/mgt_stevedore.c \ storage/stevedore.c \ storage/stevedore_utils.c \ From geoff at uplex.de Wed Mar 14 16:10:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 16:10:09 +0000 (UTC) Subject: [master] 36e6d35 Line length OCD. Message-ID: <20180314161009.E3625A7E79@lists.varnish-cache.org> commit 36e6d351ae4dc4893aa1ea81ca203de09bd78955 Author: Geoff Simmons Date: Wed Mar 14 17:00:33 2018 +0100 Line length OCD. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 196a273..1f9410f 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -280,8 +280,10 @@ VPX_tlv(const struct req *req, int tlv, void **dst, int *len) if (ssltlv) { char *sd; int sl; - sd = d + sizeof(struct pp2_tlv) + sizeof(struct pp2_tlv_ssl); - sl = l - sizeof(struct pp2_tlv) - sizeof(struct pp2_tlv_ssl); + sd = d + sizeof(struct pp2_tlv) + + sizeof(struct pp2_tlv_ssl); + sl = l - sizeof(struct pp2_tlv) - + sizeof(struct pp2_tlv_ssl); while (sl > sizeof(struct pp2_tlv)) { uint16_t subv_len = vbe16dec(sd + 1); if (sd[0] == ssltlv) { @@ -445,7 +447,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) while (l > sizeof(struct pp2_tlv)) { int el = vbe16dec(d + 1) + 3; if (el > l) { - VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: Ignoring TLV"); + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: Ignoring TLV"); return (0); } switch(d[0]) { @@ -454,7 +457,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) uint32_t n_crc32c = vbe32dec(d+3); *(d+3) = 0; *(d+4) = 0; *(d+5) = 0; *(d+6) = 0; if (crc32c(p, hdr_len) != n_crc32c) { - VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: CRC error"); + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: CRC error"); return (-1); } break; @@ -463,12 +467,15 @@ vpx_proto2(const struct worker *wrk, struct req *req) { const char *sd; int sl; - sd = d + sizeof(struct pp2_tlv) + sizeof(struct pp2_tlv_ssl); - sl = l - sizeof(struct pp2_tlv) - sizeof(struct pp2_tlv_ssl); + sd = d + sizeof(struct pp2_tlv) + + sizeof(struct pp2_tlv_ssl); + sl = l - sizeof(struct pp2_tlv) - + sizeof(struct pp2_tlv_ssl); while (sl > sizeof(struct pp2_tlv)) { int esl = vbe16dec(sd + 1) + 3; if (esl > sl) { - VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: Ignoring SSL TLV"); + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: Ignoring SSL TLV"); return (0); } sd += esl; @@ -481,7 +488,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) l -= el; } if (l) { - VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: header length mismatch"); + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: header length mismatch"); return (0); } if (tlv_len && WS_Reserve(req->sp->ws, 2 + tlv_len)) { From geoff at uplex.de Wed Mar 14 16:10:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 16:10:10 +0000 (UTC) Subject: [master] bbf079a Fix VMOD proxy title line. Message-ID: <20180314161010.0A8D1A7E84@lists.varnish-cache.org> commit bbf079ad6051139653bb5d6e5b44c3b486aab4b6 Author: Geoff Simmons Date: Wed Mar 14 17:04:40 2018 +0100 Fix VMOD proxy title line. diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod.vcc index c3774db..d986b52 100644 --- a/lib/libvmod_proxy/vmod.vcc +++ b/lib/libvmod_proxy/vmod.vcc @@ -25,7 +25,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module proxy 3 Varnish Standard Module +$Module proxy 3 Varnish Module to extract TLV attributes from PROXYv2 $ABI strict DESCRIPTION From geoff at uplex.de Wed Mar 14 16:50:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 16:50:10 +0000 (UTC) Subject: [master] d396134 Delete unused fields from the TLV structs. Message-ID: <20180314165010.E9D24A8A9F@lists.varnish-cache.org> commit d3961346254153341c9d38acb29fe808752f8e00 Author: Geoff Simmons Date: Wed Mar 14 17:49:36 2018 +0100 Delete unused fields from the TLV structs. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 1f9410f..8aea3c7 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -160,13 +160,11 @@ struct pp2_tlv { uint8_t type; uint8_t length_hi; uint8_t length_lo; - uint8_t value[0]; }__attribute__((packed)); struct pp2_tlv_ssl { uint8_t client; uint32_t verify; - struct pp2_tlv sub_tlv[0]; }__attribute__((packed)); static const char vpx2_sig[] = { From dridi at varni.sh Wed Mar 14 17:58:11 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 14 Mar 2018 18:58:11 +0100 Subject: [master] d396134 Delete unused fields from the TLV structs. In-Reply-To: <20180314165010.E9D24A8A9F@lists.varnish-cache.org> References: <20180314165010.E9D24A8A9F@lists.varnish-cache.org> Message-ID: On Wed, Mar 14, 2018 at 5:50 PM, Geoff Simmons wrote: > > commit d3961346254153341c9d38acb29fe808752f8e00 > Author: Geoff Simmons > Date: Wed Mar 14 17:49:36 2018 +0100 > > Delete unused fields from the TLV structs. > > diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c > index 1f9410f..8aea3c7 100644 > --- a/bin/varnishd/proxy/cache_proxy_proto.c > +++ b/bin/varnishd/proxy/cache_proxy_proto.c > @@ -160,13 +160,11 @@ struct pp2_tlv { > uint8_t type; > uint8_t length_hi; > uint8_t length_lo; > - uint8_t value[0]; > }__attribute__((packed)); > > struct pp2_tlv_ssl { > uint8_t client; > uint32_t verify; > - struct pp2_tlv sub_tlv[0]; > }__attribute__((packed)); > > static const char vpx2_sig[] = { I said I would review the vmod-proxy patch set but also said you shouldn't wait for me, so to be clear I'm not complaining as I'm discovering this after the facts. My understanding is that PHK reviewed and OK'd this change, but this is also the first occurrence of packed structs in the code base to my knowledge. Since they are usually meant to ease [de]serialization of data (one read(2) and you're done) we may need to keep the fields even if they aren't used. I have yet to look at the patch set. Dridi From dridi at varni.sh Wed Mar 14 18:18:54 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 14 Mar 2018 19:18:54 +0100 Subject: [master] e89586b Add proxy/cache_proxy.h to varnishd_SOURCES. In-Reply-To: <20180314161009.CD170A7E72@lists.varnish-cache.org> References: <20180314161009.CD170A7E72@lists.varnish-cache.org> Message-ID: On Wed, Mar 14, 2018 at 5:10 PM, Geoff Simmons wrote: > > commit e89586b69f919d4f836db10a33ad2136cff1340c > Author: Geoff Simmons > Date: Wed Mar 14 16:52:54 2018 +0100 > > Add proxy/cache_proxy.h to varnishd_SOURCES. > > diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am > index a033001..739b16c 100644 > --- a/bin/varnishd/Makefile.am > +++ b/bin/varnishd/Makefile.am > @@ -90,6 +90,7 @@ varnishd_SOURCES = \ > mgt/mgt_vcc.c \ > mgt/mgt_vcl.c \ > proxy/cache_proxy_proto.c \ > + proxy/cache_proxy.h \ > storage/mgt_stevedore.c \ > storage/stevedore.c \ > storage/stevedore_utils.c \ I said I would review the vmod-proxy patch set but also said you shouldn't wait for me, but still, I'm complaining as I'm discovering this after the facts. Please don't take the habit of checking in patches that are known to be broken. When we look for regressions with git-bisect it diminishes the benefits of automation when we have to waste eyeballs time on false-negative. Dridi From geoff at uplex.de Wed Mar 14 18:52:12 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 19:52:12 +0100 Subject: [master] d396134 Delete unused fields from the TLV structs. In-Reply-To: References: <20180314165010.E9D24A8A9F@lists.varnish-cache.org> Message-ID: <821314a3-b733-5974-be3c-b2e84b97803b@uplex.de> On 03/14/2018 06:58 PM, Dridi Boukelmoune wrote: >> >> commit d3961346254153341c9d38acb29fe808752f8e00 >> Author: Geoff Simmons >> Date: Wed Mar 14 17:49:36 2018 +0100 >> >> Delete unused fields from the TLV structs. > > My understanding is that PHK reviewed and OK'd this change, but this > is also the first occurrence of packed structs in the code base to my > knowledge. Since they are usually meant to ease [de]serialization of > data (one read(2) and you're done) we may need to keep the fields even > if they aren't used. The problem was that suncc refused to compile: uint8_t value[0]; ... it wouldn't accept the 0 subscript, and it wouldn't tolerate value[] either. So all of the VTESTs were failing. Leaving them out got the VTESTs back to green (ehocdet agreed). No objection to revisiting this after the release. Best, Geoff -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg Tel +49 40 2880 5731 Mob +49 176 636 90917 Fax +49 40 42949753 http://uplex.de -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From geoff at uplex.de Wed Mar 14 18:55:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 19:55:10 +0100 Subject: [master] e89586b Add proxy/cache_proxy.h to varnishd_SOURCES. In-Reply-To: References: <20180314161009.CD170A7E72@lists.varnish-cache.org> Message-ID: On 03/14/2018 07:18 PM, Dridi Boukelmoune wrote: >> >> commit e89586b69f919d4f836db10a33ad2136cff1340c >> Author: Geoff Simmons >> Date: Wed Mar 14 16:52:54 2018 +0100 >> >> Add proxy/cache_proxy.h to varnishd_SOURCES. >> > > Please don't take the habit of checking in patches that are known to > be broken. When we look for regressions with git-bisect it diminishes > the benefits of automation when we have to waste eyeballs time on > false-negative. Understood. I was thinking of keeping my commits separate from ehocdet's for the PR, but yes this one could have been squashed. I'm curious as to why Travis put the green checkmark on the PR. Doesn't Travis run make distcheck? Best, Geoff -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg Tel +49 40 2880 5731 Mob +49 176 636 90917 Fax +49 40 42949753 http://uplex.de -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From phk at FreeBSD.org Wed Mar 14 19:17:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 19:17:13 +0000 (UTC) Subject: [master] d34fd8f Start flexelinting vmod_proxy Message-ID: <20180314191713.62683AD541@lists.varnish-cache.org> commit d34fd8fb68dccad85c71f2df69f53c8ff9770ada Author: Poul-Henning Kamp Date: Wed Mar 14 18:47:03 2018 +0000 Start flexelinting vmod_proxy diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 8f7ecb8..af06e75 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -25,6 +25,8 @@ FLOPS=' ../../lib/libvmod_debug/*.c ../../lib/libvmod_directors/flint.lnt ../../lib/libvmod_directors/*.c + ../../lib/libvmod_proxy/flint.lnt + ../../lib/libvmod_proxy/*.c ../../lib/libvmod_purge/flint.lnt ../../lib/libvmod_purge/*.c ../../lib/libvmod_std/flint.lnt diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 8aea3c7..9ec6f84 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -172,7 +172,7 @@ static const char vpx2_sig[] = { 'Q', 'U', 'I', 'T', '\n', }; -static uint32_t crctable[256] = { +static const uint32_t crctable[256] = { 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, 0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, diff --git a/lib/libvmod_proxy/flint.lnt b/lib/libvmod_proxy/flint.lnt new file mode 100644 index 0000000..e69de29 From geoff at uplex.de Wed Mar 14 19:30:11 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 19:30:11 +0000 (UTC) Subject: [master] 26a39f2 Write up some more varnishd params in "Upgrading to 6.0". Message-ID: <20180314193011.5FD7FAD8FA@lists.varnish-cache.org> commit 26a39f2acdf245f2dd8bb179ed78988c6ed0a889 Author: Geoff Simmons Date: Wed Mar 14 20:28:42 2018 +0100 Write up some more varnishd params in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index aa551cc..b4c4ed3 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -96,7 +96,12 @@ continue using VCL 4.0. varnishd parameters =================== -XXX: ... +The ``cli_buffer`` parameter, which was deprecated as of Varnish 5.2, +is now retired. + +:ref:`ref_param_max_restarts` now works more correctly -- it is the +number of ``return(restart)`` calls permitted per request. (It had +been one less than the number of permitted restarts.) The parameters :ref:`ref_param_tcp_keepalive_intvl`, :ref:`ref_param_tcp_keepalive_probes` and @@ -109,6 +114,15 @@ these parameters with a UDS; you may get error messages in the log for ``accept_filter`` or ``tcp_fastopen`` (with the VSL tag ``Error`` in raw grouping), but they are harmless. +:ref:`ref_param_workspace_thread` is now used for IO buffers during +the delivery of the client response. This space had previously been +taken from :ref:`ref_param_workspace_client`. If you need to reduce +memory footprint, consider reducing ``workspace_client`` by the amount +in ``workspace_thread``. + +Added :ref:`ref_param_esi_iovs`. tl;dr: Don't touch it, unless advised +to do so by someone familiar with the innards of Varnish. + Changes to VCL ============== From geoff at uplex.de Wed Mar 14 19:57:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 19:57:08 +0000 (UTC) Subject: [master] a47bb4f Availability of VCL vars on the client side in "Upgrading to 6.0". Message-ID: <20180314195708.EFCE3AE03B@lists.varnish-cache.org> commit a47bb4f9da0d823acf125b4a9c00562cd6a38746 Author: Geoff Simmons Date: Wed Mar 14 20:56:17 2018 +0100 Availability of VCL vars on the client side in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index b4c4ed3..baff80d 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -225,6 +225,13 @@ of VCL 4.1, but are still available in 4.0. Note that ``beresp.storage_hint`` has been deprecated since Varnish 5.1; you should use ``beresp.storage`` instead. +Client-side variable access +--------------------------- + +``req.storage``, ``req.hash_ignore_busy`` and ``req.hash_always_miss`` +are now accessible from all of the client side subroutines (previously +only in ``vcl_recv{}``). + Unix domain sockets and VCL ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From geoff at uplex.de Wed Mar 14 20:04:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 20:04:07 +0000 (UTC) Subject: [master] 9c7e864 Write up changes concerning restarts in "Upgrading to 6.0". Message-ID: <20180314200407.79543AE24F@lists.varnish-cache.org> commit 9c7e86416ad6c68140651433c7cd626e8bc61f38 Author: Geoff Simmons Date: Wed Mar 14 21:03:26 2018 +0100 Write up changes concerning restarts in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index baff80d..fce4ded 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -450,6 +450,15 @@ VMOD std listener is UDS. :ref:`std.set_ip_tos(INT) ` is silently ignored when the listener is UDS. +Restarts +~~~~~~~~ + +Restarts now leave all of the properties of the client request +unchanged (all of the ``req.*`` variables, including the headers), +except for ``req.restarts`` and ``req.xid``, which change by design. + +``return(restart)`` can now be called from ``vcl_recv{}``. + New VMODs ~~~~~~~~~ From geoff at uplex.de Wed Mar 14 20:15:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 20:15:10 +0000 (UTC) Subject: [master] c661da5 Mention the use of std.rollback() to reset req.http.* in "Upgrading". Message-ID: <20180314201510.E0B47AE59C@lists.varnish-cache.org> commit c661da5e5fc51473da0e0b701075dbaf72c249d3 Author: Geoff Simmons Date: Wed Mar 14 21:13:57 2018 +0100 Mention the use of std.rollback() to reset req.http.* in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index fce4ded..21ddbf0 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -457,6 +457,10 @@ Restarts now leave all of the properties of the client request unchanged (all of the ``req.*`` variables, including the headers), except for ``req.restarts`` and ``req.xid``, which change by design. +If you need to reset the client request headers to their original +state (before changes in VCL), call +:ref:`std.rollback(req) `. + ``return(restart)`` can now be called from ``vcl_recv{}``. New VMODs From geoff at uplex.de Wed Mar 14 20:31:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 20:31:08 +0000 (UTC) Subject: [master] b574701 Write up VMOD proxy in "Upgrading to 6.0". Message-ID: <20180314203108.CD19BAEA48@lists.varnish-cache.org> commit b574701e0436c13111f13f376776783029ce0019 Author: Geoff Simmons Date: Wed Mar 14 21:30:04 2018 +0100 Write up VMOD proxy in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 21ddbf0..03c7274 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -488,6 +488,26 @@ This is not available on every platform. As always, check the documentation and test the code before you attempt something like this in production. +VMOD proxy +---------- + +:ref:`vmod_proxy(3)` provides functions to extract TLV attributes that +may be optionally sent over a PROXYv2 connection to a Varnish listener. +Most of these are properties of the peer component's TLS connection:: + + import proxy; + + # Get the authority attribute -- corresponds to the SNI of a TLS + # connection. + set req.http.Authority = proxy.authority(); + +Not all implementations send TLV attributes, and those that do don't +necessarily support all of them; test your code to see what works in +your configuration. + +See the +`PROXY v2 specification `_ for more information about TLV attributes. + Packaging changes ================= From geoff at uplex.de Wed Mar 14 21:05:11 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 21:05:11 +0000 (UTC) Subject: [master] 2031df5 Write up changes to the shard director in "Upgrading to 6.0". Message-ID: <20180314210511.0A34EAF3FB@lists.varnish-cache.org> commit 2031df54f52044c921460ba0839817e3d734d1a9 Author: Geoff Simmons Date: Wed Mar 14 22:04:09 2018 +0100 Write up changes to the shard director in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 03c7274..833c158 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -450,6 +450,45 @@ VMOD std listener is UDS. :ref:`std.set_ip_tos(INT) ` is silently ignored when the listener is UDS. +The ``shard`` director +---------------------- + +The ``alg`` argument of the shard director's ``.reconfigure()`` and +``.key()`` methods has been removed. The choice of hash algorithms was +experimental, and we have settled on SHA256 as providing the best +dispersal. + +If you have been using other choices of ``alg`` for +``.reconfigure()``, then after upgrading and removing ``alg``, the +sharding of requests to backends will change once and only once. + +If you have been using other values of ``alg`` for ``.key()`` and need +to preserve the previous behavior, see the +`change log `_ +for advice on how to do so. + +With the ``resolve=LAZY`` argument of the ``.backend()`` method, the +shard director will now defer the selection of a a backend to when a +backend connection is actually made, which is how all other bundled +directors work as well. This enables layering the shard director below +other directors -- you can use ``.backend(resolve=LAZY)`` to set the +shard director as a backend for another director. ``resolve=LAZY`` +MUST be used in ``vcl_init`` and on the client side. + +The shard director now provides a ``shard_param`` object that serves +as a store for a set of parameters for the director's ``.backend()`` +method. This makes it possible to re-use a set of parameter values +without having to restate them in every ``.backend()`` call. The +``.backend()`` method has an argument ``param`` whose value, if it is +used, must be returned from the ``shard_param.use()`` method. + +Because of these changes, support for positional arguments of the +shard director ``.backend()`` method had to be removed. In other +words, all parameters to the shard director ``.backend()`` method now +need to be named. + +See :ref:`vmod_directors(3)` for details. + Restarts ~~~~~~~~ From geoff at uplex.de Wed Mar 14 21:21:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 21:21:07 +0000 (UTC) Subject: [master] 4487206 Write up changes to VSL/varnishlog in "Upgrading to 6.0". Message-ID: <20180314212107.EB79AAF8AD@lists.varnish-cache.org> commit 448720620a7bb211c5630f9ea7c12c803bbe812d Author: Geoff Simmons Date: Wed Mar 14 22:20:16 2018 +0100 Write up changes to VSL/varnishlog in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 833c158..60af892 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -678,6 +678,19 @@ Other changes with the backend name field -- the second field in ``BackendOpen``. + * The byte counters logged with ``ReqAcct`` now report the numbers + returned from the operating system telling us how many bytes were + actually sent in a request and response, rather than what Varnish + thought it was going to send. This gives a more accurate account + when there are errors, for example when a client hung up early + without receiving the entire response. The figures also include + any overhead in a request or response body, for example due to + chunked encoding. + + * Debugging logs for the PROXY protocol are turned off by default. + They can be turned on with the ``protocol`` flag of the varnishd + :ref:`ref_param_debug` parameter (``-p debug=+protocol``). + * ``varnishncsa(1)`` * The ``%h`` formatter (remote host) gets its value from From geoff at uplex.de Wed Mar 14 21:32:06 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 21:32:06 +0000 (UTC) Subject: [master] ed9954c Document the addition of cache_hit_grace in "Upgrading". Message-ID: <20180314213206.5EAB3AFB80@lists.varnish-cache.org> commit ed9954c96972a662713de657a5c377023e39b854 Author: Geoff Simmons Date: Wed Mar 14 22:30:56 2018 +0100 Document the addition of cache_hit_grace in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 60af892..7953fa4 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -691,6 +691,12 @@ Other changes They can be turned on with the ``protocol`` flag of the varnishd :ref:`ref_param_debug` parameter (``-p debug=+protocol``). +* ``varnishstat(1)`` + + * Added the counter ``cache_hit_grace`` -- how often objects in the + cache were hit when their TTL had expired, but they were still + in grace. + * ``varnishncsa(1)`` * The ``%h`` formatter (remote host) gets its value from From geoff at uplex.de Wed Mar 14 21:37:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 21:37:08 +0000 (UTC) Subject: [master] f0263e6 Another note about varnishncsa in "Upgrading". Message-ID: <20180314213708.6FCC3AFD7F@lists.varnish-cache.org> commit f0263e6487c82439a7a53823ecc7f01a90ed2a42 Author: Geoff Simmons Date: Wed Mar 14 22:36:14 2018 +0100 Another note about varnishncsa in "Upgrading". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 7953fa4..f7eb4f7 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -723,6 +723,11 @@ Other changes For the backend name, use ``%{VSL:BackendOpen[2]}x`` for backend logs. + * varnishncsa does not accept output format strings (from the ``-F`` + command-line argument or a configuration file) if they specify + tags for log entries whose payloads may contain control or binary + characters. + * ``varnishtest(1)`` and ``vtc(7)``: * The ``client -connect`` and ``server -listen`` commands in vtc From phk at FreeBSD.org Wed Mar 14 21:50:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 21:50:09 +0000 (UTC) Subject: [master] 9d3ec1d Intro paragraph Message-ID: <20180314215009.A77A9B0101@lists.varnish-cache.org> commit 9d3ec1da53cc680470dab7aff3309282c2cc184c Author: Poul-Henning Kamp Date: Wed Mar 14 21:46:06 2018 +0000 Intro paragraph diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index c66386c..253da0b 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1,5 +1,7 @@ -Variables ---------- +.. _vcl_variables: + +VCL Variables +------------- Variables provide read, write and delete access to almost all aspects of the work at hand. diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index 9b860fe..d228df9 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -3,7 +3,42 @@ Changes in Varnish 6.0 ====================== -XXX: ... intro paragraphs ... +Usually when we do dot-zero releases in Varnish, it means that +users are in for a bit of work to upgrade to the new version, +but 6.0 is actually not that scary, because most of the changes +are either under the hood or entirely new features. + +The biggest user-visible change is probably that we, or to be totally +honest here: Geoff Simons, have added support for Unix Domain +Sockets, both for clients and backend servers. + +Because UNIX Domain Sockets have nothing like IP numbers, we were +forced to define a new level of the VCL language ``vcl 4.1`` to +cope with UDS. + +Both ``vcl 4.0`` and ``vcl 4.1`` are supported, and it is the primary +source-file which controls which it will be, and you can ``include`` +lower versions, but not higher versions that that. + +Some old variables are not available in 4.1 and some new variables +are not available in 4.0. Please see :ref:`vcl_variables` for +specifics. + +There are a few other changes to the ``vcl 4.0``, most notably that +we now consider upper- and lower-case the same for symbols. + +There are new and improved VMODs: + +* :ref:`vmod_purge(3)` -- fine-grained and "soft" purges + +* :ref:`vmod_unix(3)` -- Unix Domain Socket information + +* :ref:`vmod_blob(3)` -- Handling of binary blobs (think: Cookies) + +* :ref:`vmod_proxy(3)` -- Proxy protocol information + +* :ref:`vmod_vtc(3)` -- Utility functions for writing :ref:`varnishtest(1)` cases. + .. _whatsnew_new_subhead_1: From phk at FreeBSD.org Wed Mar 14 21:54:06 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 21:54:06 +0000 (UTC) Subject: [master] f3d2704 More M's to Geoff Message-ID: <20180314215406.CEA62B0263@lists.varnish-cache.org> commit f3d2704ac10b3c8bfa1bce13fb40d09b7c1266a8 Author: Poul-Henning Kamp Date: Wed Mar 14 21:53:34 2018 +0000 More M's to Geoff diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index d228df9..0043103 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -9,7 +9,7 @@ but 6.0 is actually not that scary, because most of the changes are either under the hood or entirely new features. The biggest user-visible change is probably that we, or to be totally -honest here: Geoff Simons, have added support for Unix Domain +honest here: Geoff Simmons, have added support for Unix Domain Sockets, both for clients and backend servers. Because UNIX Domain Sockets have nothing like IP numbers, we were @@ -40,6 +40,7 @@ There are new and improved VMODs: * :ref:`vmod_vtc(3)` -- Utility functions for writing :ref:`varnishtest(1)` cases. + .. _whatsnew_new_subhead_1: XXX subhead 1 From phk at FreeBSD.org Wed Mar 14 22:17:05 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 22:17:05 +0000 (UTC) Subject: [master] 3fcccc8 Merge all the UDS news into this file. Message-ID: <20180314221705.C6C47B08A5@lists.varnish-cache.org> commit 3fcccc8a4f314a0fdfdec869ccfe238d720b3d68 Author: Poul-Henning Kamp Date: Wed Mar 14 22:16:00 2018 +0000 Merge all the UDS news into this file. diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index f7eb4f7..b93d79d 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -7,7 +7,7 @@ Upgrading to Varnish 6.0 XXX: Most important change first ================================ -XXX ... +.. _upd_6_0_uds_acceptor: Unix domain sockets as listen addresses ======================================= @@ -57,6 +57,8 @@ not supported. If you continue using only IP addresses in your ``-a`` arguments, you won't have to change them, and you can continue using VCL 4.0. +.. _upd_6_0_uds_backend: + Unix domain sockets as backend addresses ======================================== From geoff at uplex.de Wed Mar 14 22:35:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 22:35:08 +0000 (UTC) Subject: [master] fcb04d9 Add some more "Changes for Developers" to "Upgrading to 6.0". Message-ID: <20180314223508.B5A9FB0DF7@lists.varnish-cache.org> commit fcb04d9d56ea72ae808b01d9231674a1ef758ac6 Author: Geoff Simmons Date: Wed Mar 14 23:33:39 2018 +0100 Add some more "Changes for Developers" to "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index b93d79d..c3cc3bd 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -789,10 +789,30 @@ Other changes other two. Its value is the path when the peer address is a UDS, and NULL otherwise (matching ```` in the latter case). - * XXX ... - * Changes for developers: + * The VRT API version has been bumped to 7.0, and comprises a variety + of new additions and changes. See ``vrt.h`` and the + `change log `_ + for details. + + * There are new rules about including API headers -- some may only + be included once, others must included in a specific order. Only + ``cache.h`` *or* ``vrt.h`` may be included (``cache.h`` includes + ``vrt.h``). See the ``#error`` directives in the headers. + + * VMOD authors can use the ``VRT_VSC_*()`` series of functions and + the new ``vsctool`` to create statistics for a VMOD that will be + displayed by varnishstat. Varnish uses the same technique to + create its counters, so you can look to the core code to see how + it's done. + + * The ``VCL_INT`` and ``VCL_BYTES`` types are now defined to be + strictly 64 bit (rather than leave it to whatever your platform + defines as ``long``). But you may not get that full precision, + for reasons discussed in the + `change log `_. + * As part of VRT version 7.0, the ``path`` field has been added to to ``struct vrt_backend``, which a VMOD can use with ``VRT_new_backend()`` to create a dynamic backend with a UDS @@ -804,8 +824,13 @@ Other changes debug (available in the source tree) illustrates how this can be done. - * VMOD vcc sources may now include a directive ``$Synopsis`` whose - value may be ``auto`` or ``manual``, default ``auto``. + * VMOD vcc sources may now include a directive ``$Prefix``, whose + value is the string prepended to the names of C objects and + functions in the generated C interface (in ``vcc_if.h``). So you + may choose another prefix besides ``vmod_``, if so desired. + + * vcc sources may also include a directive ``$Synopsis`` whose value + may be ``auto`` or ``manual``, default ``auto``. When ``$Synopsis`` is ``auto``, the vmodtool generates a more comprehensive ``SYNOPSIS`` section in the documentation than in @@ -816,6 +841,4 @@ Other changes the generated docs altogether; so you can write the ``SYNOPSIS`` section yourself, if you prefer. - * XXX ... - *eof* From geoff at uplex.de Wed Mar 14 22:37:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 22:37:07 +0000 (UTC) Subject: [master] c9ff40d Grammar fix Message-ID: <20180314223707.396A5B0EAF@lists.varnish-cache.org> commit c9ff40d9041b951c3b88ad0b425781b6478a1e32 Author: Geoff Simmons Date: Wed Mar 14 23:35:49 2018 +0100 Grammar fix diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index c3cc3bd..c1bef35 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -470,7 +470,7 @@ to preserve the previous behavior, see the for advice on how to do so. With the ``resolve=LAZY`` argument of the ``.backend()`` method, the -shard director will now defer the selection of a a backend to when a +shard director will now defer the selection of a backend to when a backend connection is actually made, which is how all other bundled directors work as well. This enables layering the shard director below other directors -- you can use ``.backend(resolve=LAZY)`` to set the From geoff at uplex.de Wed Mar 14 22:41:06 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 22:41:06 +0000 (UTC) Subject: [master] 9a9bdf8 Mention the return of umem in "Upgrading to 6.0". Message-ID: <20180314224106.D90FFB1046@lists.varnish-cache.org> commit 9a9bdf8d84e43245d8b32929a70a240930e1bdd2 Author: Geoff Simmons Date: Wed Mar 14 23:39:48 2018 +0100 Mention the return of umem in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index c1bef35..2bb277d 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -661,6 +661,12 @@ of what you will be able to do once VIP 20 is completed. Other changes ============= +* ``varnishd(1)``: + + * The ``umem`` storage allocator, which was removed as of Varnish + 5.1, has been restored and is now the default on a system where + ``libumem`` is available (SunOS and descendants). + * ``varnishlog(1)``: * Added a third field to the ``ReqStart`` log record that contains the From geoff at uplex.de Wed Mar 14 22:53:05 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 22:53:05 +0000 (UTC) Subject: [master] 7ecac97 Update changes.rst. Message-ID: <20180314225305.60E37B13F1@lists.varnish-cache.org> commit 7ecac9749e882d126090a81346121d891a4ef1ec Author: Geoff Simmons Date: Wed Mar 14 23:51:49 2018 +0100 Update changes.rst. diff --git a/doc/changes.rst b/doc/changes.rst index 30fb551..57a4b66 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -30,6 +30,9 @@ Usage used during ESI delivery. It should not be tuned unless advised by a developer. +* Support Unix domain sockets for the ``-a`` and ``-b`` command-line + arguments, and for backend declarations. This requires VCL >= 4.1. + VCL and bundled VMODs --------------------- @@ -53,6 +56,12 @@ VCL and bundled VMODs retrieve it back. Doing so will now yield the last successfully set stevedore or the undefined (``NULL``) string. +* IP-valued elements of VCL are equivalent to ``0.0.0.0:0`` when the + connection in question was addressed as a UDS. This is implemented + with the ``bogo_ip`` in ``vsa.c``. + +* ``beresp.backend.ip`` is retired as of VCL 4.1. + * workspace overflows in ``std.log()`` now trigger a VCL failure * workspace overflows in ``std.syslog()`` are ignored @@ -138,6 +147,9 @@ VCL and bundled VMODs rounded) and may even have their sign changed or trigger a C compiler warning / error. +* Add VMOD unix. + +* Add VMOD proxy. Logging / statistics -------------------- @@ -163,6 +175,13 @@ bundled tools ``%{H2RxHdr}x``, ``%{H2RxBody}x``, ``%{H2TxHdr}x``, ``%{H2TxBody}x``, ``%{Debug}x``, ``%{HttpGarbage}x`` and ``%{Hash}x`` +* The vtc ``server -listen`` command supports UDS addresses, as does + the ``client -connect`` command. vtc ``remote.path`` and + ``remote.port`` have the values ``0.0.0.0`` and ``0`` when the peer + address is UDS. Added ``remote.path`` to vtc, whose value is the + path when the address is UDS, and NULL (matching ) for IP + addresses. + C APIs (for vmod and utility authors) ------------------------------------- @@ -190,6 +209,13 @@ C APIs (for vmod and utility authors) * vcc files can now contain a ``$Prefix`` stanza to define the prefix for vmod function names (which was fixed to ``vmod`` before) +* vcc files can contain a ``$Synopsis`` stanza with one of the values + ``auto`` or ``manual``, default ``auto``. With ``auto``, a more + comprehensive SYNOPSIS is generated in the doc output with an + overview of objects, methods, functions and their signatures. With + ``manual``, the auto-SYNOPSIS is left out, for VMOD authors who + prefer to write their own. + * All varnish internal ``SHA256*`` symbols have been renamed to ``VSHA256*`` From phk at FreeBSD.org Wed Mar 14 23:01:05 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Mar 2018 23:01:05 +0000 (UTC) Subject: [master] 101c9aa Probably as finished as it will ever be Message-ID: <20180314230105.D37E1B1641@lists.varnish-cache.org> commit 101c9aa8fe03ea76007d1a0c014d9296f48f9d82 Author: Poul-Henning Kamp Date: Wed Mar 14 23:00:05 2018 +0000 Probably as finished as it will ever be diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index 0043103..26b3707 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -9,8 +9,9 @@ but 6.0 is actually not that scary, because most of the changes are either under the hood or entirely new features. The biggest user-visible change is probably that we, or to be totally -honest here: Geoff Simmons, have added support for Unix Domain -Sockets, both for clients and backend servers. +honest here: Geoff Simmons (UPLEX), have added support for Unix Domain +Sockets, both :ref:`for clients ` and for +:ref:`backend servers `. Because UNIX Domain Sockets have nothing like IP numbers, we were forced to define a new level of the VCL language ``vcl 4.1`` to @@ -18,7 +19,7 @@ cope with UDS. Both ``vcl 4.0`` and ``vcl 4.1`` are supported, and it is the primary source-file which controls which it will be, and you can ``include`` -lower versions, but not higher versions that that. +lower versions, but not higher versions than that. Some old variables are not available in 4.1 and some new variables are not available in 4.0. Please see :ref:`vcl_variables` for @@ -27,109 +28,118 @@ specifics. There are a few other changes to the ``vcl 4.0``, most notably that we now consider upper- and lower-case the same for symbols. -There are new and improved VMODs: - -* :ref:`vmod_purge(3)` -- fine-grained and "soft" purges +The HTTP/2 code has received a lot of attention from Dag Haavi +Finstad (Varnish Software) and it holds up in production on several +large sites now. -* :ref:`vmod_unix(3)` -- Unix Domain Socket information +There are new and improved VMODs: -* :ref:`vmod_blob(3)` -- Handling of binary blobs (think: Cookies) +* :ref:`vmod_directors(3)` -- Much work on the ``shard`` director * :ref:`vmod_proxy(3)` -- Proxy protocol information -* :ref:`vmod_vtc(3)` -- Utility functions for writing :ref:`varnishtest(1)` cases. - - +* :ref:`vmod_unix(3)` -- Unix Domain Socket information -.. _whatsnew_new_subhead_1: +* :ref:`vmod_vtc(3)` -- Utility functions for writing :ref:`varnishtest(1)` cases. -XXX subhead 1 -~~~~~~~~~~~~~ +The ``umem`` stevedore has been brought back on Solaris +and it is the default storage method there now. -XXX ... +More error situations now get vcl ``failure`` handling, +this should make life simpler for everybody we hope. -XXX subsubhead 1.1 ------------------- +And it goes without saying that we have fixed a lot of bugs too. -XXX: ... -XXX subsubhead 1.2 ------------------- +Under the hood (mostly for developers) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -XXX: ... +The big thing is that the ``$Abi [vrt|strict]`` should now +have settled. We have removed all the stuff from ```` +which is not available under ``$Abi vrt``, and this hopefully +means that VMODS will work without recompilation on several +subsequent varnish versions. (There are some stuff related +to packaging which takes advantage of this, but it didn't +get into this release.) -.. _whatsnew_new_uds: +VMODS can define their own stats counters now, and they work +just like builtin counters, because there is no difference. -Unix domain sockets as listen and backend addresses -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The counters are described in a ``.vsc`` file which is +processed with a new python script which does a lot of +magic etc. There is a tiny example in ``vmod_debug`` in +the source tree. -If you are using VCL 4.1, the ``varnishd -a`` command-line argument -allows you to specify Unix domain sockets as listener addresses -(beginning with ``/``, see varnishd :ref:`ref-varnishd-options`):: +This took a major retooling of the stats counters in general, and +the VSM, VSC and VSL apis have all subtly or not so subtly changed +as a result. - varnishd -a /path/to/listen.sock,PROXY,user=vcache,group=varnish,mode=660 +VMOD functions can take optional arguments, these are different +from defaulted arguments in that a separate flag tells if they +were specified or not in the call. For reasons of everybodys +sanity, all the arguments gets wrapped in a function-specific +structure when this is used. -The ``user``, ``group`` and ``mode`` sub-arguments set the permissions -of the newly created socket file. +The ``vmodtool.py`` script has learned other new tricks, and +as a result also produces nicer ``.rst`` output. -A backend declaration in VCL 4.1 may now include the ``.path`` field -(instead of ``.host``) for the absolute path of a Unix domain socket -to which Varnish connects:: +We have a new way of passing text-string arguments around +called ``STRANDS``. Instead of the ``stdarg.h`` representation, +an ``argc+argv`` struct is passed, which means that a function +can take multiple STRANDS arguments, which again means that +many functions will not need to reassemble strings on a +workspace any more, most notably string comparisons. - backend uds { - .path = "/path/to/backend.sock"; - } +VCL types ``INT`` and ``BYTES`` are now 64bits on all platforms. -This of course can only be used to communicate with other processes on -the same host, if they also support socket file addresses. Until now, -connections with other process co-located with Varnish were only -possible over locally available IP addresses, such as loopback. Unix sockets -may have some advantages for such a configuration: +VCL ENUM have gotten a new implementation, so the pointers +are now constant and can be compared as such, rather than +with ``strcmp(3)``. -* Network traffic over Unix sockets does not have the overhead of the - TCP stack. You may see a significant improvement in throughput - compared to using the loopback address. +We have a new type of ``binary`` VSL records which are hexdumped +by default, but on the API side, rather than in ``varnishd``. +This saves both VSL bandwidth and processing power, as they are +usually only used for deep debugging and mostly turned off. -* The file permission system can be used to impose restrictions on the - processes that can connect to Varnish, and the processes to which - Varnish can connect. +The ``VCC`` compilers has received a lot of work in two areas: -* Using paths as addresses may be easier to configure than searching - for open port numbers on loopback, especially for automated - configurations. +The symbol table has been totally revamped to make it ready for +variant symbols, presently symbols which are different in +``vcl 4.0`` and ``vcl 4.1``. -The obvious use cases are SSL offloaders that decrypt traffic from the -network and pass requests on to Varnish, and SSL "onloaders" that -encrypt backend traffic from Varnish and forward requests over -untrusted networks. But this should be useful for any configuration in -which Varnish talks to processes on the same host. +The "prototype" information in the VMOD shared library has been +changed to JSON, (look in your vcc_if.c file if you don't belive +me), and this can express more detailed information, presently +the optional arguments. -The distribution has a new :ref:`VMOD unix ` that you -may be able to use in VCL to get information about the credentials of -a process (user and group of the process owner) that is connected to a -Varnish listener over a Unix socket. This is not supported on every -platform, so check the VMOD docs to see if it will work for you. +The stuff only we care about +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -XXX subsubhead 2.1 ------------------- +Varnishtest's ``process`` has grown ``pty(4)`` support, so that +we can test curses-based programs like our own utilities. -XXX: ... +This has (finally!) pushed our code coverage, across all the +source code in the project up to 90%. -News for authors of VMODs and Varnish API client applications -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +We have also decided to make our python scripts PEP8 compliant, and +``vmodtool.py`` is already be there. -.. _whatsnew_dev_subhead_1: +The VCL variables are now defined in the ``.rst`` file, rather +than the other way around, this makes the documentation better +at the cost of minor python-script complexity. -XXX dev subhead 1 ------------------ +We now produce weekly snapshots from ``-trunk``, this makes it +easier for people to test all the new stuff. -XXX ... +We have not quite gotten the half-yearly release-procedure under +control. -.. _whatsnew_dev_subhead_2: +I'm writing this the evening before the release, trying to squeeze +out of my brain what I should have written here long time ago, +and we have had far more commits this last week than is reasonable. -XXX dev subhead 2 ------------------ +But we *have* gotten better at it. -XXX ... +Really! *eof* From geoff at uplex.de Wed Mar 14 23:04:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 23:04:07 +0000 (UTC) Subject: [master] 98965de Write up VCL 4.0 -> 4.1 in "Upgrading to 6.0". Message-ID: <20180314230407.E361DB1732@lists.varnish-cache.org> commit 98965de4257bbcf72f14f429d4ff470842837309 Author: Geoff Simmons Date: Thu Mar 15 00:02:39 2018 +0100 Write up VCL 4.0 -> 4.1 in "Upgrading to 6.0". diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 2bb277d..c13889d 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -128,7 +128,20 @@ to do so by someone familiar with the innards of Varnish. Changes to VCL ============== -XXX: ... intro paragraph +VCL 4.0 and 4.1 +~~~~~~~~~~~~~~~ + +The first line of code in a VCL program may now be either ``vcl 4.0;`` +or ``vcl 4.1;``, establishing the version of the language for that +instance of VCL. Varnish 6.0 supports both versions. + +The VCL version mainly affects which variables may be used in your VCL +program, or in some cases, whether the variable is writable or +read-only. Only VCL 4.1 is permitted when Unix domain sockets are in +use. + +For details, see :ref:`vcl_variables`, and the notes in the present +document. VCL variables ~~~~~~~~~~~~~ From geoff at uplex.de Wed Mar 14 23:14:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 14 Mar 2018 23:14:07 +0000 (UTC) Subject: [master] ca062d6 Add an example of varnishd -a /uds usage. Message-ID: <20180314231407.D8E18B1A36@lists.varnish-cache.org> commit ca062d6b99aa4d39154ecd00599dc5201d0bae35 Author: Geoff Simmons Date: Thu Mar 15 00:13:15 2018 +0100 Add an example of varnishd -a /uds usage. And remove the last XXX from the docs. diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index c13889d..973ac3b 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -4,9 +4,6 @@ Upgrading to Varnish 6.0 %%%%%%%%%%%%%%%%%%%%%%%% -XXX: Most important change first -================================ - .. _upd_6_0_uds_acceptor: Unix domain sockets as listen addresses @@ -18,6 +15,10 @@ begins with ``/`` (see varnishd :ref:`ref-varnishd-options`):: -a [name=][address][:port][,PROTO][,user=][,group=][,mode=] +For example:: + + varnishd -a /path/to/listen.sock,PROXY,user=vcache,group=varnish,mode=660 + That means that an absolute path must always be specified for the socket file. The socket file is created when Varnish starts, and any file that may exist at that path is unlinked first. You can use the From hermunn at varnish-software.com Wed Mar 14 23:54:07 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 14 Mar 2018 23:54:07 +0000 (UTC) Subject: [master] 1d58ae2 Bug fixes between 5.2 and 6.0 Message-ID: <20180314235407.B7B72B253E@lists.varnish-cache.org> commit 1d58ae299f90a93af00fcf27848f23853398d407 Author: P?l Hermunn Johansen Date: Thu Mar 15 00:34:05 2018 +0100 Bug fixes between 5.2 and 6.0 We do not list bugs that were both introduced and fixed between the two versions. diff --git a/doc/changes.rst b/doc/changes.rst index 57a4b66..b96424b 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -159,6 +159,8 @@ Logging / statistics * added ``cache_hit_grace`` statistics counter +* added ``n_lru_limited`` counter + * The byte counters in ReqAcct now show the numbers reported from the operating system rather than what we anticipated to send. This will give more accurate numbers when e.g. the client hung up early without @@ -259,13 +261,95 @@ Fixed bugs which may influence VCL behaviour Fixed bugs ---------- +* Honor first_byte_timeout for recycled backend connections. (1772_) + +* Limit backend connection retries to a single retry (2135_) + +* H2: Move the req-specific PRIV pointers to struct req. (2268_) + +* H2: Don't panic if we reembark with a request body (2305_) + +* Clear the objcore attributes flags when (re)initializing an stv object. (2319_) + +* H2: Fail streams with missing :method or :path. (2351_) + +* H2: Enforce sequence requirement of header block frames. (2387_) + +* H2: Hold the sess mutex when evaluating r2->cond. (2434_) + +* Use the idle read timeout only on empty requests. (2492_) + +* OH leak in http1_reembark. (2495_) + +* Fix objcore reference count leak. (2502_) + +* Close a race between backend probe and vcl.state=Cold by removing + the be->vsc under backend mtx. (2505_) + +* Fail gracefully if shard.backend() is called in housekeeping subs (2506_) + +* Fix issue #1799 for keep. (2519_) + +* oc->last_lru as float gives too little precision. (2527_) + +* H2: Don't HTC_RxStuff with a non-reserved workspace. (2539_) + +* Various optimalizations of VSM. (2430_, 2470_, 2518_, 2535_, 2541_, 2545_, 2546_) + * Problems during late socket initialization performed by the Varnish child process can now be reported back to the management process with an error message. (2551_) -.. _2551: https://github.com/varnishcache/varnish-cache/issues/2551 +* Fail if ESI is attempted on partial (206) objects. + +* Assert error in ban_mark_completed() - ban lurker edge case. (2556_) + +* Accurate byte counters (2558_). See Logging / statistics above. + +* H2: Fix reembark failure handling. (2563_ and 2592_) -**TODO** +* Working directory permissions insufficient when starting with + umask 027. (2570_) + +* Always use HTTP/1.1 on backend connections for pass & fetch. (2574_) + +* EPIPE is a documented errno in tcp(7) on linux. (2582_) + +* H2: Handle failed write(2) in h2_ou_session. (2607_) + +.. _1772: https://github.com/varnishcache/varnish-cache/issues/1772 +.. _2135: https://github.com/varnishcache/varnish-cache/pull/2135 +.. _2268: https://github.com/varnishcache/varnish-cache/issues/2268 +.. _2305: https://github.com/varnishcache/varnish-cache/issues/2305 +.. _2319: https://github.com/varnishcache/varnish-cache/issues/2319 +.. _2351: https://github.com/varnishcache/varnish-cache/issues/2351 +.. _2387: https://github.com/varnishcache/varnish-cache/issues/2387 +.. _2430: https://github.com/varnishcache/varnish-cache/issues/2430 +.. _2434: https://github.com/varnishcache/varnish-cache/issues/2434 +.. _2470: https://github.com/varnishcache/varnish-cache/issues/2470 +.. _2492: https://github.com/varnishcache/varnish-cache/issues/2492 +.. _2495: https://github.com/varnishcache/varnish-cache/issues/2495 +.. _2502: https://github.com/varnishcache/varnish-cache/issues/2502 +.. _2505: https://github.com/varnishcache/varnish-cache/issues/2505 +.. _2506: https://github.com/varnishcache/varnish-cache/issues/2506 +.. _2518: https://github.com/varnishcache/varnish-cache/issues/2518 +.. _2519: https://github.com/varnishcache/varnish-cache/pull/2519 +.. _2527: https://github.com/varnishcache/varnish-cache/issues/2527 +.. _2535: https://github.com/varnishcache/varnish-cache/issues/2535 +.. _2539: https://github.com/varnishcache/varnish-cache/issues/2539 +.. _2541: https://github.com/varnishcache/varnish-cache/issues/2541 +.. _2545: https://github.com/varnishcache/varnish-cache/pull/2545 +.. _2546: https://github.com/varnishcache/varnish-cache/issues/2546 +.. _2551: https://github.com/varnishcache/varnish-cache/issues/2551 +.. _2554: https://github.com/varnishcache/varnish-cache/pull/2554 +.. _2556: https://github.com/varnishcache/varnish-cache/issues/2556 +.. _2558: https://github.com/varnishcache/varnish-cache/pull/2558 +.. _2563: https://github.com/varnishcache/varnish-cache/issues/2563 +.. _2570: https://github.com/varnishcache/varnish-cache/issues/2570 +.. _2574: https://github.com/varnishcache/varnish-cache/issues/2574 +.. _2582: https://github.com/varnishcache/varnish-cache/issues/2582 +.. _2592: https://github.com/varnishcache/varnish-cache/issues/2592 +.. _2607: https://github.com/varnishcache/varnish-cache/issues/2607 ================================ Varnish Cache 5.2.1 (2017-11-14) From hermunn at varnish-software.com Wed Mar 14 23:54:07 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 14 Mar 2018 23:54:07 +0000 (UTC) Subject: [master] 7dc7292 Add note on obj.storage Message-ID: <20180314235407.CAD63B2541@lists.varnish-cache.org> commit 7dc7292b19674383611ae8e2c1e7718fd87dab25 Author: P?l Hermunn Johansen Date: Thu Mar 15 00:46:41 2018 +0100 Add note on obj.storage diff --git a/doc/changes.rst b/doc/changes.rst index b96424b..5b6495d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -44,6 +44,8 @@ VCL and bundled VMODs ``req.hash_always_miss`` are now accessible from all of the client side subs, not just ``vcl_recv{}`` +* ``obj.storage`` is now available in ``vcl_hit{}`` and ``vcl_deliver{}``. + * Removed ``beresp.storage_hint`` for VCL 4.1 (was deprecated since Varnish 5.1) From dridi at varni.sh Thu Mar 15 00:01:17 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 15 Mar 2018 01:01:17 +0100 Subject: [master] fcb04d9 Add some more "Changes for Developers" to "Upgrading to 6.0". In-Reply-To: <20180314223508.B5A9FB0DF7@lists.varnish-cache.org> References: <20180314223508.B5A9FB0DF7@lists.varnish-cache.org> Message-ID: > + * VMOD authors can use the ``VRT_VSC_*()`` series of functions and > + the new ``vsctool`` to create statistics for a VMOD that will be > + displayed by varnishstat. Varnish uses the same technique to > + create its counters, so you can look to the core code to see how > + it's done. I forgot to add VSC support to varnish.m4 to streamline its use out of tree. Since I had such a patch for 5.2 it may only be a case of cherry-picking it with s/5.2/6.0/ because I tested it back then but it was refused "until we decide to allow VMODs to do that". I will look at this tomorrow morning and refrain from checking in the patch if anything looks suspicious. Since there hasn't been any changes in varnish.m4 in the last 6 months there should also be no conflict, I'll consider a conflict suspicious and thus hold back for inclusion. (at the time I submitted macros for VUTs and VSCs and only the VUT macro got merged) We should be fine with vsctool.py, I made sure it would land in the -dev package for both Debs and RPMs. Dridi From hermunn at varnish-software.com Thu Mar 15 08:25:12 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 08:25:12 +0000 (UTC) Subject: [master] 0d60039 Changelog typos Message-ID: <20180315082512.C632A644F6@lists.varnish-cache.org> commit 0d600390f01b17c50c92db5bd7df28d826ab0b3a Author: P?l Hermunn Johansen Date: Thu Mar 15 09:20:57 2018 +0100 Changelog typos diff --git a/doc/changes.rst b/doc/changes.rst index 5b6495d..0e19dc7 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -11,7 +11,7 @@ Usage * The ``cli_buffer`` parameter has been removed -* Added back ``umem`` storage for Solaris descendents +* Added back ``umem`` storage for Solaris descendants * The new storage backend type (stevedore) ``default`` now resolves to either ``umem`` (where available) or ``malloc``. @@ -21,7 +21,7 @@ Usage the client workspace. We are now taking delivery IO vectors from the thread workspace, so - the parameter documemtation is in sync with reality again. + the parameter documentation is in sync with reality again. Users who need to minimize memory footprint might consider decreasing ``workspace_client`` by ``workspace_thread``. @@ -296,7 +296,7 @@ Fixed bugs * H2: Don't HTC_RxStuff with a non-reserved workspace. (2539_) -* Various optimalizations of VSM. (2430_, 2470_, 2518_, 2535_, 2541_, 2545_, 2546_) +* Various optimizations of VSM. (2430_, 2470_, 2518_, 2535_, 2541_, 2545_, 2546_) * Problems during late socket initialization performed by the Varnish child process can now be reported back to the management process with an From hermunn at varnish-software.com Thu Mar 15 08:31:08 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 08:31:08 +0000 (UTC) Subject: [master] f92d42a Update Via header to Varnish/6.0 Message-ID: <20180315083108.87F1D646A4@lists.varnish-cache.org> commit f92d42a0475c4b74c748ae47bc925ce3abd8cfd8 Author: P?l Hermunn Johansen Date: Wed Mar 14 11:13:07 2018 +0100 Update Via header to Varnish/6.0 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index f7d96cc..1db47c4 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -151,7 +151,7 @@ cnt_deliver(struct worker *wrk, struct req *req) http_PrintfHeader(req->resp, "Age: %.0f", floor(fmax(0., req->t_prev - req->objcore->t_origin))); - http_SetHeader(req->resp, "Via: 1.1 varnish (Varnish/5.2)"); + http_SetHeader(req->resp, "Via: 1.1 varnish (Varnish/6.0)"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && From hermunn at varnish-software.com Thu Mar 15 08:57:08 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 08:57:08 +0000 (UTC) Subject: [master] 15ddc05 Remove XXX comments from changes.rst Message-ID: <20180315085708.1DE0064DDC@lists.varnish-cache.org> commit 15ddc05987351942b23d0f69ec739d3c2aea9f58 Author: P?l Hermunn Johansen Date: Thu Mar 15 09:45:01 2018 +0100 Remove XXX comments from changes.rst diff --git a/doc/changes.rst b/doc/changes.rst index 0e19dc7..b2c5b65 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -190,8 +190,7 @@ C APIs (for vmod and utility authors) ------------------------------------- * We have now defined three API Stability levels: ``VRT``, - ``PACKAGE``, ``SOURCE``. XXX: someone should write up an rst when - we've settled this. + ``PACKAGE``, ``SOURCE``. * New API namespace rules, see `phk_api_spaces_` @@ -200,8 +199,6 @@ C APIs (for vmod and utility authors) * some headers require specific include ordering * only ``cache.h`` _or_ ``vrt.h`` can be included - XXX: More details? - * Signatures of functions in the VLU API for bytestream into text serialization have been changed @@ -244,8 +241,6 @@ C APIs (for vmod and utility authors) have the ``(struct director).admin_health`` initialized to ``VDI_AH_*`` (usually ``VDI_AH_HEALTHY``). - XXX improve / more details - Other changes relevant for VMODs -------------------------------- From dridi at varni.sh Thu Mar 15 08:57:32 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 15 Mar 2018 09:57:32 +0100 Subject: [master] e89586b Add proxy/cache_proxy.h to varnishd_SOURCES. In-Reply-To: References: <20180314161009.CD170A7E72@lists.varnish-cache.org> Message-ID: > Understood. I was thinking of keeping my commits separate from ehocdet's > for the PR, but yes this one could have been squashed. The nature of a bisect means that we have low probability to hit one of the patches from Emmanuel but still, regardless of the failure it takes some time to figure out when it's a false positive (because the regression could also come from a commit with a false-negative failure). > I'm curious as to why Travis put the green checkmark on the PR. Doesn't > Travis run make distcheck? Looks like we only make check, that's on us :( Dridi From hermunn at varnish-software.com Thu Mar 15 09:05:11 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 09:05:11 +0000 (UTC) Subject: [master] 5834b8c State that return(fetch) is not allowed any more Message-ID: <20180315090511.23ED465049@lists.varnish-cache.org> commit 5834b8c44ef4ff4e0baa74b5655be96d0ea5319d Author: P?l Hermunn Johansen Date: Thu Mar 15 10:02:37 2018 +0100 State that return(fetch) is not allowed any more diff --git a/doc/changes.rst b/doc/changes.rst index b2c5b65..aa887e7 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -36,6 +36,10 @@ Usage VCL and bundled VMODs --------------------- +* ``return (fetch)`` is no longer allowed in ``vcl_hit {}``, use + ``return (miss)`` instread. Note that ``return (fetch)`` has been + deprecated since 4.0. + * Fix behaviour of restarts to how it was originally intended: Restarts now leave all the request properties in place except for ``req.restarts`` and ``req.xid``, which need to change by design. From nils.goroll at uplex.de Thu Mar 15 09:35:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 09:35:11 +0000 (UTC) Subject: [master] 48bb211 fix shard director upgrade notes Message-ID: <20180315093511.589646590A@lists.varnish-cache.org> commit 48bb211b22f54791abe30e086be9b0d50a6fdbf4 Author: Nils Goroll Date: Thu Mar 15 10:33:41 2018 +0100 fix shard director upgrade notes diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 973ac3b..4dbc4ae 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -486,10 +486,15 @@ for advice on how to do so. With the ``resolve=LAZY`` argument of the ``.backend()`` method, the shard director will now defer the selection of a backend to when a backend connection is actually made, which is how all other bundled -directors work as well. This enables layering the shard director below -other directors -- you can use ``.backend(resolve=LAZY)`` to set the -shard director as a backend for another director. ``resolve=LAZY`` -MUST be used in ``vcl_init`` and on the client side. +directors work as well. + +In ``vcl_init``, ``resolve=LAZY`` is default and enables layering the +shard director below other directors -- you can now use something like +``mydirector.add_backend(myshard.backend())`` to set the shard +director as a backend for another director. + +Use of ``resolve=LAZY`` on the client side is limited to using the +default or associated parameters. The shard director now provides a ``shard_param`` object that serves as a store for a set of parameters for the director's ``.backend()`` From nils.goroll at uplex.de Thu Mar 15 09:54:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 09:54:07 +0000 (UTC) Subject: [master] 6286ddd document vcc optional arguments in changelog Message-ID: <20180315095407.6654C65DEC@lists.varnish-cache.org> commit 6286ddd28fc729a7349875832a7e66525c57e582 Author: Nils Goroll Date: Thu Mar 15 10:53:07 2018 +0100 document vcc optional arguments in changelog diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 4dbc4ae..cf2af34 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -866,4 +866,22 @@ Other changes the generated docs altogether; so you can write the ``SYNOPSIS`` section yourself, if you prefer. + * Support for a new declaration of optional arguments in vcc files + has been added: ``[ argname ]`` can be used to mark *argname* as + optional. + + If this declaration is used for any argument, _all_ user arguments + and ``PRIV_*`` pointers (no object pointers) to the respective + function/method will be passed in a ``struct`` *funcname*\ + ``_arg`` specific to this function which contains the arguments by + their name (or the name ``arg``\ *n* for unnamed arguments, *n* + being the argument position starting with 1) plus ``valid_``\ + *argname* members for optional arguments which are being set to + non-zero iff the respective *argname* was provided. + + Argument presence is determined at VCC time, so it is not possible + to pass an unset argument from another function call. + + + *eof* From nils.goroll at uplex.de Thu Mar 15 09:58:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 09:58:11 +0000 (UTC) Subject: [master] c304b8c remove the mention of STRANDS from the changes for now Message-ID: <20180315095811.5AEF66CF1B@lists.varnish-cache.org> commit c304b8c1e9a9f42c90ee98b6f7f05980150928e6 Author: Nils Goroll Date: Thu Mar 15 10:55:59 2018 +0100 remove the mention of STRANDS from the changes for now we are not even using it in our own code yet, so I feel uneasy about pointing devs to them. diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index 26b3707..e49a8a3 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -83,13 +83,6 @@ structure when this is used. The ``vmodtool.py`` script has learned other new tricks, and as a result also produces nicer ``.rst`` output. -We have a new way of passing text-string arguments around -called ``STRANDS``. Instead of the ``stdarg.h`` representation, -an ``argc+argv`` struct is passed, which means that a function -can take multiple STRANDS arguments, which again means that -many functions will not need to reassemble strings on a -workspace any more, most notably string comparisons. - VCL types ``INT`` and ``BYTES`` are now 64bits on all platforms. VCL ENUM have gotten a new implementation, so the pointers From dridi.boukelmoune at gmail.com Thu Mar 15 10:03:08 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 15 Mar 2018 10:03:08 +0000 (UTC) Subject: [master] 26f31d8 New VARNISH_COUNTERS macro for varnish.m4 Message-ID: <20180315100308.E2754910D0@lists.varnish-cache.org> commit 26f31d84ecec807f75b2a20d265c55f6dc6031c5 Author: Dridi Boukelmoune Date: Thu Mar 15 11:00:49 2018 +0100 New VARNISH_COUNTERS macro for varnish.m4 Initially written and tested for Varnish 5.2, but not included then because VMODs weren't allowed custom counters yet. Tested again with success for Varnish 6.0 and documented. diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst index e49a8a3..55001dd 100644 --- a/doc/sphinx/whats-new/changes-6.0.rst +++ b/doc/sphinx/whats-new/changes-6.0.rst @@ -68,7 +68,9 @@ just like builtin counters, because there is no difference. The counters are described in a ``.vsc`` file which is processed with a new python script which does a lot of magic etc. There is a tiny example in ``vmod_debug`` in -the source tree. +the source tree. If you're using autotools, a new +``VARNISH_COUNTERS`` macro helps you set everything up, +and is documented in ``varnish.m4``. This took a major retooling of the stats counters in general, and the VSM, VSC and VSL apis have all subtly or not so subtly changed diff --git a/varnish.m4 b/varnish.m4 index 07dbcc8..8a92d5f 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2017 Varnish Software AS +# Copyright (c) 2016-2018 Varnish Software AS # All rights reserved. # # Author: Dridi Boukelmoune @@ -29,7 +29,7 @@ # OF THE POSSIBILITY OF SUCH DAMAGE. # varnish.m4 - Macros to build against Varnish. -*- Autoconf -*- -# serial 9 (varnish-5.2.1) +# serial 10 (varnish-6.0.0) # # This collection of macros helps create VMODs or tools interacting with # Varnish Cache using the GNU build system (autotools). In order to work @@ -375,6 +375,98 @@ AC_DEFUN([VARNISH_VMODS], [ [_VARNISH_VMOD(_vmod_name)]) ]) +# _VARNISH_VSC_CONFIG +# -------------------- +AC_DEFUN([_VARNISH_VSC_CONFIG], [ + + AC_REQUIRE([_VARNISH_PKG_CONFIG]) + AC_REQUIRE([_VARNISH_CHECK_DEVEL]) + AC_REQUIRE([_VARNISH_CHECK_PYTHON]) + + dnl Define an automake silent execution for vmodtool + [am__v_VSCTOOL_0='@echo " VSCTOOL " $''@;'] + [am__v_VSCTOOL_1=''] + [am__v_VSCTOOL_='$(am__v_VSCTOOL_$(AM_DEFAULT_VERBOSITY))'] + [AM_V_VSCTOOL='$(am__v_VSCTOOL_$(V))'] + AC_SUBST([am__v_VSCTOOL_0]) + AC_SUBST([am__v_VSCTOOL_1]) + AC_SUBST([am__v_VSCTOOL_]) + AC_SUBST([AM_V_VSCTOOL]) +]) + +# _VARNISH_COUNTER(NAME) +# ---------------------- +AC_DEFUN([_VARNISH_COUNTER], [ + + AC_REQUIRE([_VARNISH_VSC_CONFIG]) + + VSC_RULES=" + +VSC_$1.h: $1.vsc + \$(A""M_V_VSCTOOL) \$(PYTHON) \$(VSCTOOL) -h \$(srcdir)/$1.vsc + +VSC_$1.c: $1.vsc + \$(A""M_V_VSCTOOL) \$(PYTHON) \$(VSCTOOL) -c \$(srcdir)/$1.vsc + +VSC_$1.rst: $1.vsc + \$(A""M_V_VSCTOOL) \$(PYTHON) \$(VSCTOOL) -r \$(srcdir)/$1.vsc >VSC_$1.rst + +clean: clean-vsc-$1 + +distclean: clean-vsc-$1 + +clean-vsc-$1: + rm -f VSC_$1.h VSC_$1.c VSC_$1.rst + +" + + AC_SUBST(m4_toupper(BUILD_VSC_$1), [$VSC_RULES]) + m4_ifdef([_AM_SUBST_NOTMAKE], + [_AM_SUBST_NOTMAKE(m4_toupper(BUILD_VSC_$1))]) +]) + +# VARNISH_COUNTERS(NAMES) +# ----------------------- +# Since: Varnish 6.0.0 +# +# In order to manipulate custom counters that tools like varnishstat can +# report, it is possible to do that via a VMOD. This macro allows you +# to declare sets of counters, but does not associates them automatically +# with their respective VMODs: +# +# VARNISH_UTILITIES([foo bar]) +# +# Two build rules will be available for use in Makefile.am for the counters +# foo and bar: +# +# @BUILD_VSC_FOO@ +# @BUILD_VSC_BAR@ +# +# They take care of turning VSC_foo.vsc and VCS_bar.vcs into C code and +# RST documentation. +# +# Just like the vcc_*_if.[ch] files, you need to manually add the generated +# sources to the appropriate VMODs: +# +# nodist_libvmod_baz_la_SOURCES = \ +# vcc_baz_if.c \ +# vcc_baz_if.h \ +# VSC_foo.c \ +# VSC_foo.h +# +# You can then include the counters documentation somewhere in the VMOD's +# VCC descriptor: +# +# .. include:: VSC_foo.rst +# +# That should be all you need to do to start implementing custom counters. +# +AC_DEFUN([VARNISH_COUNTERS], [ + m4_foreach([_vsc_name], + m4_split(m4_normalize([$1])), + [_VARNISH_COUNTER(_vsc_name)]) +]) + # _VARNISH_UTILITY(NAME) # ---------------------- AC_DEFUN([_VARNISH_UTILITY], [ From hermunn at varnish-software.com Thu Mar 15 10:21:09 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 10:21:09 +0000 (UTC) Subject: [master] a068361 Prepare for 6.0.0 Message-ID: <20180315102109.4FAE2916ED@lists.varnish-cache.org> commit a068361dff0d25a0d85cf82a6e5fdaf315e06a7d Author: P?l Hermunn Johansen Date: Thu Mar 15 11:10:45 2018 +0100 Prepare for 6.0.0 diff --git a/configure.ac b/configure.ac index 45bdb45..e16e2d5 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2017 Varnish Software]) +Copyright (c) 2006-2018 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [6.0.0], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index aa887e7..af44652 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,6 +1,6 @@ -============================= -Varnish Cache Trunk (ongoing) -============================= +================================ +Varnish Cache 6.0.0 (2018-03-15) +================================ Usage ----- From hermunn at varnish-software.com Thu Mar 15 12:40:11 2018 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Thu, 15 Mar 2018 12:40:11 +0000 (UTC) Subject: [master] 6b8e44f We do not produce packages for Debian 8 Message-ID: <20180315124011.EEBA5946D2@lists.varnish-cache.org> commit 6b8e44ffb748c3acbf0ad62ea0c0f3f981dbd9dd Author: P?l Hermunn Johansen Date: Thu Mar 15 13:39:40 2018 +0100 We do not produce packages for Debian 8 diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index cf2af34..eaea169 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -575,7 +575,7 @@ Supported platforms ~~~~~~~~~~~~~~~~~~~ Official Varnish packages went through major changes for this release, -and target Debian 8 and 9, Ubuntu 16.04 LTS and (Red Hat) Enterprise +and target Debian 9, Ubuntu 16.04 LTS and (Red Hat) Enterprise Linux 7. Ubuntu 14.04 LTS will likely reach its end of life before Varnish 6 and the venerable Enterprise Linux 6 is getting too old and forced time-consuming workarounds so for these reasons we dropped From nils.goroll at uplex.de Thu Mar 15 14:23:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 14:23:08 +0000 (UTC) Subject: [master] dbbb1fb keep Content-Encoding intact if http_gzip_support is disabled Message-ID: <20180315142308.2F94896631@lists.varnish-cache.org> commit dbbb1fb2eb2dcd155e9a09e4a4dcbe978a2db944 Author: Nils Goroll Date: Thu Mar 15 15:17:31 2018 +0100 keep Content-Encoding intact if http_gzip_support is disabled Fixes #2610 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 69dcfcc..36b1c50 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -521,6 +521,11 @@ vbf_figure_out_vfp(struct busyobj *bo) * */ + if (! cache_param->http_gzip_support) { + bo->do_gzip = bo->do_gunzip = 0; + return (0); + } + if (http_GetStatus(bo->beresp) == 206) { if (bo->do_esi) { VSLb(bo->vsl, SLT_VCL_Error, @@ -531,10 +536,9 @@ vbf_figure_out_vfp(struct busyobj *bo) return (0); } - /* No body or no GZIP support -> done */ + /* No body -> done */ if (bo->htc->body_status == BS_NONE || - bo->htc->content_length == 0 || - !cache_param->http_gzip_support) { + bo->htc->content_length == 0) { http_Unset(bo->beresp, H_Content_Encoding); bo->do_gzip = bo->do_gunzip = 0; bo->do_stream = 0; diff --git a/bin/varnishtest/tests/g00003.vtc b/bin/varnishtest/tests/g00003.vtc index ee032e3..1ba77ba 100644 --- a/bin/varnishtest/tests/g00003.vtc +++ b/bin/varnishtest/tests/g00003.vtc @@ -15,6 +15,18 @@ server s1 { expect req.url == "/foobar" expect req.http.accept-encoding == "gzip" txresp -bodylen 43 + + rxreq + expect req.url == "/nogzip" + expect req.http.accept-encoding == "gzip" + txresp -hdr "Vary: Accept-Encoding" \ + -gzipbody "keep gzip real" + + rxreq + expect req.url == "/nogzip" + expect req.http.accept-encoding == + txresp -hdr "Vary: Accept-Encoding" \ + -body "keep plain real" } -start varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { @@ -53,3 +65,18 @@ client c1 { varnish v1 -expect n_gzip == 1 varnish v1 -expect n_gunzip == 2 varnish v1 -expect n_test_gunzip == 0 + +varnish v1 -cliok "param.set http_gzip_support false" + +client c1 { + txreq -url /nogzip -hdr "Accept-Encoding: gzip" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.body == "keep gzip real" + + txreq -url /nogzip + rxresp + expect resp.http.content-encoding == + expect resp.body == "keep plain real" +} -run From nils.goroll at uplex.de Thu Mar 15 15:07:05 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 15:07:05 +0000 (UTC) Subject: [master] d06e9af move the http_gzip_support check to the right place Message-ID: <20180315150705.D639697440@lists.varnish-cache.org> commit d06e9af2cb2c49b52c6db43061395dfa1650d06d Author: Nils Goroll Date: Thu Mar 15 16:04:24 2018 +0100 move the http_gzip_support check to the right place Ref: #2610 Thank you to @fgsch for spotting Apologies to everyone for my insufficient understanding of the agreement about which fixes we push during the soak in period after release. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 36b1c50..9de528b 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -521,11 +521,6 @@ vbf_figure_out_vfp(struct busyobj *bo) * */ - if (! cache_param->http_gzip_support) { - bo->do_gzip = bo->do_gunzip = 0; - return (0); - } - if (http_GetStatus(bo->beresp) == 206) { if (bo->do_esi) { VSLb(bo->vsl, SLT_VCL_Error, @@ -536,6 +531,11 @@ vbf_figure_out_vfp(struct busyobj *bo) return (0); } + if (! cache_param->http_gzip_support) { + bo->do_gzip = bo->do_gunzip = 0; + return (0); + } + /* No body -> done */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { From nils.goroll at uplex.de Thu Mar 15 15:38:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 15:38:09 +0000 (UTC) Subject: [master] 5e2b0d8 If gzip support is off, we still need to esi process Message-ID: <20180315153809.52C919852E@lists.varnish-cache.org> commit 5e2b0d8abda18e807b73ed2f6c0746688001e85b Author: Nils Goroll Date: Thu Mar 15 16:30:38 2018 +0100 If gzip support is off, we still need to esi process Ref: #2610 with another thank you to @fgsch diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 9de528b..eb906b8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -531,10 +531,8 @@ vbf_figure_out_vfp(struct busyobj *bo) return (0); } - if (! cache_param->http_gzip_support) { + if (! cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; - return (0); - } /* No body -> done */ if (bo->htc->body_status == BS_NONE || From dridi at varni.sh Thu Mar 15 17:13:51 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 15 Mar 2018 18:13:51 +0100 Subject: [master] 5e2b0d8 If gzip support is off, we still need to esi process In-Reply-To: <20180315153809.52C919852E@lists.varnish-cache.org> References: <20180315153809.52C919852E@lists.varnish-cache.org> Message-ID: On Thu, Mar 15, 2018 at 4:38 PM, Nils Goroll wrote: > > commit 5e2b0d8abda18e807b73ed2f6c0746688001e85b > Author: Nils Goroll > Date: Thu Mar 15 16:30:38 2018 +0100 > > If gzip support is off, we still need to esi process > > Ref: #2610 with another thank you to @fgsch > > diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c > index 9de528b..eb906b8 100644 > --- a/bin/varnishd/cache/cache_fetch.c > +++ b/bin/varnishd/cache/cache_fetch.c > @@ -531,10 +531,8 @@ vbf_figure_out_vfp(struct busyobj *bo) > return (0); > } > > - if (! cache_param->http_gzip_support) { > + if (! cache_param->http_gzip_support) > bo->do_gzip = bo->do_gunzip = 0; > - return (0); > - } My understanding was that http_gzip_support enabled means that Varnish will sanitize the client Accept-Encoding header and ask backends to compress (which they may not) and handle clients that support gzip or not depending on what's present in storage. Do we really want to neuter do_g[un]zip set _manually_ in VCL when _automatic_ gzip support is disabled? Dridi From nils.goroll at uplex.de Thu Mar 15 19:08:22 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Mar 2018 20:08:22 +0100 Subject: [master] 5e2b0d8 If gzip support is off, we still need to esi process In-Reply-To: References: <20180315153809.52C919852E@lists.varnish-cache.org> Message-ID: <512c951d-39b2-08d9-d7f0-835f2ba6dced@uplex.de> On 15/03/18 18:13, Dridi Boukelmoune wrote: > > Do we really want to neuter do_g[un]zip set _manually_ in VCL when > _automatic_ gzip support is disabled? Not sure, I think this is how it was before, but we might want to change our mind. More interesting question: Does esi really work with these changes, even if the backend responds with CE: gzip? Or do we maybe want to disable gzip if esi and ! cache_param->http_gzip_support? Need to invest more time. This regression is much more severe than I thought, apologies again. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From geoff at uplex.de Fri Mar 23 12:45:16 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 23 Mar 2018 12:45:16 +0000 (UTC) Subject: [master] 4db00af Test error handling when the -a UDS path is too long. Message-ID: <20180323124516.2C11061F22@lists.varnish-cache.org> commit 4db00afa9face7fe55bf12f3b0f8cbcfd7282a09 Author: Geoff Simmons Date: Fri Mar 23 13:16:02 2018 +0100 Test error handling when the -a UDS path is too long. We currently suppress the error message that actually explains the problem. diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index 9dfb220..1c369bb 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -18,6 +18,11 @@ shell -err -expect "Too many protocol sub-args" { varnishd -a 127.0.0.1:80000,HTTP,FOO -d } +# -a UDS path too long +shell -err -match "Got no socket" { + varnishd -a /Although/sizeof/sockaddr_un/sun_path/is/platform/specific/this/path/is/really/definitely/and/most/assuredly/too/long/on/any/platform/--/any/length/that/results/in/sizeof/sockaddr_un/being/greater/than/128/will/probably/be/enough/to/blow/it/up. -d +} + # -a relative path for a UDS address not permitted shell -err -expect "Unix domain socket addresses must be absolute paths" { varnishd -a foo/bar.sock -d From geoff at uplex.de Fri Mar 23 12:45:16 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 23 Mar 2018 12:45:16 +0000 (UTC) Subject: [master] ce91292 Always use bprintf() write to sockaddr_un.sun_path. Message-ID: <20180323124516.4B1B661F25@lists.varnish-cache.org> commit ce91292da0a95cd874b97ba6af97564b99bb92f8 Author: Geoff Simmons Date: Fri Mar 23 13:42:21 2018 +0100 Always use bprintf() write to sockaddr_un.sun_path. We already check the -a user input and emit an error if the path is too long. bprintf() should help persuade static analysis tools that we won't overflow the fixed-size buffer. diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 796a11b..8ef0f3a 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -90,7 +90,7 @@ mac_opensocket(struct listen_sock *ls) ls->sock = VTCP_bind(ls->addr, NULL); else { uds.sun_family = PF_UNIX; - strcpy(uds.sun_path, ls->endpoint); + bprintf(uds.sun_path, "%s", ls->endpoint); ls->sock = VUS_bind(&uds, NULL); } fail = errno; diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index 69a223d..6315d95 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "vdef.h" #include "vas.h" @@ -55,7 +56,7 @@ VUS_resolver(const char *path, vus_resolved_f *func, void *priv, *err = "Path too long for a Unix domain socket"; return(-1); } - strcpy(uds.sun_path, path); + bprintf(uds.sun_path, "%s", path); uds.sun_family = PF_UNIX; if (func != NULL) ret = func(priv, &uds); @@ -108,10 +109,8 @@ VUS_connect(const char *path, int msec) if (path == NULL) return (-1); - /* Attempt the connect */ - assert(strlen(path) + 1 <= sizeof(uds.sun_path)); uds.sun_family = PF_UNIX; - strcpy(uds.sun_path, path); + bprintf(uds.sun_path, "%s", path); AN(sl); s = socket(PF_UNIX, SOCK_STREAM, 0); From fgsch at lodoss.net Fri Mar 23 14:57:10 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 23 Mar 2018 14:57:10 +0000 (UTC) Subject: [master] 9cf1ae9 Exercise ESI with gzip support disabled Message-ID: <20180323145710.1B79564976@lists.varnish-cache.org> commit 9cf1ae9bdecf2e95c986a360ecf0dec0de2a4d67 Author: Federico G. Schwindt Date: Fri Mar 23 11:39:31 2018 -0300 Exercise ESI with gzip support disabled Related to #2626. diff --git a/bin/varnishtest/tests/e00031.vtc b/bin/varnishtest/tests/e00031.vtc new file mode 100644 index 0000000..8089633 --- /dev/null +++ b/bin/varnishtest/tests/e00031.vtc @@ -0,0 +1,21 @@ +varnishtest "ESI with http_gzip_support off" + +server s1 { + rxreq + txresp -gzipbody {} + rxreq + txresp -gzipbody "bar" +} -start + +varnish v1 -arg "-p http_gzip_support=off" -vcl+backend { + sub vcl_backend_response { + set beresp.do_esi = true; + } +} -start + +client c1 { + txreq + rxresp + gunzip + expect resp.body == "bar" +} -run From nils.goroll at uplex.de Mon Mar 26 11:02:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Mar 2018 11:02:12 +0000 (UTC) Subject: [master] efb5d80 avoid duplicate init Message-ID: <20180326110212.DC32F957CB@lists.varnish-cache.org> commit efb5d801864f7746d656ff4b843cd3e5f0680348 Author: Nils Goroll Date: Wed Mar 21 14:22:34 2018 +0100 avoid duplicate init diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 3f72597..dd7e497 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -506,7 +506,6 @@ main(int argc, char **argv) pthread_t thr; int fnum = -1; struct profile cli_p = {0}; - cli_p.name = 0; vut = VUT_InitProg(argc, argv, &vopt_spec); AN(vut); From nils.goroll at uplex.de Mon Mar 26 11:02:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Mar 2018 11:02:12 +0000 (UTC) Subject: [master] 221830d Add a histogram profile for ReqBody time Message-ID: <20180326110212.D83EE957C9@lists.varnish-cache.org> commit 221830d47b51ccf826949f1d502809aeecff8863 Author: Nils Goroll Date: Wed Mar 21 10:39:48 2018 +0100 Add a histogram profile for ReqBody time diff --git a/bin/varnishhist/varnishhist_profiles.h b/bin/varnishhist/varnishhist_profiles.h index 68e109c..a7295b6 100644 --- a/bin/varnishhist/varnishhist_profiles.h +++ b/bin/varnishhist/varnishhist_profiles.h @@ -42,6 +42,16 @@ HIS_PROF( " client response" ) HIS_PROF( + "reqbodytime", // name + HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND + SLT_Timestamp, // tag + "ReqBody:", // prefix + 3, // field + -6, // hist_low + 3, // hist_high + "graph the time for reading the request body" + ) +HIS_PROF( "size", // name HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND SLT_ReqAcct, // tag From nils.goroll at uplex.de Mon Mar 26 11:02:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Mar 2018 11:02:13 +0000 (UTC) Subject: [master] 50926ac varnishhish: Add a prefix to custom Profile definitions Message-ID: <20180326110213.124CF957CE@lists.varnish-cache.org> commit 50926acbe55b4f851e5f593dab5d17ca5dbd6e9b Author: Nils Goroll Date: Wed Mar 21 14:38:02 2018 +0100 varnishhish: Add a prefix to custom Profile definitions Without it, graphing Timestamp tags was impossible for all practical purposes diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index dd7e497..27d3174 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -501,8 +501,8 @@ int main(int argc, char **argv) { int i; - const char *colon, *ptag; - const char *profile = "responsetime"; + char *colon; + const char *ptag, *profile = "responsetime"; pthread_t thr; int fnum = -1; struct profile cli_p = {0}; @@ -528,7 +528,7 @@ main(int argc, char **argv) profile = optarg; break; } - /* else it's a definition, we hope */ + /* else check if valid definition */ if (colon == optarg + 1 && (*optarg == 'b' || *optarg == 'c')) { cli_p.VSL_arg = *optarg; @@ -542,15 +542,27 @@ main(int argc, char **argv) } assert(colon); - if (sscanf(colon + 1, "%d:%d:%d", &cli_p.field, - &cli_p.hist_low, &cli_p.hist_high) != 3) - profile_error(optarg); match_tag = VSL_Name2Tag(ptag, colon - ptag); if (match_tag < 0) VUT_Error(vut, 1, "-P: '%s' is not a valid tag name", optarg); + + cli_p.prefix = colon + 1; + colon = strchr(colon + 1, ':'); + + if (colon == NULL) + profile_error(optarg); + + *colon = '\0'; + if (*cli_p.prefix == '\0') + cli_p.prefix = NULL; + + if (sscanf(colon + 1, "%d:%d:%d", &cli_p.field, + &cli_p.hist_low, &cli_p.hist_high) != 3) + profile_error(optarg); + if (VSL_tagflags[match_tag]) VUT_Error(vut, 1, "-P: '%s' is an unsafe or binary record", diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 7ad374b..999a678 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -47,13 +47,14 @@ ) #define HIS_OPT_P \ - VOPT("P:", "[-P <[cb:]tag:field_num:min:max>]", \ + VOPT("P:", "[-P <[cb:]tag:[prefix]:field_num:min:max>]", \ "Custom profile definition", \ "Graph the given custom definition defined as: an optional" \ " (c)lient or (b)ackend filter (defaults to client), the" \ - " tag we'll look for, and the field number of the value we" \ - " are interested in. min and max are the boundaries of the" \ - " graph (these are power of ten)." \ + " tag we'll look for, a prefix to look for (can be empty," \ + " but must be terminated by a colon) and the field number" \ + " of the value we are interested in. min and max are the" \ + " boundaries of the graph (these are power of ten)." \ ) #define HIS_OPT_B \ diff --git a/doc/changes.rst b/doc/changes.rst index af44652..bb85dc2 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,14 @@ +==================== +Varnish Cache master +==================== + +bundled tools +------------- + +* ``varnishhist``: The format of the ``-P`` argument has been changed + for custom profile definitions to also contain a prefix to match the + tag against. + ================================ Varnish Cache 6.0.0 (2018-03-15) ================================ From nils.goroll at uplex.de Mon Mar 26 11:02:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Mar 2018 11:02:13 +0000 (UTC) Subject: [master] 480da50 varnishhist -P: make graph boundary parameters optional Message-ID: <20180326110213.2A6D4957D4@lists.varnish-cache.org> commit 480da504f9fe623be1e38dd4da7ec34fb8421e67 Author: Nils Goroll Date: Wed Mar 21 15:00:02 2018 +0100 varnishhist -P: make graph boundary parameters optional diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 27d3174..b33f5ce 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -549,9 +549,14 @@ main(int argc, char **argv) "-P: '%s' is not a valid tag name", optarg); + if (VSL_tagflags[match_tag]) + VUT_Error(vut, 1, + "-P: '%s' is an unsafe or binary record", + optarg); + cli_p.prefix = colon + 1; - colon = strchr(colon + 1, ':'); + colon = strchr(colon + 1, ':'); if (colon == NULL) profile_error(optarg); @@ -559,20 +564,24 @@ main(int argc, char **argv) if (*cli_p.prefix == '\0') cli_p.prefix = NULL; - if (sscanf(colon + 1, "%d:%d:%d", &cli_p.field, - &cli_p.hist_low, &cli_p.hist_high) != 3) + if (sscanf(colon + 1, "%d", &cli_p.field) != 1) profile_error(optarg); - if (VSL_tagflags[match_tag]) - VUT_Error(vut, 1, - "-P: '%s' is an unsafe or binary record", - optarg); - cli_p.name = "custom"; cli_p.tag = match_tag; + cli_p.hist_low = -6; + cli_p.hist_high = 3; profile = NULL; active_profile = &cli_p; + colon = strchr(colon + 1, ':'); + if (colon == NULL) + break; + + if (sscanf(colon + 1, "%d:%d", &cli_p.hist_low, + &cli_p.hist_high) != 2) + profile_error(optarg); + break; case 'B': timebend = strtod(optarg, NULL); diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 999a678..0c04278 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -47,14 +47,15 @@ ) #define HIS_OPT_P \ - VOPT("P:", "[-P <[cb:]tag:[prefix]:field_num:min:max>]", \ + VOPT("P:", "[-P <[cb:]tag:[prefix]:field_num[:min:max]>]", \ "Custom profile definition", \ "Graph the given custom definition defined as: an optional" \ " (c)lient or (b)ackend filter (defaults to client), the" \ " tag we'll look for, a prefix to look for (can be empty," \ " but must be terminated by a colon) and the field number" \ " of the value we are interested in. min and max are the" \ - " boundaries of the graph (these are power of ten)." \ + " boundaries of the graph in power of ten and default to" \ + " -6 and 3." \ ) #define HIS_OPT_B \ From nils.goroll at uplex.de Mon Mar 26 11:02:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Mar 2018 11:02:13 +0000 (UTC) Subject: [master] c531973 varnishhist test update Message-ID: <20180326110213.52AA2957D9@lists.varnish-cache.org> commit c531973031fc747fd0bc670ecef2a0b14ae610d8 Author: Nils Goroll Date: Wed Mar 21 16:32:37 2018 +0100 varnishhist test update diff --git a/bin/varnishtest/tests/u00007.vtc b/bin/varnishtest/tests/u00007.vtc index 8a095d5..64e9dc7 100644 --- a/bin/varnishtest/tests/u00007.vtc +++ b/bin/varnishtest/tests/u00007.vtc @@ -23,7 +23,7 @@ shell -err -expect "Invalid grouping mode: raw" \ "varnishhist -g raw" shell -err -expect "-P: No such profile 'foo'" \ "varnishhist -P foo" -shell -err -expect "-P: 'foo:bar' is not a valid profile name or definition" \ - "varnishhist -P foo:bar" -shell -err -expect "-P: 'foo:0:0:0' is not a valid tag name" \ - "varnishhist -P foo:0:0:0" +shell -err -expect "-P: 'Timestamp:' is not a valid profile name or definition" \ + "varnishhist -P Timestamp::" +shell -err -expect "-P: 'foo::0:0:0' is not a valid tag name" \ + "varnishhist -P foo::0:0:0" From phk at FreeBSD.org Mon Mar 26 12:32:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Mar 2018 12:32:12 +0000 (UTC) Subject: [master] dd84d50 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. Message-ID: <20180326123212.2C16D97404@lists.varnish-cache.org> commit dd84d50be041a766072d52987fd63062433dc8df Author: Poul-Henning Kamp Date: Mon Mar 26 12:30:29 2018 +0000 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. Suggested by: HAproxy diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 26a2002..62c83af 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -189,6 +189,10 @@ macro_get(const char *b, const char *e) int l; char *retval = NULL; + AN(b); + if (e == NULL) + e = strchr(b, '\0'); + AN(e); l = e - b; if (l == 4 && !memcmp(b, "date", l)) { @@ -889,6 +893,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir, char *logbuf, unsigned loglen) { FILE *f; + struct vsb *vsb; + const char *p; (void)signal(SIGPIPE, SIG_IGN); @@ -900,6 +906,21 @@ exec_file(const char *fn, const char *script, const char *tmpdir, init_macro(); init_server(); + vsb = VSB_new_auto(); + AN(vsb); + if (*fn != '/') + VSB_cat(vsb, macro_get("pwd", NULL)); + p = strrchr(fn, '/'); + if (p != NULL) { + VSB_putc(vsb, '/'); + VSB_bcat(vsb, fn, p - fn); + } + if (VSB_len == 0) + VSB_putc(vsb, '/'); + AZ(VSB_finish(vsb)); + macro_def(vltop, NULL, "testdir", "%s", VSB_data(vsb)); + VSB_destroy(&vsb); + /* Move into our tmpdir */ AZ(chdir(tmpdir)); macro_def(vltop, NULL, "tmpdir", "%s", tmpdir); From dridi.boukelmoune at gmail.com Mon Mar 26 20:31:10 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 26 Mar 2018 20:31:10 +0000 (UTC) Subject: [master] 2a0725c Incomplete condition Message-ID: <20180326203110.77EB4A5C95@lists.varnish-cache.org> commit 2a0725c0930b09c120ef3912b7f96e41f162df0f Author: Dridi Boukelmoune Date: Mon Mar 26 22:30:04 2018 +0200 Incomplete condition diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 62c83af..8d7f764 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -915,7 +915,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir, VSB_putc(vsb, '/'); VSB_bcat(vsb, fn, p - fn); } - if (VSB_len == 0) + if (VSB_len(vsb) == 0) VSB_putc(vsb, '/'); AZ(VSB_finish(vsb)); macro_def(vltop, NULL, "testdir", "%s", VSB_data(vsb)); From dridi at varni.sh Mon Mar 26 20:43:56 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 26 Mar 2018 22:43:56 +0200 Subject: [master] dd84d50 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. In-Reply-To: <20180326123212.2C16D97404@lists.varnish-cache.org> References: <20180326123212.2C16D97404@lists.varnish-cache.org> Message-ID: > + if (VSB_len == 0) > + VSB_putc(vsb, '/'); This check is lacking an actual function call, but what I'm wondering is rather how to properly test that. Probably by running varnishtest from within a VTC as we do in a few cases. Do I understand correctly that the absolute path may look like this? /path/to/build/dir/../../path/to/vtc/dir The ".." components should be interpreted as the distcheck scenario where the $(top_builddir) is nested under the $(top_srcdir). I'm rather skeptical we need this since we can already define macros via the command line. Dridi From phk at phk.freebsd.dk Mon Mar 26 20:51:23 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 26 Mar 2018 20:51:23 +0000 Subject: [master] dd84d50 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. In-Reply-To: References: <20180326123212.2C16D97404@lists.varnish-cache.org> Message-ID: <12947.1522097483@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> + if (VSB_len == 0) >> + VSB_putc(vsb, '/'); > >This check is lacking an actual function call, Duh! Fixed. >I'm rather skeptical we need this since we can already define macros >via the command line. We do not need it, but the HAproxy project does. I have other patches from them in the pipeline that will allow varnishtest to run and test HAproxy instances. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Mon Mar 26 21:02:31 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 26 Mar 2018 23:02:31 +0200 Subject: [master] dd84d50 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. In-Reply-To: <12947.1522097483@critter.freebsd.dk> References: <20180326123212.2C16D97404@lists.varnish-cache.org> <12947.1522097483@critter.freebsd.dk> Message-ID: >>I'm rather skeptical we need this since we can already define macros >>via the command line. > > We do not need it, but the HAproxy project does. Maybe they don't but didn't realize it? > I have other patches from them in the pipeline that will allow > varnishtest to run and test HAproxy instances. That is something I'm looking forward to! Depending on how the patches look we could consider support for Hitch too? Dridi From phk at phk.freebsd.dk Mon Mar 26 23:02:49 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 26 Mar 2018 23:02:49 +0000 Subject: [master] dd84d50 Introduce the macro 'testdir' which is an absolute path to the directory where the .vtc file lives. In-Reply-To: References: <20180326123212.2C16D97404@lists.varnish-cache.org> <12947.1522097483@critter.freebsd.dk> Message-ID: <13804.1522105369@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> We do not need it, but the HAproxy project does. > >Maybe they don't but didn't realize it? I trust them on their judgement of this. >> I have other patches from them in the pipeline that will allow >> varnishtest to run and test HAproxy instances. > >That is something I'm looking forward to! Depending on how the patches >look we could consider support for Hitch too? If the Hitch-maintainers what to contribute it, I'll happily import it. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Tue Mar 27 08:52:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 08:52:12 +0000 (UTC) Subject: [master] 5823ee9 Extract the wait4() handling to a subroutine Message-ID: <20180327085212.5CFBCB4FB2@lists.varnish-cache.org> commit 5823ee99c3238e28bc1068e5fdc3ecdfffa29c86 Author: Poul-Henning Kamp Date: Tue Mar 27 08:05:12 2018 +0000 Extract the wait4() handling to a subroutine diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index b404552..8cb2640 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -138,6 +138,7 @@ void b64_settings(const struct http *hp, const char *s); struct vsb *vtc_hex_to_bin(struct vtclog *vl, const char *arg); void vtc_expect(struct vtclog *, const char *, const char *, const char *, const char *, const char *); +void vtc_wait4(struct vtclog *, long, int, int); /* vtc_term.c */ struct term *Term_New(struct vtclog *, int, int); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 2c1439a..4cdea40 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -29,9 +29,12 @@ #include "config.h" #include +#include #include #include #include +#include +#include #include "vct.h" #include "vnum.h" @@ -128,3 +131,37 @@ vtc_expect(struct vtclog *vl, vtc_log(vl, 4, "EXPECT %s (%s) %s \"%s\" match", olhs, lhs, cmp, rhs); } + +void +vtc_wait4(struct vtclog *vl, long pid, int expect_status, int expect_signal) +{ + int status, r; + struct rusage ru; + + r = wait4(pid, &status, 0, &ru); + if (r < 0) + vtc_fatal(vl, "wait4 failed on pid %ld: %s", + pid, strerror(errno)); + vtc_log(vl, 2, "WAIT4 pid=%ld r=%d status=0x%04x (user %.6f sys %.6f)", + pid, r, status, + ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec, + ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec + ); + + if (WIFEXITED(status) && (WEXITSTATUS(status) == expect_status)) + return; + if (WIFSIGNALED(status) && (WTERMSIG(status) == expect_signal)) + return; +#ifdef WCOREDUMP + vtc_fatal(vl, "Bad exit code: 0x%04x exit 0x%x signal %d core %d", + status, + WEXITSTATUS(status), + WIFSIGNALED(status) ? WTERMSIG(status) : 0, + WCOREDUMP(status)); +#else + vtc_fatal(vl, "Bad exit code: 0x%04x exit 0x%x signal %d", + status, + WEXITSTATUS(status), + WIFSIGNALED(status) ? WTERMSIG(status) : 0); +#endif +} diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 12e4244..680c1aa 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -29,9 +29,7 @@ #include "config.h" #include -#include #include -#include #include #include @@ -198,7 +196,7 @@ vsl_catchup(const struct varnish *v) int vsl_idle; vsl_idle = v->vsl_idle; - while (!vtc_error && v->pid && vsl_idle == v->vsl_idle) + while (!vtc_error && vsl_idle == v->vsl_idle) VTIM_sleep(0.1); } @@ -224,7 +222,7 @@ varnishlog_thread(void *priv) c = NULL; opt = 0; - while (v->pid || c != NULL) { + while (v->fds[1] > 0 || c != NULL) { if (c == NULL) { if (vtc_error) break; @@ -386,7 +384,7 @@ varnish_thread(void *priv) memset(fds, 0, sizeof fds); fds->fd = v->fds[0]; fds->events = POLLIN; - i = poll(fds, 1, 1000); + i = poll(fds, 1, 10000); if (i == 0) continue; if (fds->revents & POLLIN) { @@ -635,8 +633,6 @@ static void varnish_cleanup(struct varnish *v) { void *p; - int status, r; - struct rusage ru; /* Close the CLI connection */ closefd(&v->cli_fd); @@ -648,26 +644,11 @@ varnish_cleanup(struct varnish *v) AZ(pthread_join(v->tp, &p)); closefd(&v->fds[0]); - r = wait4(v->pid, &status, 0, &ru); - v->pid = 0; - vtc_log(v->vl, 2, "R %d Status: %04x (u %.6f s %.6f)", r, status, - ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec, - ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec - ); - /* Pick up the VSL thread */ AZ(pthread_join(v->tp_vsl, &p)); - if (WIFEXITED(status) && (WEXITSTATUS(status) == v->expect_exit)) - return; -#ifdef WCOREDUMP - vtc_fatal(v->vl, "Bad exit code: %04x sig %x exit %x core %x", - status, WTERMSIG(status), WEXITSTATUS(status), - WCOREDUMP(status)); -#else - vtc_fatal(v->vl, "Bad exit code: %04x sig %x exit %x", - status, WTERMSIG(status), WEXITSTATUS(status)); -#endif + vtc_wait4(v->vl, v->pid, v->expect_exit, 0); + v->pid = 0; } /********************************************************************** From phk at FreeBSD.org Tue Mar 27 08:52:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 08:52:12 +0000 (UTC) Subject: [master] 1d5b03e Improve vtc_wait4() so it can also handle vtc_process' needs. Message-ID: <20180327085212.72506B4FB5@lists.varnish-cache.org> commit 1d5b03ebb856edc4bfedb71af0fa332d06fe8123 Author: Poul-Henning Kamp Date: Tue Mar 27 08:50:57 2018 +0000 Improve vtc_wait4() so it can also handle vtc_process' needs. diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 8cb2640..8f0d130 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -138,7 +138,7 @@ void b64_settings(const struct http *hp, const char *s); struct vsb *vtc_hex_to_bin(struct vtclog *vl, const char *arg); void vtc_expect(struct vtclog *, const char *, const char *, const char *, const char *, const char *); -void vtc_wait4(struct vtclog *, long, int, int); +void vtc_wait4(struct vtclog *, long, int, int, int); /* vtc_term.c */ struct term *Term_New(struct vtclog *, int, int); diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index c2e9c9a..7c37103 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -32,8 +32,6 @@ #include "config.h" #include // Linux: struct winsize -#include -#include #include #include @@ -85,7 +83,6 @@ struct process { pthread_mutex_t mtx; pthread_t tp; unsigned hasthread; - int status; struct term *term; int lin; @@ -247,10 +244,9 @@ static void * process_thread(void *priv) { struct process *p; - struct rusage ru; struct vev_root *evb; struct vev *ev; - int core, sig, ext, r; + int r; CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC); @@ -293,8 +289,8 @@ process_thread(void *priv) vtc_fatal(p->vl, "VEV_Once() = %d, error %s", r, strerror(errno)); - r = wait4(p->pid, &p->status, 0, &ru); - + vtc_wait4(p->vl, p->pid, + p->expect_exit, p->expect_signal, p->allow_core); closefd(&p->f_stdout); closefd(&p->f_stderr); @@ -304,35 +300,7 @@ process_thread(void *priv) macro_undef(p->vl, p->name, "pid"); p->pid = -1; - vtc_log(p->vl, 2, "R 0x%04x Status: %04x (u %.6f s %.6f)", - r, p->status, - ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec, - ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec - ); - AZ(pthread_mutex_unlock(&p->mtx)); - sig = WTERMSIG(p->status); - ext = WEXITSTATUS(p->status); -#ifdef WCOREDUMP - core = WCOREDUMP(p->status); - vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d core %d", - p->status, sig, ext, core); -#else - core = 0; - vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d", - p->status, sig, ext); -#endif - if (core && !p->allow_core) - vtc_fatal(p->vl, "Core dump"); - if (p->expect_signal >= 0 && sig != p->expect_signal) - vtc_fatal(p->vl, "Expected signal %d got %d", - p->expect_signal, sig); - else if (sig != 0 && sig != -p->expect_signal) - vtc_fatal(p->vl, "Expected signal %d got %d", - -p->expect_signal, sig); - if (ext != p->expect_exit) - vtc_fatal(p->vl, "Expected exit %d got %d", - p->expect_exit, ext); VEV_Destroy(&evb); if (p->log == 1) { diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 4cdea40..1a9ff10 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -132,8 +132,22 @@ vtc_expect(struct vtclog *vl, olhs, lhs, cmp, rhs); } +/********************************************************************** + * Wait for a subprocess. + * + * if expect_signal > 0, the process must die on that signal. + * if expect_signal < 0, dying on that signal is allowed, but not required. + * if allow_core > 0, a coredump is allowed, but not required. + * otherwise, the process must die on exit(expect_status) + */ + +#ifndef WCOREDUMP +# define WCOREDUMP(s) (-1) +#endif + void -vtc_wait4(struct vtclog *vl, long pid, int expect_status, int expect_signal) +vtc_wait4(struct vtclog *vl, long pid, + int expect_status, int expect_signal, int allow_core) { int status, r; struct rusage ru; @@ -148,20 +162,21 @@ vtc_wait4(struct vtclog *vl, long pid, int expect_status, int expect_signal) ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec ); - if (WIFEXITED(status) && (WEXITSTATUS(status) == expect_status)) + if (WIFEXITED(status) && expect_signal <= 0 && + (WEXITSTATUS(status) == expect_status)) return; - if (WIFSIGNALED(status) && (WTERMSIG(status) == expect_signal)) + + if (expect_signal < 0) + expect_signal = -expect_signal; + + if (WIFSIGNALED(status) && WCOREDUMP(status) <= allow_core && + WTERMSIG(status) == expect_signal) return; -#ifdef WCOREDUMP - vtc_fatal(vl, "Bad exit code: 0x%04x exit 0x%x signal %d core %d", + vtc_log(vl, 1, "Expected exit: 0x%x signal: %d core: %d", + expect_status, expect_signal, allow_core); + vtc_fatal(vl, "Bad exit status: 0x%04x exit 0x%x signal %d core %d", status, WEXITSTATUS(status), WIFSIGNALED(status) ? WTERMSIG(status) : 0, WCOREDUMP(status)); -#else - vtc_fatal(vl, "Bad exit code: 0x%04x exit 0x%x signal %d", - status, - WEXITSTATUS(status), - WIFSIGNALED(status) ? WTERMSIG(status) : 0); -#endif } diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 680c1aa..a1a5ed8 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -647,7 +647,7 @@ varnish_cleanup(struct varnish *v) /* Pick up the VSL thread */ AZ(pthread_join(v->tp_vsl, &p)); - vtc_wait4(v->vl, v->pid, v->expect_exit, 0); + vtc_wait4(v->vl, v->pid, v->expect_exit, 0, 0); v->pid = 0; } From phk at FreeBSD.org Tue Mar 27 09:13:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 09:13:08 +0000 (UTC) Subject: [master] a33b931 List commands in a single tbl-include file. Move misc commands to separate source file. Message-ID: <20180327091308.87FB8B56CA@lists.varnish-cache.org> commit a33b931e6ddd9b6fa74782560b7334636a23749f Author: Poul-Henning Kamp Date: Tue Mar 27 09:11:23 2018 +0000 List commands in a single tbl-include file. Move misc commands to separate source file. diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 974aaf9..efbac2a 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -27,22 +27,24 @@ bin_PROGRAMS = varnishtest varnishtest_SOURCES = \ hpack.h \ programs.h \ + cmds.h \ vmods.h \ - vtc.c \ vtc.h \ + vtc.c \ vtc_barrier.c \ vtc_client.c \ - vtc_http.c \ - vtc_http.h \ - vtc_log.c \ - vtc_http2.c \ vtc_h2_dectbl.h \ vtc_h2_enctbl.h \ vtc_h2_hpack.c \ vtc_h2_priv.h \ vtc_h2_stattbl.h \ vtc_h2_tbl.c \ + vtc_http.c \ + vtc_http.h \ + vtc_http2.c \ + vtc_log.c \ vtc_logexp.c \ + vtc_misc.c \ vtc_main.c \ vtc_process.c \ vtc_proxy.c \ diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h new file mode 100644 index 0000000..58c2e7b --- /dev/null +++ b/bin/varnishtest/cmds.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2018 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/*lint -save -e525 -e539 */ + +CMD(barrier) +CMD(client) +CMD(delay) +CMD(err_shell) +CMD(feature) +//CMD(haproxy) +CMD(logexpect) +CMD(process) +CMD(server) +CMD(setenv) +CMD(shell) +CMD(varnish) +CMD(varnishtest) +#undef CMD + +/*lint -restore */ diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt index 1dc2716..25eace2 100644 --- a/bin/varnishtest/flint.lnt +++ b/bin/varnishtest/flint.lnt @@ -35,6 +35,7 @@ -e788 // enum value not used in defaulted switch +-efile(451, cmds.h) -efile(451, vmods.h) -efile(451, programs.h) -efile(451, vtc_h2_stattbl.h) diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 8d7f764..76f1e7a 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -31,10 +31,6 @@ #include #include -#include -#include -#include -#include #include #include #include @@ -44,8 +40,6 @@ #include "vtc.h" #include "vav.h" -#include "vnum.h" -#include "vre.h" #include "vtim.h" #ifdef HAVE_SYS_PERSONALITY_H @@ -57,8 +51,8 @@ volatile sig_atomic_t vtc_error; /* Error encountered */ int vtc_stop; /* Stops current test without error */ pthread_t vtc_thread; +int ign_unknown_macro = 0; static struct vtclog *vltop; -static int ign_unknown_macro = 0; /********************************************************************** * Macro facility @@ -426,436 +420,13 @@ reset_cmds(const struct cmds *cmd) cmd->cmd(NULL, NULL, NULL, NULL); } -/* SECTION: varnishtest varnishtest - * - * This should be the first command in your vtc as it will identify the test - * case with a short yet descriptive sentence. It takes exactly one argument, a - * string, eg:: - * - * varnishtest "Check that varnishtest is actually a valid command" - * - * It will also print that string in the log. - */ - -static void -cmd_varnishtest(CMD_ARGS) -{ - - (void)priv; - (void)cmd; - (void)vl; - - if (av == NULL) - return; - AZ(strcmp(av[0], "varnishtest")); - - vtc_log(vl, 1, "TEST %s", av[1]); - AZ(av[2]); -} - -/* SECTION: shell shell - * - * Pass the string given as argument to a shell. If you have multiple - * commands to run, you can use curly barces to describe a multi-lines - * script, eg:: - * - * shell { - * echo begin - * cat /etc/fstab - * echo end - * } - * - * By default a zero exit code is expected, otherwise the vtc will fail. - * - * Notice that the commandstring is prefixed with "exec 2>&1;" to join - * stderr and stdout back to the varnishtest process. - * - * Optional arguments: - * - * \-err - * Expect non-zero exit code. - * - * \-exit N - * Expect exit code N instead of zero. - * - * \-expect STRING - * Expect string to be found in stdout+err. - * - * \-match REGEXP - * Expect regexp to match the stdout+err output. - */ -/* SECTION: client-server.spec.shell shell - * - * Same as for the top-level shell. - */ - -static void -cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, - const char *expect, const char *re) -{ - struct vsb *vsb; - FILE *fp; - vre_t *vre = NULL; - const char *errptr; - int r, c; - int err; - - AN(vl); - AN(cmd); - vsb = VSB_new_auto(); - AN(vsb); - if (re != NULL) { - vre = VRE_compile(re, 0, &errptr, &err); - if (vre == NULL) - vtc_fatal(vl, "shell_match invalid regexp (\"%s\")", - re); - } - VSB_printf(vsb, "exec 2>&1 ; %s", cmd); - AZ(VSB_finish(vsb)); - vtc_dump(vl, 4, "shell_cmd", VSB_data(vsb), -1); - fp = popen(VSB_data(vsb), "r"); - if (fp == NULL) - vtc_fatal(vl, "popen fails: %s", strerror(errno)); - VSB_clear(vsb); - do { - c = getc(fp); - if (c != EOF) - VSB_putc(vsb, c); - } while (c != EOF); - r = pclose(fp); - AZ(VSB_finish(vsb)); - vtc_dump(vl, 4, "shell_out", VSB_data(vsb), VSB_len(vsb)); - vtc_log(vl, 4, "shell_status = 0x%04x", WEXITSTATUS(r)); - if (WIFSIGNALED(r)) - vtc_log(vl, 4, "shell_signal = %d", WTERMSIG(r)); - - if (ok < 0 && !WEXITSTATUS(r) && !WIFSIGNALED(r)) - vtc_fatal(vl, "shell did not fail as expected"); - else if (ok >= 0 && WEXITSTATUS(r) != ok) - vtc_fatal(vl, "shell_exit not as expected: " - "got 0x%04x wanted 0x%04x", WEXITSTATUS(r), ok); - - if (expect != NULL) { - if (strstr(VSB_data(vsb), expect) == NULL) - vtc_fatal(vl, - "shell_expect not found: (\"%s\")", expect); - else - vtc_log(vl, 4, "shell_expect found"); - } else if (vre != NULL) { - if (VRE_exec(vre, VSB_data(vsb), VSB_len(vsb), 0, 0, - NULL, 0, NULL) < 1) - vtc_fatal(vl, - "shell_match failed: (\"%s\")", re); - else - vtc_log(vl, 4, "shell_match succeeded"); - VRE_free(&vre); - } - VSB_destroy(&vsb); -} - - -void -cmd_shell(CMD_ARGS) -{ - const char *expect = NULL; - const char *re = NULL; - int n; - int ok = 0; - - (void)priv; - (void)cmd; - - if (av == NULL) - return; - for (n = 1; av[n] != NULL; n++) { - if (!strcmp(av[n], "-err")) { - ok = -1; - } else if (!strcmp(av[n], "-exit")) { - n += 1; - ok = atoi(av[n]); - } else if (!strcmp(av[n], "-expect")) { - if (re != NULL) - vtc_fatal(vl, - "Cannot use -expect with -match"); - n += 1; - expect = av[n]; - } else if (!strcmp(av[n], "-match")) { - if (expect != NULL) - vtc_fatal(vl, - "Cannot use -match with -expect"); - n += 1; - re = av[n]; - } else { - break; - } - } - AN(av[n]); - cmd_shell_engine(vl, ok, av[n], expect, re); -} - -/* SECTION: err_shell err_shell - * - * This is very similar to the the ``shell`` command, except it takes a first - * string as argument before the command:: - * - * err_shell "foo" "echo foo" - * - * err_shell expect the shell command to fail AND stdout to match the string, - * failing the test case otherwise. - */ - -static void -cmd_err_shell(CMD_ARGS) -{ - (void)priv; - (void)cmd; - - if (av == NULL) - return; - AN(av[1]); - AN(av[2]); - AZ(av[3]); - vtc_log(vl, 1, - "NOTICE: err_shell is deprecated, use 'shell -err -expect'"); - cmd_shell_engine(vl, -1, av[2], av[1], NULL); -} - -/* SECTION: setenv setenv - * - * Set or change an environment variable:: - * - * setenv FOO "bar baz" - * - * The above will set the environment variable $FOO to the value - * provided. There is also an ``-ifunset`` argument which will only - * set the value if the the environment variable does not already - * exist:: - * - * setenv -ifunset FOO quux - */ -static void -cmd_setenv(CMD_ARGS) -{ - int r; - int force; - - (void)priv; - (void)cmd; - - if (av == NULL) - return; - AN(av[1]); - AN(av[2]); - - force = 1; - if (strcmp("-ifunset", av[1]) == 0) { - force = 0; - av++; - AN(av[2]); - } - if (av[3] != NULL) - vtc_fatal(vl, "CMD setenv: Unexpected argument '%s'", av[3]); - r = setenv(av[1], av[2], force); - if (r != 0) - vtc_log(vl, 0, "CMD setenv %s=\"%s\" failed: %s", - av[1], av[2], strerror(errno)); -} - -/* SECTION: delay delay - * - * Sleep for the number of seconds specified in the argument. The number - * can include a fractional part, e.g. 1.5. - */ -/* SECTION: stream.spec.delay delay - * - * Same as for the top-level delay. - */ -void -cmd_delay(CMD_ARGS) -{ - double f; - - (void)priv; - (void)cmd; - if (av == NULL) - return; - AN(av[1]); - AZ(av[2]); - f = VNUM(av[1]); - if (isnan(f)) - vtc_fatal(vl, "Syntax error in number (%s)", av[1]); - vtc_log(vl, 3, "delaying %g second(s)", f); - VTIM_sleep(f); -} - -/* SECTION: feature feature - * - * Test that the required feature(s) for a test are available, and skip - * the test otherwise; or change the interpretation of the test, as - * documented below. feature takes any number of arguments from this list: - * - * SO_RCVTIMEO_WORKS - * The SO_RCVTIMEO socket option is working - * 64bit - * The environment is 64 bits - * !OSX - * The environment is not OSX - * dns - * DNS lookups are working - * topbuild - * varnishtest has been started with '-i' - * root - * varnishtest has been invoked by the root user - * user_varnish - * The varnish user is present - * user_vcache - * The vcache user is present - * group_varnish - * The varnish group is present - * cmd - * A command line that should execute with a zero exit status - * ignore_unknown_macro - * Do not fail the test if a string of the form ${...} is not - * recognized as a macro. - * term - * Support for ADM3A terminal - * - * persistent_storage - * Varnish was built with the deprecated persistent storage. - * - * Be careful with ignore_unknown_macro, because it may cause a test with a - * misspelled macro to fail silently. You should only need it if you must - * run a test with strings of the form "${...}". - */ - -#if WITH_PERSISTENT_STORAGE -static const unsigned with_persistent_storage = 1; -#else -static const unsigned with_persistent_storage = 0; -#endif - -static int -test_term(struct vtclog *vl) -{ - FILE *p; - int a, b; - - p = popen("tput -T ansi.sys clear 2>&1", "r"); - if (p == NULL) - return (0); - a = fgetc(p); - b = fgetc(p); - if (a == 0x1b && b == '[') - return (1); - vtc_log(vl, 3, "No 'ansi.sys' terminfo entry."); - return (0); -} - -static void -cmd_feature(CMD_ARGS) -{ - int r; - int good; - - (void)priv; - (void)cmd; - - if (av == NULL) - return; - -#define FEATURE(nm, tst) \ - do { \ - if (!strcmp(*av, nm)) { \ - if (tst) { \ - good = 1; \ - } else { \ - vtc_stop = 2; \ - } \ - } \ - } while (0) - - for (av++; *av != NULL; av++) { - good = 0; - if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) { -#ifdef SO_RCVTIMEO_WORKS - good = 1; -#else - vtc_stop = 2; -#endif - } - - if (!strcmp(*av, "!OSX")) { -#if !defined(__APPLE__) || !defined(__MACH__) - good = 1; -#else - vtc_stop = 2; -#endif - } - FEATURE("pcre_jit", VRE_has_jit); - FEATURE("64bit", sizeof(void*) == 8); - FEATURE("dns", feature_dns); - FEATURE("topbuild", iflg); - FEATURE("root", !geteuid()); - FEATURE("user_varnish", getpwnam("varnish") != NULL); - FEATURE("user_vcache", getpwnam("vcache") != NULL); - FEATURE("group_varnish", getgrnam("varnish") != NULL); - FEATURE("term", test_term(vl)); - FEATURE("persistent_storage", with_persistent_storage); - - if (!strcmp(*av, "disable_aslr")) { - good = 1; -#ifdef HAVE_SYS_PERSONALITY_H - r = personality(0xffffffff); - r = personality(r | ADDR_NO_RANDOMIZE); - if (r < 0) { - good = 0; - vtc_stop = 2; - } -#endif - } else if (!strcmp(*av, "cmd")) { - av++; - if (*av == NULL) - vtc_fatal(vl, "Missing the command-line"); - r = system(*av); - if (WEXITSTATUS(r) == 0) - good = 1; - else - vtc_stop = 2; - } else if (!strcmp(*av, "ignore_unknown_macro")) { - ign_unknown_macro = 1; - good = 1; - } - if (good) - continue; - - if (!vtc_stop) - vtc_fatal(vl, "FAIL test, unknown feature: %s", *av); - else - vtc_log(vl, 1, - "SKIPPING test, lacking feature: %s", *av); - return; - } -} - /********************************************************************** * Execute a file */ static const struct cmds cmds[] = { #define CMD(n) { #n, cmd_##n }, - CMD(server) - CMD(client) - CMD(varnish) - CMD(delay) - CMD(varnishtest) - CMD(shell) - CMD(err_shell) - CMD(barrier) - CMD(feature) - CMD(logexpect) - CMD(process) - CMD(setenv) -#undef CMD +#include "cmds.h" { NULL, NULL } }; diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 8f0d130..387181c 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -69,16 +69,8 @@ void parse_string(const char *spec, const struct cmds *cmd, void *priv, struct vtclog *vl); int fail_out(void); -#define CMD(n) cmd_f cmd_##n -CMD(delay); -CMD(server); -CMD(client); -CMD(varnish); -CMD(barrier); -CMD(logexpect); -CMD(process); -CMD(shell); -#undef CMD +#define CMD(n) cmd_f cmd_##n; +#include "cmds.h" extern volatile sig_atomic_t vtc_error; /* Error, bail out */ extern int vtc_stop; /* Abandon current test, no error */ @@ -90,6 +82,7 @@ extern struct vsb *params_vsb; extern int leave_temp; extern int vtc_witness; extern int feature_dns; +extern int ign_unknown_macro; void init_server(void); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c new file mode 100644 index 0000000..30f404e --- /dev/null +++ b/bin/varnishtest/vtc_misc.c @@ -0,0 +1,458 @@ +/*- + * Copyright (c) 2008-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vtc.h" + +#include "vnum.h" +#include "vre.h" +#include "vtim.h" + +/* SECTION: varnishtest varnishtest + * + * This should be the first command in your vtc as it will identify the test + * case with a short yet descriptive sentence. It takes exactly one argument, a + * string, eg:: + * + * varnishtest "Check that varnishtest is actually a valid command" + * + * It will also print that string in the log. + */ + +void v_matchproto_(cmd_f) +cmd_varnishtest(CMD_ARGS) +{ + + (void)priv; + (void)cmd; + (void)vl; + + if (av == NULL) + return; + AZ(strcmp(av[0], "varnishtest")); + + vtc_log(vl, 1, "TEST %s", av[1]); + AZ(av[2]); +} + +/* SECTION: shell shell + * + * Pass the string given as argument to a shell. If you have multiple + * commands to run, you can use curly barces to describe a multi-lines + * script, eg:: + * + * shell { + * echo begin + * cat /etc/fstab + * echo end + * } + * + * By default a zero exit code is expected, otherwise the vtc will fail. + * + * Notice that the commandstring is prefixed with "exec 2>&1;" to join + * stderr and stdout back to the varnishtest process. + * + * Optional arguments: + * + * \-err + * Expect non-zero exit code. + * + * \-exit N + * Expect exit code N instead of zero. + * + * \-expect STRING + * Expect string to be found in stdout+err. + * + * \-match REGEXP + * Expect regexp to match the stdout+err output. + */ +/* SECTION: client-server.spec.shell shell + * + * Same as for the top-level shell. + */ + +static void +cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, + const char *expect, const char *re) +{ + struct vsb *vsb; + FILE *fp; + vre_t *vre = NULL; + const char *errptr; + int r, c; + int err; + + AN(vl); + AN(cmd); + vsb = VSB_new_auto(); + AN(vsb); + if (re != NULL) { + vre = VRE_compile(re, 0, &errptr, &err); + if (vre == NULL) + vtc_fatal(vl, "shell_match invalid regexp (\"%s\")", + re); + } + VSB_printf(vsb, "exec 2>&1 ; %s", cmd); + AZ(VSB_finish(vsb)); + vtc_dump(vl, 4, "shell_cmd", VSB_data(vsb), -1); + fp = popen(VSB_data(vsb), "r"); + if (fp == NULL) + vtc_fatal(vl, "popen fails: %s", strerror(errno)); + VSB_clear(vsb); + do { + c = getc(fp); + if (c != EOF) + VSB_putc(vsb, c); + } while (c != EOF); + r = pclose(fp); + AZ(VSB_finish(vsb)); + vtc_dump(vl, 4, "shell_out", VSB_data(vsb), VSB_len(vsb)); + vtc_log(vl, 4, "shell_status = 0x%04x", WEXITSTATUS(r)); + if (WIFSIGNALED(r)) + vtc_log(vl, 4, "shell_signal = %d", WTERMSIG(r)); + + if (ok < 0 && !WEXITSTATUS(r) && !WIFSIGNALED(r)) + vtc_fatal(vl, "shell did not fail as expected"); + else if (ok >= 0 && WEXITSTATUS(r) != ok) + vtc_fatal(vl, "shell_exit not as expected: " + "got 0x%04x wanted 0x%04x", WEXITSTATUS(r), ok); + + if (expect != NULL) { + if (strstr(VSB_data(vsb), expect) == NULL) + vtc_fatal(vl, + "shell_expect not found: (\"%s\")", expect); + else + vtc_log(vl, 4, "shell_expect found"); + } else if (vre != NULL) { + if (VRE_exec(vre, VSB_data(vsb), VSB_len(vsb), 0, 0, + NULL, 0, NULL) < 1) + vtc_fatal(vl, + "shell_match failed: (\"%s\")", re); + else + vtc_log(vl, 4, "shell_match succeeded"); + VRE_free(&vre); + } + VSB_destroy(&vsb); +} + + +void +cmd_shell(CMD_ARGS) +{ + const char *expect = NULL; + const char *re = NULL; + int n; + int ok = 0; + + (void)priv; + (void)cmd; + + if (av == NULL) + return; + for (n = 1; av[n] != NULL; n++) { + if (!strcmp(av[n], "-err")) { + ok = -1; + } else if (!strcmp(av[n], "-exit")) { + n += 1; + ok = atoi(av[n]); + } else if (!strcmp(av[n], "-expect")) { + if (re != NULL) + vtc_fatal(vl, + "Cannot use -expect with -match"); + n += 1; + expect = av[n]; + } else if (!strcmp(av[n], "-match")) { + if (expect != NULL) + vtc_fatal(vl, + "Cannot use -match with -expect"); + n += 1; + re = av[n]; + } else { + break; + } + } + AN(av[n]); + cmd_shell_engine(vl, ok, av[n], expect, re); +} + +/* SECTION: err_shell err_shell + * + * This is very similar to the the ``shell`` command, except it takes a first + * string as argument before the command:: + * + * err_shell "foo" "echo foo" + * + * err_shell expect the shell command to fail AND stdout to match the string, + * failing the test case otherwise. + */ + +void v_matchproto_(cmd_f) +cmd_err_shell(CMD_ARGS) +{ + (void)priv; + (void)cmd; + + if (av == NULL) + return; + AN(av[1]); + AN(av[2]); + AZ(av[3]); + vtc_log(vl, 1, + "NOTICE: err_shell is deprecated, use 'shell -err -expect'"); + cmd_shell_engine(vl, -1, av[2], av[1], NULL); +} + +/* SECTION: setenv setenv + * + * Set or change an environment variable:: + * + * setenv FOO "bar baz" + * + * The above will set the environment variable $FOO to the value + * provided. There is also an ``-ifunset`` argument which will only + * set the value if the the environment variable does not already + * exist:: + * + * setenv -ifunset FOO quux + */ + +void v_matchproto_(cmd_f) +cmd_setenv(CMD_ARGS) +{ + int r; + int force; + + (void)priv; + (void)cmd; + + if (av == NULL) + return; + AN(av[1]); + AN(av[2]); + + force = 1; + if (strcmp("-ifunset", av[1]) == 0) { + force = 0; + av++; + AN(av[2]); + } + if (av[3] != NULL) + vtc_fatal(vl, "CMD setenv: Unexpected argument '%s'", av[3]); + r = setenv(av[1], av[2], force); + if (r != 0) + vtc_log(vl, 0, "CMD setenv %s=\"%s\" failed: %s", + av[1], av[2], strerror(errno)); +} + +/* SECTION: delay delay + * + * Sleep for the number of seconds specified in the argument. The number + * can include a fractional part, e.g. 1.5. + */ +/* SECTION: stream.spec.delay delay + * + * Same as for the top-level delay. + */ +void +cmd_delay(CMD_ARGS) +{ + double f; + + (void)priv; + (void)cmd; + if (av == NULL) + return; + AN(av[1]); + AZ(av[2]); + f = VNUM(av[1]); + if (isnan(f)) + vtc_fatal(vl, "Syntax error in number (%s)", av[1]); + vtc_log(vl, 3, "delaying %g second(s)", f); + VTIM_sleep(f); +} + +/* SECTION: feature feature + * + * Test that the required feature(s) for a test are available, and skip + * the test otherwise; or change the interpretation of the test, as + * documented below. feature takes any number of arguments from this list: + * + * SO_RCVTIMEO_WORKS + * The SO_RCVTIMEO socket option is working + * 64bit + * The environment is 64 bits + * !OSX + * The environment is not OSX + * dns + * DNS lookups are working + * topbuild + * varnishtest has been started with '-i' + * root + * varnishtest has been invoked by the root user + * user_varnish + * The varnish user is present + * user_vcache + * The vcache user is present + * group_varnish + * The varnish group is present + * cmd + * A command line that should execute with a zero exit status + * ignore_unknown_macro + * Do not fail the test if a string of the form ${...} is not + * recognized as a macro. + * term + * Support for ADM3A terminal + * + * persistent_storage + * Varnish was built with the deprecated persistent storage. + * + * Be careful with ignore_unknown_macro, because it may cause a test with a + * misspelled macro to fail silently. You should only need it if you must + * run a test with strings of the form "${...}". + */ + +#if WITH_PERSISTENT_STORAGE +static const unsigned with_persistent_storage = 1; +#else +static const unsigned with_persistent_storage = 0; +#endif + +static int +test_term(struct vtclog *vl) +{ + FILE *p; + int a, b; + + p = popen("tput -T ansi.sys clear 2>&1", "r"); + if (p == NULL) + return (0); + a = fgetc(p); + b = fgetc(p); + if (a == 0x1b && b == '[') + return (1); + vtc_log(vl, 3, "No 'ansi.sys' terminfo entry."); + return (0); +} + +void v_matchproto_(cmd_f) +cmd_feature(CMD_ARGS) +{ + int r; + int good; + + (void)priv; + (void)cmd; + + if (av == NULL) + return; + +#define FEATURE(nm, tst) \ + do { \ + if (!strcmp(*av, nm)) { \ + if (tst) { \ + good = 1; \ + } else { \ + vtc_stop = 2; \ + } \ + } \ + } while (0) + + for (av++; *av != NULL; av++) { + good = 0; + if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) { +#ifdef SO_RCVTIMEO_WORKS + good = 1; +#else + vtc_stop = 2; +#endif + } + + if (!strcmp(*av, "!OSX")) { +#if !defined(__APPLE__) || !defined(__MACH__) + good = 1; +#else + vtc_stop = 2; +#endif + } + FEATURE("pcre_jit", VRE_has_jit); + FEATURE("64bit", sizeof(void*) == 8); + FEATURE("dns", feature_dns); + FEATURE("topbuild", iflg); + FEATURE("root", !geteuid()); + FEATURE("user_varnish", getpwnam("varnish") != NULL); + FEATURE("user_vcache", getpwnam("vcache") != NULL); + FEATURE("group_varnish", getgrnam("varnish") != NULL); + FEATURE("term", test_term(vl)); + FEATURE("persistent_storage", with_persistent_storage); + + if (!strcmp(*av, "disable_aslr")) { + good = 1; +#ifdef HAVE_SYS_PERSONALITY_H + r = personality(0xffffffff); + r = personality(r | ADDR_NO_RANDOMIZE); + if (r < 0) { + good = 0; + vtc_stop = 2; + } +#endif + } else if (!strcmp(*av, "cmd")) { + av++; + if (*av == NULL) + vtc_fatal(vl, "Missing the command-line"); + r = system(*av); + if (WEXITSTATUS(r) == 0) + good = 1; + else + vtc_stop = 2; + } else if (!strcmp(*av, "ignore_unknown_macro")) { + ign_unknown_macro = 1; + good = 1; + } + if (good) + continue; + + if (!vtc_stop) + vtc_fatal(vl, "FAIL test, unknown feature: %s", *av); + else + vtc_log(vl, 1, + "SKIPPING test, lacking feature: %s", *av); + return; + } +} From nils.goroll at uplex.de Tue Mar 27 09:14:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 27 Mar 2018 09:14:09 +0000 (UTC) Subject: [master] 943bbfa grammar micro-fix Message-ID: <20180327091409.1A609B5766@lists.varnish-cache.org> commit 943bbfa21d63f5913023cd6fed87cd7412054ad2 Author: Nils Goroll Date: Tue Mar 27 11:12:26 2018 +0200 grammar micro-fix diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 0c04278..1bf1de7 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -54,7 +54,7 @@ " tag we'll look for, a prefix to look for (can be empty," \ " but must be terminated by a colon) and the field number" \ " of the value we are interested in. min and max are the" \ - " boundaries of the graph in power of ten and default to" \ + " boundaries of the graph in powers of ten and default to" \ " -6 and 3." \ ) From phk at FreeBSD.org Tue Mar 27 09:17:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 09:17:08 +0000 (UTC) Subject: [master] 3db9785 Also move personality.h #include Message-ID: <20180327091708.3FACCB5882@lists.varnish-cache.org> commit 3db978554a69b5aea1a2baca0be7de1d7288df77 Author: Poul-Henning Kamp Date: Tue Mar 27 09:15:06 2018 +0000 Also move personality.h #include diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 76f1e7a..53d3232 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -42,10 +42,6 @@ #include "vav.h" #include "vtim.h" -#ifdef HAVE_SYS_PERSONALITY_H -# include -#endif - #define MAX_TOKENS 200 volatile sig_atomic_t vtc_error; /* Error encountered */ diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 30f404e..55294e8 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -39,6 +39,10 @@ #include #include +#ifdef HAVE_SYS_PERSONALITY_H +# include +#endif + #include "vtc.h" #include "vnum.h" From phk at FreeBSD.org Tue Mar 27 11:48:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 11:48:11 +0000 (UTC) Subject: [master] 0f53c78 Expose VPF_Read() Message-ID: <20180327114811.DC7E8616F6@lists.varnish-cache.org> commit 0f53c785d4e815cc2800eec24a7fd008f47ef35f Author: Poul-Henning Kamp Date: Tue Mar 27 11:11:34 2018 +0000 Expose VPF_Read() diff --git a/include/vpf.h b/include/vpf.h index 822f2d3..d5266ac 100644 --- a/include/vpf.h +++ b/include/vpf.h @@ -36,5 +36,6 @@ struct vpf_fh *VPF_Open(const char *path, mode_t mode, pid_t *pidptr); int VPF_Write(struct vpf_fh *pfh); int VPF_Close(struct vpf_fh *pfh); int VPF_Remove(struct vpf_fh *pfh); +int VPF_read(const char *path, pid_t *); #endif diff --git a/lib/libvarnish/vpf.c b/lib/libvarnish/vpf.c index 5626b9e..915a028 100644 --- a/lib/libvarnish/vpf.c +++ b/lib/libvarnish/vpf.c @@ -72,8 +72,8 @@ vpf_verify(const struct vpf_fh *pfh) return (0); } -static int -vpf_read(const char *path, pid_t *pidptr) +int +VPF_read(const char *path, pid_t *pidptr) { char buf[16], *endptr; int error, fd, i; @@ -129,7 +129,7 @@ VPF_Open(const char *path, mode_t mode, pid_t *pidptr) O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NONBLOCK, mode); if (fd == -1) { if (errno == EWOULDBLOCK && pidptr != NULL) { - errno = vpf_read(pfh->pf_path, pidptr); + errno = VPF_read(pfh->pf_path, pidptr); if (errno == 0) errno = EEXIST; } From phk at FreeBSD.org Tue Mar 27 11:48:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 11:48:11 +0000 (UTC) Subject: [master] c4d8fad Make listening to stdout/err a convenience function Message-ID: <20180327114811.F4189616F9@lists.varnish-cache.org> commit c4d8fadc5490306576e0d5ed2284cca9cd2d3177 Author: Poul-Henning Kamp Date: Tue Mar 27 11:20:19 2018 +0000 Make listening to stdout/err a convenience function diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 387181c..8878e1d 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -132,6 +132,7 @@ struct vsb *vtc_hex_to_bin(struct vtclog *vl, const char *arg); void vtc_expect(struct vtclog *, const char *, const char *, const char *, const char *, const char *); void vtc_wait4(struct vtclog *, long, int, int, int); +void *vtc_record(struct vtclog *, int); /* vtc_term.c */ struct term *Term_New(struct vtclog *, int, int); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 1a9ff10..39cafbe 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -31,15 +31,19 @@ #include #include #include +#include #include #include +#include #include #include +#include "vtc.h" + #include "vct.h" #include "vnum.h" #include "vre.h" -#include "vtc.h" +#include "vtcp.h" struct vsb * vtc_hex_to_bin(struct vtclog *vl, const char *arg) @@ -180,3 +184,34 @@ vtc_wait4(struct vtclog *vl, long pid, WIFSIGNALED(status) ? WTERMSIG(status) : 0, WCOREDUMP(status)); } + +void * +vtc_record(struct vtclog *vl, int fd) +{ + char buf[65536]; + struct pollfd fds[1]; + int i; + + (void)VTCP_nonblocking(fd); + while (1) { + memset(fds, 0, sizeof fds); + fds->fd = fd; + fds->events = POLLIN; + i = poll(fds, 1, 10000); + if (i == 0) + continue; + if (fds->revents & POLLIN) { + i = read(fd, buf, sizeof buf - 1); + if (i > 0) { + buf[i] = '\0'; + vtc_dump(vl, 3, "debug", buf, -2); + } + } + if (fds->revents & (POLLERR|POLLHUP)) { + vtc_log(vl, 4, "STDOUT poll 0x%x", fds->revents); + break; + } + } + return (NULL); +} + diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index a1a5ed8..d09d7cf 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -374,32 +374,9 @@ static void * varnish_thread(void *priv) { struct varnish *v; - char buf[65536]; - struct pollfd fds[1]; - int i; CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC); - (void)VTCP_nonblocking(v->fds[0]); - while (1) { - memset(fds, 0, sizeof fds); - fds->fd = v->fds[0]; - fds->events = POLLIN; - i = poll(fds, 1, 10000); - if (i == 0) - continue; - if (fds->revents & POLLIN) { - i = read(v->fds[0], buf, sizeof buf - 1); - if (i > 0) { - buf[i] = '\0'; - vtc_dump(v->vl, 3, "debug", buf, -2); - } - } - if (fds->revents & (POLLERR|POLLHUP)) { - vtc_log(v->vl, 4, "STDOUT poll 0x%x", fds->revents); - break; - } - } - return (NULL); + return (vtc_record(v->vl, v->fds[0])); } /********************************************************************** From fgsch at lodoss.net Tue Mar 27 18:40:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Mar 2018 18:40:11 +0000 (UTC) Subject: [master] 77663a9 Polish Message-ID: <20180327184011.C544E94D09@lists.varnish-cache.org> commit 77663a93f2c351fa86476bed45bcae52fe5f8f49 Author: Federico G. Schwindt Date: Mon Mar 26 13:46:23 2018 -0300 Polish diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index eb906b8..27e7e70 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -531,9 +531,6 @@ vbf_figure_out_vfp(struct busyobj *bo) return (0); } - if (! cache_param->http_gzip_support) - bo->do_gzip = bo->do_gunzip = 0; - /* No body -> done */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { @@ -543,6 +540,9 @@ vbf_figure_out_vfp(struct busyobj *bo) return (0); } + if (!cache_param->http_gzip_support) + bo->do_gzip = bo->do_gunzip = 0; + bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(bo->is_gzip == 0 || bo->is_gunzip == 0); From fgsch at lodoss.net Tue Mar 27 18:40:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Mar 2018 18:40:11 +0000 (UTC) Subject: [master] 5005595 Whitespace and related OCD Message-ID: <20180327184011.E071294D0C@lists.varnish-cache.org> commit 5005595f263dcbc129b70e2445a393195cc9cc02 Author: Federico G. Schwindt Date: Tue Mar 27 01:06:49 2018 -0300 Whitespace and related OCD diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 74b29d6..392baba 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -443,11 +443,11 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); if (vrt->path == NULL) - assert(vrt->ipv4_suckaddr != NULL - || vrt->ipv6_suckaddr != NULL); + assert(vrt->ipv4_suckaddr != NULL || + vrt->ipv6_suckaddr != NULL); else - assert(vrt->ipv4_suckaddr == NULL - && vrt->ipv6_suckaddr == NULL); + assert(vrt->ipv4_suckaddr == NULL && + vrt->ipv6_suckaddr == NULL); vcl = ctx->vcl; AN(vcl); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 1db47c4..4c8fad0 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -186,9 +186,9 @@ cnt_deliver(struct worker *wrk, struct req *req) assert(wrk->handling == VCL_RET_DELIVER); - if (req->esi_level == 0 - && http_IsStatus(req->resp, 200) - && req->http->conds && RFC2616_Do_Cond(req)) + if (req->esi_level == 0 && + http_IsStatus(req->resp, 200) && + req->http->conds && RFC2616_Do_Cond(req)) http_PutResponse(req->resp, "HTTP/1.1", 304, NULL); req->req_step = R_STP_TRANSMIT; diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index e9fa812..86777ae 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -656,8 +656,8 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, struct vtp_cs vcs; const struct cp_methods *methods; - assert((uds != NULL && ip4 == NULL && ip6 == NULL) - || (uds == NULL && (ip4 != NULL || ip6 != NULL))); + assert((uds != NULL && ip4 == NULL && ip6 == NULL) || + (uds == NULL && (ip4 != NULL || ip6 != NULL))); INIT_OBJ(&vcs, VTP_CS_MAGIC); vcs.ip4 = ip4; vcs.ip6 = ip6; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 12280e1..30adf54 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -1132,7 +1132,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, ctx.ws = req->ws; } if (bo != NULL) { - if(req) + if (req) assert(method == VCL_MET_PIPE); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 4e53969..9580a32 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -103,8 +103,8 @@ vrt_priv_dynamic(VRT_CTX, struct vrt_privs *vps, uintptr_t id, VTAILQ_FOREACH(vp, &vps->privs, list) { CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC); - if (vp->vcl == ctx->vcl && vp->id == id - && vp->vmod_id == vmod_id) + if (vp->vcl == ctx->vcl && vp->id == id && + vp->vmod_id == vmod_id) return (vp->priv); } ALLOC_OBJ(vp, VRT_PRIV_MAGIC); diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 8ef0f3a..fbf7f26 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -102,8 +102,8 @@ mac_opensocket(struct listen_sock *ls) CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC); assert(ls->uds); errno = 0; - if (ls->perms->mode != 0 - && chmod(ls->endpoint, ls->perms->mode) != 0) + if (ls->perms->mode != 0 && + chmod(ls->endpoint, ls->perms->mode) != 0) return errno; if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0) return errno; @@ -265,8 +265,8 @@ MAC_Arg(const char *spec) la->name = name; if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL) - ARGV_ERR("Unix domain socket addresses must be absolute paths " - "in -a (%s)\n", la->endpoint); + ARGV_ERR("Unix domain socket addresses must be" + " absolute paths in -a (%s)\n", la->endpoint); if (*la->endpoint == '/' && heritage.min_vcl < 41) heritage.min_vcl = 41; @@ -277,16 +277,16 @@ MAC_Arg(const char *spec) if ((eq = strchr(av[i], '=')) == NULL) { if (xp != NULL) - ARGV_ERR("Too many protocol sub-args in -a " - "(%s)\n", av[i]); + ARGV_ERR("Too many protocol sub-args" + " in -a (%s)\n", av[i]); xp = XPORT_Find(av[i]); if (xp == NULL) ARGV_ERR("Unknown protocol '%s'\n", av[i]); continue; } if (la->endpoint[0] != '/') - ARGV_ERR("Invalid sub-arg %s for IP addresses in -a\n", - av[i]); + ARGV_ERR("Invalid sub-arg %s for IP addresses" + " in -a\n", av[i]); val = eq + 1; len = eq - av[i]; diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 9ec6f84..5f821ac 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -449,7 +449,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) "PROXY2: Ignoring TLV"); return (0); } - switch(d[0]) { + switch (d[0]) { case PP2_TYPE_CRC32C: { uint32_t n_crc32c = vbe32dec(d+3); diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 53d3232..62ac1d0 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -256,8 +256,7 @@ macro_expand(struct vtclog *vl, const char *text) NEEDLESS(return (NULL)); } VSB_printf(vsb, "${%.*s}", (int)(q - p), p); - } - else { + } else { VSB_printf(vsb, "%s", m); free(m); } diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index d950608..0bc0925 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -544,8 +544,8 @@ VSB_quote_pfx(struct vsb *s, const char *pfx, const void *v, int len, int how) } if (!quote && !(how & (VSB_QUOTE_JSON|VSB_QUOTE_CSTR))) { (void)VSB_bcat(s, p, len); - if ((how & (VSB_QUOTE_UNSAFE|VSB_QUOTE_NONL)) - && p[len-1] != '\n') + if ((how & (VSB_QUOTE_UNSAFE|VSB_QUOTE_NONL)) && + p[len-1] != '\n') (void)VSB_putc(s, '\n'); return; } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 17209dc..032a6e9 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -491,7 +491,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, if (extra == NULL) extra = ""; VTAILQ_INIT(&head); - for(;vv != NULL; vv = VTAILQ_NEXT(vv, list)) { + for (;vv != NULL; vv = VTAILQ_NEXT(vv, list)) { assert(vv->type == VJSN_ARRAY); fa = calloc(1, sizeof *fa); AN(fa); diff --git a/lib/libvmod_blob/url.c b/lib/libvmod_blob/url.c index 8ba47fc..84f5d0f 100644 --- a/lib/libvmod_blob/url.c +++ b/lib/libvmod_blob/url.c @@ -150,8 +150,8 @@ url_decode(const enum encoding dec, char *restrict const buf, } break; case PERCENT: - if (isoutofrange(*s) - || (nib = nibble[*s - '0']) == ILL) { + if (isoutofrange(*s) || + (nib = nibble[*s - '0']) == ILL) { errno = EINVAL; return -1; } @@ -162,8 +162,8 @@ url_decode(const enum encoding dec, char *restrict const buf, errno = ENOMEM; return -1; } - if (isoutofrange(*s) - || (nib2 = nibble[*s - '0']) == ILL) { + if (isoutofrange(*s) || + (nib2 = nibble[*s - '0']) == ILL) { errno = EINVAL; return -1; } diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index f053601..fcf74a7 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -128,6 +128,7 @@ xyzzy_obj_test_priv_call(VRT_CTX, (void)o; xyzzy_test_priv_call(ctx, priv); } + VCL_VOID v_matchproto_() xyzzy_obj_test_priv_vcl(VRT_CTX, struct xyzzy_debug_obj *o, struct vmod_priv *priv) @@ -135,6 +136,7 @@ xyzzy_obj_test_priv_vcl(VRT_CTX, (void)o; xyzzy_test_priv_vcl(ctx, priv); } + VCL_STRING v_matchproto_() xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) @@ -142,6 +144,7 @@ xyzzy_obj_test_priv_task(VRT_CTX, (void)o; return (xyzzy_test_priv_task(ctx, priv, s)); } + VCL_STRING v_matchproto_() xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) From fgsch at lodoss.net Tue Mar 27 18:40:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Mar 2018 18:40:12 +0000 (UTC) Subject: [master] 90db2ad Plug harmless leak Message-ID: <20180327184012.03A4494D10@lists.varnish-cache.org> commit 90db2adaf8492b82ddf660e40e5d9b3d9bb9f11c Author: Federico G. Schwindt Date: Tue Mar 27 01:06:55 2018 -0300 Plug harmless leak diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 62ac1d0..1f72492 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -460,7 +460,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir, { FILE *f; struct vsb *vsb; - const char *p; + char *p; (void)signal(SIGPIPE, SIG_IGN); @@ -474,8 +474,12 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vsb = VSB_new_auto(); AN(vsb); - if (*fn != '/') - VSB_cat(vsb, macro_get("pwd", NULL)); + if (*fn != '/') { + p = macro_get("pwd", NULL); + AN(p); + VSB_cat(vsb, p); + free(p); + } p = strrchr(fn, '/'); if (p != NULL) { VSB_putc(vsb, '/'); From fgsch at lodoss.net Tue Mar 27 18:40:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Mar 2018 18:40:12 +0000 (UTC) Subject: [master] 3c90fdc Typos and formatting Message-ID: <20180327184012.218D994D1A@lists.varnish-cache.org> commit 3c90fdcc3a88989d1639b9ef482682a44fafee48 Author: Federico G. Schwindt Date: Tue Mar 27 08:53:16 2018 -0300 Typos and formatting diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 1c05520..4e0762f 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -91,7 +91,7 @@ * Along the way the BSD people figured out that it was a bother * to carry the length argument separately, and added that to the * format of sockaddr, but other groups found this unclean, as - * the length was already an explicit paramter. + * the length was already an explicit parameter. * * The net result of this is that your "portable" code, must take * care to handle the "sa_len" member on kernels which have it, diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc index 341f3fe..0a7e652 100644 --- a/lib/libvmod_blob/vmod.vcc +++ b/lib/libvmod_blob/vmod.vcc @@ -138,7 +138,7 @@ character ``=``. The ``BASE64URLNOPAD`` encoding uses the same alphabet as ``BASE6URL``, but leaves out the padding. Thus the length of an -encoding with this scheme is not necessarily a mutltiple of four. +encoding with this scheme is not necessarily a multiple of four. The ``case`` ENUM MUST be set to ``DEFAULT`` for for all of the ``BASE64*`` encodings. @@ -202,7 +202,7 @@ $Function STRING encode(ENUM {IDENTITY, BASE64, BASE64URL, BASE64URLNOPAD, HEX, URL} encoding="IDENTITY", ENUM {LOWER, UPPER, DEFAULT} case="DEFAULT", BLOB blob) -Returns a string representation of the BLOB ``blob`` as specifed by +Returns a string representation of the BLOB ``blob`` as specified by ``encoding``. ``case`` determines the case of hex digits for the ``HEX`` and ``URL`` encodings, and is ignored for the other encodings. diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 586c9db..a1a625f 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -78,7 +78,6 @@ Description Add a backend to the round-robin director. Example vdir.add_backend(backend1); - vdir.add_backend(backend2); $Method VOID .remove_backend(BACKEND) @@ -86,7 +85,6 @@ Description Remove a backend from the round-robin director. Example vdir.remove_backend(backend1); - vdir.remove_backend(backend2); $Method BACKEND .backend() @@ -121,7 +119,6 @@ Description Example vdir.add_backend(backend1); - vdir.add_backend(backend2); $Method VOID .remove_backend(BACKEND) @@ -129,7 +126,6 @@ Description Remove a backend from the director. Example vdir.remove_backend(backend1); - vdir.remove_backend(backend2); $Method BACKEND .backend() @@ -162,9 +158,9 @@ Description director. Example - # 2/3 to backend1, 1/3 to backend2. - vdir.add_backend(backend1, 10.0); - vdir.add_backend(backend2, 5.0); + | # 2/3 to backend1, 1/3 to backend2. + | vdir.add_backend(backend1, 10.0); + | vdir.add_backend(backend2, 5.0); $Method VOID .remove_backend(BACKEND) @@ -172,7 +168,6 @@ Description Remove a backend from the director. Example vdir.remove_backend(backend1); - vdir.remove_backend(backend2); $Method BACKEND .backend() @@ -205,7 +200,6 @@ Description Example vdir.add_backend(backend1, 1.0); - vdir.add_backend(backend2, 1.0); $Method VOID .remove_backend(BACKEND) @@ -213,7 +207,6 @@ Description Remove a backend from the director. Example vdir.remove_backend(backend1); - vdir.remove_backend(backend2); $Method BACKEND .backend(STRING_LIST) @@ -222,8 +215,8 @@ Description Use the string or list of strings provided to pick the backend. Example - # pick a backend based on the cookie header from the client - set req.backend_hint = vdir.backend(req.http.cookie); + | # pick a backend based on the cookie header from the client + | set req.backend_hint = vdir.backend(req.http.cookie); $Object shard() @@ -573,7 +566,7 @@ is _not_ the order given when backends are added. associated parameter set affect the sharding decision once the director resolves to an actual backend. - * If other paramter arguments are also given, they have preference + * If other parameter arguments are also given, they have preference and are kept even if the parameter set given by the `param` argument is subsequently changed within the same backend request. @@ -647,17 +640,17 @@ use. See `func_shard.backend`_. $Method INT .get_alt() -Get the `alt` paramter which a shard director using this parameter +Get the `alt` parameter which a shard director using this parameter object would use. See `func_shard.backend`_. $Method REAL .get_warmup() -Get the `warmup` paramter which a shard director using this parameter +Get the `warmup` parameter which a shard director using this parameter object would use. See `func_shard.backend`_. $Method BOOL .get_rampup() -Get the `rampup` paramter which a shard director using this parameter +Get the `rampup` parameter which a shard director using this parameter object would use. See `func_shard.backend`_. $Method STRING .get_healthy() diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 9b42e5a..168f15a 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -86,7 +86,7 @@ struct vmod_directors_shard_param { const struct vmod_directors_shard_param *defaults; enum vmod_directors_shard_param_scope scope; - /* paramters */ + /* parameters */ enum by_e by; enum healthy_e healthy; uint32_t mask; From fgsch at lodoss.net Tue Mar 27 18:40:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Mar 2018 18:40:12 +0000 (UTC) Subject: [master] 39d6244 Document do_g[un]zip wrt http_gzip_support Message-ID: <20180327184012.40B6A94D21@lists.varnish-cache.org> commit 39d6244acfc19fdc646c0f9bbf3cefb6a3faecbe Author: Federico G. Schwindt Date: Tue Mar 27 09:02:12 2018 -0300 Document do_g[un]zip wrt http_gzip_support Addresses #2626. diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 253da0b..c4ded7c 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -739,6 +739,9 @@ beresp.do_gzip Set to `true` to gzip the object while storing it. + If `http_gzip_support` is disabled, setting this variable + has no effect. + beresp.do_gunzip Type: BOOL @@ -752,6 +755,9 @@ beresp.do_gunzip Set to `true` to gunzip the object while storing it in the cache. + If `http_gzip_support` is disabled, setting this variable + has no effect. + beresp.was_304 Type: BOOL From phk at FreeBSD.org Tue Mar 27 20:03:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 20:03:11 +0000 (UTC) Subject: [master] 4abcaa4 Add support for testing HA-proxy Message-ID: <20180327200311.33D9496813@lists.varnish-cache.org> commit 4abcaa49f109184d3de501c58f1abf3a7db1944e Author: Poul-Henning Kamp Date: Tue Mar 27 19:56:11 2018 +0000 Add support for testing HA-proxy Also a trivial coverage tes which runs if haproxy is in $PATH. Submitted by: Frederic Lecaille diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index efbac2a..c2c81d7 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -33,6 +33,7 @@ varnishtest_SOURCES = \ vtc.c \ vtc_barrier.c \ vtc_client.c \ + vtc_haproxy.c \ vtc_h2_dectbl.h \ vtc_h2_enctbl.h \ vtc_h2_hpack.c \ diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h index 58c2e7b..290237d 100644 --- a/bin/varnishtest/cmds.h +++ b/bin/varnishtest/cmds.h @@ -34,7 +34,7 @@ CMD(client) CMD(delay) CMD(err_shell) CMD(feature) -//CMD(haproxy) +CMD(haproxy) CMD(logexpect) CMD(process) CMD(server) diff --git a/bin/varnishtest/programs.h b/bin/varnishtest/programs.h index 4849005..f8875ce 100644 --- a/bin/varnishtest/programs.h +++ b/bin/varnishtest/programs.h @@ -27,6 +27,7 @@ * */ +VTC_PROG(haproxy) VTC_PROG(varnishadm) VTC_PROG(varnishd) VTC_PROG(varnishhist) diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index caadbd9..35d598d 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -21,6 +21,7 @@ Naming scheme id ~ ^e --> ESI tests id ~ ^f --> Security related tests id ~ ^g --> GZIP tests + id ~ ^h --> HAproxy tests id ~ ^j --> JAIL tests id ~ ^l --> VSL tests id ~ ^m --> VMOD tests excluding director diff --git a/bin/varnishtest/tests/h00001.vtc b/bin/varnishtest/tests/h00001.vtc new file mode 100644 index 0000000..3c599a5 --- /dev/null +++ b/bin/varnishtest/tests/h00001.vtc @@ -0,0 +1,32 @@ +varnishtest "Basic HAproxy test" + +feature ignore_unknown_macro + +feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'} + +server s1 { + rxreq + txresp -body "s1 >>> Hello world!" +} -start + +haproxy h1 -arg "-d" -conf { + defaults + mode http + timeout connect 5s + timeout server 30s + timeout client 30s + + backend be1 + server srv1 ${s1_addr}:${s1_port} + + frontend http1 + use_backend be1 + bind "fd@${fe1}" +} -start + +client c1 -connect ${h1_fe1_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.body == "s1 >>> Hello world!" +} -run diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c new file mode 100644 index 0000000..17c62fb --- /dev/null +++ b/bin/varnishtest/vtc_haproxy.c @@ -0,0 +1,616 @@ +/*- + * Copyright (c) 2008-2018 Varnish Software AS + * All rights reserved. + * + * Author: Fr?d?ric L?caille + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "vtc.h" + +#include "vav.h" +#include "vfil.h" +#include "vpf.h" +#include "vtcp.h" +#include "vtim.h" + +#define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" +#define HAPROXY_OPT_WORKER "-W" +#define HAPROXY_OPT_DAEMON "-D" +#define HAPROXY_OPT_CHECK_MODE "-c" +#define HAPROXY_SIGNAL SIGINT +#define HAPROXY_EXPECT_EXIT (128 + HAPROXY_SIGNAL) + +struct haproxy { + unsigned magic; +#define HAPROXY_MAGIC 0x8a45cf75 + char *name; + struct vtclog *vl; + VTAILQ_ENTRY(haproxy) list; + + char *filename; + struct vsb *args; + int opt_worker; + int opt_daemon; + int opt_check_mode; + char *pid_fn; + pid_t pid; + pid_t ppid; + int fds[4]; + char *cfg_fn; + + pthread_t tp; + int expect_exit; + int expect_signal; + int kill_status; + int kill_errno; + + const char *cli_fn; + + char *workdir; +}; + +static void haproxy_cleanup(struct haproxy *h, int is_haproxy_listener); + +static VTAILQ_HEAD(, haproxy) haproxies = + VTAILQ_HEAD_INITIALIZER(haproxies); + +/********************************************************************** + * + */ + +static void +wait_stopped(struct haproxy *h) +{ + vtc_log(h->vl, 3, "wait-stopped"); + while (1) { + if (h->kill_status >= 0 || h->kill_errno == ESRCH) + break; + + usleep(1000); + h->kill_status = kill(h->pid, 0); + h->kill_errno = errno; + } +} + +static void +wait_running(struct haproxy *h) +{ + char buf_err[1024] = {0}; + int usleep_time = 1000; + double t0; + pid_t pid; + + if (h->opt_check_mode) + return; + + if (h->pid_fn == NULL) { + /* + * If we use no pid-file, check that the process + * we started is still running. + */ + if (kill(h->pid, 0) < 0) + vtc_fatal(h->vl, "Process %ld not there: %s", + (long)h->pid, strerror(errno)); + return; + } + + vtc_log(h->vl, 3, "wait-running"); + for (t0 = VTIM_mono(); VTIM_mono() - t0 < 3;) { + if (vtc_error) + return; + + if (VPF_read(h->pid_fn, &pid) != 0) { + bprintf(buf_err, + "Could not read PID file '%s'", h->pid_fn); + usleep(usleep_time); + continue; + } + + if (!h->opt_daemon && pid != h->pid) { + bprintf(buf_err, + "PID file has different PID (%ld != %lld)", + (long)pid, (long long)h->pid); + usleep(usleep_time); + continue; + } + + if (kill(pid, 0) < 0) { + bprintf(buf_err, + "Could not find PID %ld process", (long)pid); + usleep(usleep_time); + continue; + } + + h->pid = pid; + + vtc_log(h->vl, 2, "haproxy PID %ld successfully started", + (long)pid); + return; + } + vtc_fatal(h->vl, "haproxy %s PID file check failed:\n\t%s\n", + h->name, buf_err); +} + +/********************************************************************** + * Allocate and initialize a haproxy + */ + +static struct haproxy * +haproxy_new(const char *name) +{ + struct haproxy *h; + struct vsb *vsb; + char buf[PATH_MAX]; + + ALLOC_OBJ(h, HAPROXY_MAGIC); + AN(h); + REPLACE(h->name, name); + + h->args = VSB_new_auto(); + + h->vl = vtc_logopen(name); + AN(h->vl); + + h->filename = getenv(HAPROXY_PROGRAM_ENV_VAR); + if (h->filename == NULL) + REPLACE(h->filename, "haproxy"); + + bprintf(buf, "${tmpdir}/%s", name); + vsb = macro_expand(h->vl, buf); + AN(vsb); + h->workdir = strdup(VSB_data(vsb)); + AN(h->workdir); + VSB_destroy(&vsb); + + bprintf(buf, "%s/stats.sock", h->workdir); + h->cli_fn = strdup(buf); + AN(h->cli_fn); + + bprintf(buf, "%s/cfg", h->workdir); + h->cfg_fn = strdup(buf); + AN(h->cfg_fn); + + VSB_printf(h->args, "-f %s ", h->cfg_fn); + + bprintf(buf, "rm -rf %s ; mkdir -p %s", h->workdir, h->workdir); + AZ(system(buf)); + + VTAILQ_INSERT_TAIL(&haproxies, h, list); + + return (h); +} + +/********************************************************************** + * Delete a haproxy instance + */ + +static void +haproxy_delete(struct haproxy *h) +{ + char buf[PATH_MAX]; + + CHECK_OBJ_NOTNULL(h, HAPROXY_MAGIC); + vtc_logclose(h->vl); + + if (!leave_temp) { + bprintf(buf, "rm -rf %s", h->workdir); + AZ(system(buf)); + } + + free(h->name); + free(h->workdir); + VSB_destroy(&h->args); + + /* XXX: MEMLEAK (?) */ + FREE_OBJ(h); +} + +/********************************************************************** + * HAProxy listener + */ + +static void * +haproxy_thread(void *priv) +{ + struct haproxy *h; + + CAST_OBJ_NOTNULL(h, priv, HAPROXY_MAGIC); + return (vtc_record(h->vl, h->fds[0])); +} + +/********************************************************************** + * Start a HAProxy instance. + */ + +static void +haproxy_start(struct haproxy *h) +{ + char **argv; + int i; + char buf[PATH_MAX]; + struct vsb *vsb; + + vtc_log(h->vl, 2, "%s", __func__); + + AZ(VSB_finish(h->args)); + argv = VAV_Parse(VSB_data(h->args), NULL, 0); + AN(argv); + if (argv[0] != NULL) + vtc_fatal(h->vl, "Could not parse argv-string: %s", argv[0]); + for (i = 1; argv[i] != NULL; i++) { + /* Check if HAPROXY_OPT_WORKER option was provided. */ + if (!strcmp(argv[i], HAPROXY_OPT_WORKER)) { + h->opt_worker = 1; + } else if (!strcmp(argv[i], HAPROXY_OPT_DAEMON)) { + h->opt_daemon = 1; + } else if (!strcmp(argv[i], HAPROXY_OPT_CHECK_MODE)) { + h->opt_check_mode = 1; + } + } + VAV_Free(argv); + vtc_log(h->vl, 4, "opt_worker %d opt_daemon %d opt_check_mode %d", + h->opt_worker, h->opt_daemon, h->opt_check_mode); + + vsb = VSB_new_auto(); + AN(vsb); + VSB_printf(vsb, "%s %s", h->filename, VSB_data(h->args)); + + if (h->opt_worker || h->opt_daemon) { + bprintf(buf, "%s/pid", h->workdir); + h->pid_fn = strdup(buf); + AN(h->pid_fn); + VSB_printf(vsb, " -p %s", h->pid_fn); + } + + AZ(VSB_finish(vsb)); + vtc_dump(h->vl, 4, "argv", VSB_data(vsb), -1); + + if (h->opt_worker && !h->opt_daemon) { + /* + * HAProxy master process must exit with status 128 + + * if signaled by signal. + */ + h->expect_exit = HAPROXY_EXPECT_EXIT; + } else { + h->expect_exit = 0; + } + + AZ(pipe(&h->fds[0])); + AZ(pipe(&h->fds[2])); + h->pid = h->ppid = fork(); + assert(h->pid >= 0); + if (h->pid == 0) { + AZ(chdir(h->name)); + AZ(dup2(h->fds[0], 0)); + assert(dup2(h->fds[3], 1) == 1); + assert(dup2(1, 2) == 2); + closefd(&h->fds[0]); + closefd(&h->fds[1]); + closefd(&h->fds[2]); + closefd(&h->fds[3]); + AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), NULL)); + exit(1); + } + VSB_destroy(&vsb); + + vtc_log(h->vl, 3, "PID: %ld", (long)h->pid); + macro_def(h->vl, h->name, "pid", "%ld", (long)h->pid); + macro_def(h->vl, h->name, "name", "%s", h->workdir); + + closefd(&h->fds[0]); + closefd(&h->fds[3]); + h->fds[0] = h->fds[2]; + h->fds[2] = h->fds[3] = -1; + + AZ(pthread_create(&h->tp, NULL, haproxy_thread, h)); + + wait_running(h); +} + +/********************************************************************** + * Stop a HAProxy + */ + +static void +haproxy_stop(struct haproxy *h) +{ + if (h->opt_check_mode) + return; + + if (h->pid < 0) + haproxy_start(h); + + vtc_log(h->vl, 2, "Stop"); + h->kill_status = kill(h->pid, HAPROXY_SIGNAL); + h->kill_errno = errno; + h->expect_signal = -HAPROXY_SIGNAL; + + wait_stopped(h); +} + +/********************************************************************** + * Cleanup + */ + +static void +haproxy_cleanup(struct haproxy *h, int is_haproxy_listener) +{ + void *p; + + vtc_log(h->vl, 2, "%s (%d)", __func__, is_haproxy_listener); + if (!is_haproxy_listener) { + /* Close the STDIN connection. */ + closefd(&h->fds[1]); + + /* Wait until STDOUT+STDERR closes */ + AZ(pthread_join(h->tp, &p)); + closefd(&h->fds[0]); + if (h->opt_daemon) + return; + } + + if (h->ppid == -1) + return; + + vtc_wait4(h->vl, h->ppid, h->expect_exit, h->expect_signal, 0); + h->ppid = -1; +} + +/********************************************************************** + * Wait for a HAProxy instance. + */ + +static void +haproxy_wait(struct haproxy *h) +{ + if (h->pid < 0) + return; + + vtc_log(h->vl, 2, "Wait"); + + haproxy_stop(h); + + haproxy_cleanup(h, 0); +} + +#define HAPROXY_BE_FD_STR "fd@${" +#define HAPROXY_BE_FD_STRLEN strlen(HAPROXY_BE_FD_STR) + +static int +haproxy_build_backends(const struct haproxy *h, const char *vsb_data) +{ + char *s, *p, *q; + + s = strdup(vsb_data); + if (!s) + return -1; + + p = s; + while (1) { + int sock; + char buf[128], addr[128], port[128]; + const char *err; + + p = strstr(p, HAPROXY_BE_FD_STR); + if (!p) + break; + + q = p += HAPROXY_BE_FD_STRLEN; + while (*q && *q != '}') + q++; + if (*q != '}') + break; + + *q++ = '\0'; + sock = VTCP_listen_on(":0", NULL, 1, &err); + if (err != NULL) + vtc_fatal(h->vl, + "Create listen socket failed: %s", err); + + VTCP_myname(sock, addr, sizeof addr, port, sizeof port); + bprintf(buf, "%s_%s", h->name, p); + macro_def(h->vl, buf, "sock", "%s %s", addr, port); + macro_def(h->vl, buf, "addr", "%s", addr); + macro_def(h->vl, buf, "port", "%s", port); + + bprintf(buf, "%d", sock); + vtc_log(h->vl, 4, "setenv(%s, %s)", p, buf); + if (setenv(p, buf, 0) == -1) + vtc_fatal(h->vl, "setenv() failed: %s (%d)", + strerror(errno), errno); + p = q; + } + free(s); + return (0); +} + +/********************************************************************** + * Write a configuration for HAProxy instance. + */ + +static void +haproxy_write_conf(const struct haproxy *h, const char *cfg) +{ + struct vsb *vsb, *vsb2; + + vsb = VSB_new_auto(); + AN(vsb); + + vsb2 = VSB_new_auto(); + AN(vsb2); + + VSB_printf(vsb, " global\n\tstats socket %s " + "level admin mode 600\n", h->cli_fn); + AZ(VSB_cat(vsb, cfg)); + AZ(VSB_finish(vsb)); + + AZ(haproxy_build_backends(h, VSB_data(vsb))); + + if (VFIL_writefile(h->workdir, h->cfg_fn, + VSB_data(vsb), VSB_len(vsb)) != 0) + vtc_fatal(h->vl, + "failed to write haproxy configuration file: %s (%d)", + strerror(errno), errno); + + vtc_dump(h->vl, 4, "conf", VSB_data(vsb), VSB_len(vsb)); + + VSB_destroy(&vsb2); + VSB_destroy(&vsb); +} + +/* SECTION: haproxy haproxy + * + * Define and interact with haproxy instances. + * + * To define a haproxy server, you'll use this syntax:: + * + * haproxy hNAME [-arg STRING] [-conf STRING] + * + * The first ``haproxy hNAME`` invocation will start the haproxy master + * process in the background, waiting for the ``-start`` switch to actually + * start the child. + * + * Arguments: + * + * hNAME + * Identify the HAProxy server with a string, it must starts with 'h'. + * + * \-arg STRING + * Pass an argument to haproxy, for example "-h simple_list". + * + * \-conf STRING + * Specify the configuration to be loaded by this HAProxy instance. + * + * You can decide to start the HAProxy instance and/or wait for several events:: + * + * haproxy hNAME [-start] [-wait-running] [-wait-stopped] + * + * \-start + * Start this HAProxy instance. + * + * \-stop + * Stop this HAProxy instance. + * + * \-wait-running + * Wait for that instance to terminate. + * + * \-wait-stopped + * Wait for that instance to terminate. + * + * \-cleanup + * Once HAProxy is stopped, clean everything after it. This is only used + * in very few tests and you should never need it. + * + * \-expectexit NUMBER + * Expect haproxy to exit(3) with this value + * + */ + +void +cmd_haproxy(CMD_ARGS) +{ + struct haproxy *h, *h2; + + (void)priv; + (void)cmd; + + if (av == NULL) { + /* Reset and free */ + VTAILQ_FOREACH_SAFE(h, &haproxies, list, h2) { + vtc_log(h->vl, 2, + "Reset and free %s haproxy %d", h->name, h->pid); + if (h->pid >= 0) + haproxy_wait(h); + VTAILQ_REMOVE(&haproxies, h, list); + haproxy_delete(h); + } + return; + } + + AZ(strcmp(av[0], "haproxy")); + av++; + + VTC_CHECK_NAME(vl, av[0], "haproxy", 'h'); + VTAILQ_FOREACH(h, &haproxies, list) + if (!strcmp(h->name, av[0])) + break; + if (h == NULL) + h = haproxy_new(av[0]); + av++; + + for (; *av != NULL; av++) { + if (vtc_error) + break; + if (!strcmp(*av, "-arg")) { + AN(av[1]); + AZ(h->pid); + VSB_cat(h->args, " "); + VSB_cat(h->args, av[1]); + av++; + continue; + } + if (!strcmp(*av, "-cleanup")) { + AZ(av[1]); + haproxy_cleanup(h, 0); + continue; + } + if (!strcmp(*av, "-conf")) { + AN(av[1]); + haproxy_write_conf(h, av[1]); + av++; + continue; + } + if (!strcmp(*av, "-expectexit")) { + h->expect_exit = strtoul(av[1], NULL, 0); + av++; + continue; + } + if (!strcmp(*av, "-start")) { + haproxy_start(h); + continue; + } + if (!strcmp(*av, "-stop")) { + haproxy_stop(h); + continue; + } + if (!strcmp(*av, "-wait-stopped")) { + wait_stopped(h); + continue; + } + if (!strcmp(*av, "-wait-running")) { + wait_running(h); + continue; + } + vtc_fatal(h->vl, "Unknown haproxy argument: %s", *av); + } +} diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 39cafbe..e232f95 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -160,14 +160,15 @@ vtc_wait4(struct vtclog *vl, long pid, if (r < 0) vtc_fatal(vl, "wait4 failed on pid %ld: %s", pid, strerror(errno)); - vtc_log(vl, 2, "WAIT4 pid=%ld r=%d status=0x%04x (user %.6f sys %.6f)", - pid, r, status, + assert(r == pid); + vtc_log(vl, 2, "WAIT4 pid=%ld status=0x%04x (user %.6f sys %.6f)", + pid, status, ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec, ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec ); if (WIFEXITED(status) && expect_signal <= 0 && - (WEXITSTATUS(status) == expect_status)) + WEXITSTATUS(status) == expect_status) return; if (expect_signal < 0) From phk at FreeBSD.org Tue Mar 27 20:11:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 20:11:07 +0000 (UTC) Subject: [master] 5722377 Use (char*)0 because because GCC thinks NULL is something else ? Message-ID: <20180327201107.2CFE196A9E@lists.varnish-cache.org> commit 57223773d9237f3e44c3034bb2cac8c5c6a2b712 Author: Poul-Henning Kamp Date: Tue Mar 27 20:09:44 2018 +0000 Use (char*)0 because because GCC thinks NULL is something else ? diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 17c62fb..4f9ab5a 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -318,7 +318,7 @@ haproxy_start(struct haproxy *h) closefd(&h->fds[1]); closefd(&h->fds[2]); closefd(&h->fds[3]); - AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), NULL)); + AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), (char*)0)); exit(1); } VSB_destroy(&vsb); From phk at FreeBSD.org Tue Mar 27 20:20:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Mar 2018 20:20:09 +0000 (UTC) Subject: [master] ad5d4de Printf pid using cast and %ld Message-ID: <20180327202009.5933A96DAA@lists.varnish-cache.org> commit ad5d4de5efa3be2344c21b43aea64e80251da4ec Author: Poul-Henning Kamp Date: Tue Mar 27 20:18:47 2018 +0000 Printf pid using cast and %ld diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 4f9ab5a..d97ffb4 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -548,7 +548,8 @@ cmd_haproxy(CMD_ARGS) /* Reset and free */ VTAILQ_FOREACH_SAFE(h, &haproxies, list, h2) { vtc_log(h->vl, 2, - "Reset and free %s haproxy %d", h->name, h->pid); + "Reset and free %s haproxy %ld", + h->name, (long)h->pid); if (h->pid >= 0) haproxy_wait(h); VTAILQ_REMOVE(&haproxies, h, list); From phk at FreeBSD.org Wed Mar 28 14:24:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Mar 2018 14:24:16 +0000 (UTC) Subject: [master] 4438bf4 Allow a single '\n' after the PID Message-ID: <20180328142416.8C263B3269@lists.varnish-cache.org> commit 4438bf419db814842534247c8aaa0284592f865a Author: Poul-Henning Kamp Date: Wed Mar 28 14:15:33 2018 +0000 Allow a single '\n' after the PID diff --git a/lib/libvarnish/vpf.c b/lib/libvarnish/vpf.c index 915a028..cf326fb 100644 --- a/lib/libvarnish/vpf.c +++ b/lib/libvarnish/vpf.c @@ -89,6 +89,8 @@ VPF_read(const char *path, pid_t *pidptr) return (error); else if (i == 0) return (EAGAIN); + if (i > 0 && buf[i - 1] == '\n') + i--; buf[i] = '\0'; *pidptr = strtol(buf, &endptr, 10); From phk at FreeBSD.org Wed Mar 28 14:24:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Mar 2018 14:24:16 +0000 (UTC) Subject: [master] a056f99 Migrate the shell PID to haproxy where we want it. Message-ID: <20180328142416.A448EB326C@lists.varnish-cache.org> commit a056f9981e23d16f17e3149dd2c0fa23b29ba0ae Author: Poul-Henning Kamp Date: Wed Mar 28 14:16:00 2018 +0000 Migrate the shell PID to haproxy where we want it. diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index d97ffb4..e3ea8d2 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -283,7 +283,7 @@ haproxy_start(struct haproxy *h) vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "%s %s", h->filename, VSB_data(h->args)); + VSB_printf(vsb, "exec %s %s", h->filename, VSB_data(h->args)); if (h->opt_worker || h->opt_daemon) { bprintf(buf, "%s/pid", h->workdir); From phk at FreeBSD.org Wed Mar 28 14:24:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Mar 2018 14:24:16 +0000 (UTC) Subject: [master] 082e6bc Add a test with HAproxy in daemonmode (if available) Message-ID: <20180328142416.C2CC1B326F@lists.varnish-cache.org> commit 082e6bc51f2cefaa557e32f312727afe1bdab909 Author: Poul-Henning Kamp Date: Wed Mar 28 14:16:43 2018 +0000 Add a test with HAproxy in daemonmode (if available) diff --git a/bin/varnishtest/tests/h00002.vtc b/bin/varnishtest/tests/h00002.vtc new file mode 100644 index 0000000..6e2af66 --- /dev/null +++ b/bin/varnishtest/tests/h00002.vtc @@ -0,0 +1,32 @@ +varnishtest "Basic HAproxy test (daemon mode)" + +feature ignore_unknown_macro + +feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'} + +server s1 { + rxreq + txresp -body "s1 >>> Hello world!" +} -start + +haproxy h1 -arg "-D" -conf { + defaults + mode http + timeout connect 5s + timeout server 30s + timeout client 30s + + backend be1 + server srv1 ${s1_addr}:${s1_port} + + frontend http1 + use_backend be1 + bind "fd@${fe1}" +} -start + +client c1 -connect ${h1_fe1_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.body == "s1 >>> Hello world!" +} -run From phk at FreeBSD.org Wed Mar 28 15:22:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Mar 2018 15:22:08 +0000 (UTC) Subject: [master] 296814e Don't unlock the mutex unless we locked it. Message-ID: <20180328152208.0E085B4677@lists.varnish-cache.org> commit 296814e74b9862213a0f884b9366bbc2d12490e9 Author: Poul-Henning Kamp Date: Wed Mar 28 15:21:09 2018 +0000 Don't unlock the mutex unless we locked it. It is only during cleanup, so this *may* be an adequate bandaid. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index f7f68af..035e6b2 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -422,7 +422,8 @@ cmd_barrier(CMD_ARGS) default: WRONG("Wrong barrier type"); } - AZ(pthread_mutex_unlock(&b->mtx)); + if (r == 0) + AZ(pthread_mutex_unlock(&b->mtx)); } return; } From phk at FreeBSD.org Wed Mar 28 21:30:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Mar 2018 21:30:16 +0000 (UTC) Subject: [master] dadc852 We don't want distcheck to exercise persistent by default, but we do want VTEST to do so. Message-ID: <20180328213016.79A9C63BA9@lists.varnish-cache.org> commit dadc85271fe77deaf691cc46dcc95e15c2d5d419 Author: Poul-Henning Kamp Date: Wed Mar 28 21:25:07 2018 +0000 We don't want distcheck to exercise persistent by default, but we do want VTEST to do so. diff --git a/Makefile.am b/Makefile.am index 006c770..f019082 100644 --- a/Makefile.am +++ b/Makefile.am @@ -18,7 +18,7 @@ EXTRA_DIST = \ varnish.m4 \ varnish-legacy.m4 -DISTCHECK_CONFIGURE_FLAGS = \ +AM_DISTCHECK_CONFIGURE_FLAGS = \ --enable-developer-warnings \ --enable-debugging-symbols \ --enable-dependency-tracking \ diff --git a/tools/vtest.sh b/tools/vtest.sh index b2e121d..9202039 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -131,7 +131,7 @@ makedistcheck () ( set -e cd "${SRCDIR}" nice make vtest-clean - nice make distcheck + nice make distcheck DISTCHECK_CONFIGURE_FLAGS=--with-persistent-storage ) gcovtest () ( From fgsch at lodoss.net Thu Mar 29 04:27:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 29 Mar 2018 04:27:12 +0000 (UTC) Subject: [master] 4edafdf Sync with reality Message-ID: <20180329042712.55BFF9690F@lists.varnish-cache.org> commit 4edafdffe1c2057a95e5a0f74a99291355d5f1df Author: Federico G. Schwindt Date: Tue Mar 27 20:45:24 2018 -0300 Sync with reality diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 55294e8..06b0167 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -342,7 +342,7 @@ cmd_delay(CMD_ARGS) * Do not fail the test if a string of the form ${...} is not * recognized as a macro. * term - * Support for ADM3A terminal + * Support for ansi.sys terminal * * persistent_storage * Varnish was built with the deprecated persistent storage. From fgsch at lodoss.net Thu Mar 29 04:27:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 29 Mar 2018 04:27:12 +0000 (UTC) Subject: [master] 65d0c0c Formatting Message-ID: <20180329042712.68EBD96912@lists.varnish-cache.org> commit 65d0c0c395aaeadfe0f8344ad30caa0086c7e094 Author: Federico G. Schwindt Date: Tue Mar 27 20:48:59 2018 -0300 Formatting diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index a1a625f..cc5f9ae 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -339,16 +339,13 @@ when configuring the shard director, you are advised to check:: $Method VOID .set_warmup(REAL probability=0.0) Set the default warmup probability. See the `warmup` parameter of -``shard.backend()``. - -Default: 0.0 (no warmup) +``shard.backend()``. If probability is 0.0 (default), warmup is +disabled. $Method VOID .set_rampup(DURATION duration=0) Set the default rampup duration. See `rampup` parameter of -`shard.backend()`. - -Default: 0s (no rampup) +`shard.backend()`. If duration is 0 (default), rampup is disabled. $Method VOID .associate(BLOB param=0) @@ -415,13 +412,13 @@ To generate sharding keys using other hashes, use a custom vmod like .. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst $Method BACKEND .backend( - [ ENUM {HASH, URL, KEY, BLOB} by ], + [ ENUM {HASH, URL, KEY, BLOB} by=HASH ], [ INT key ], [ BLOB key_blob ], - [ INT alt ], - [ REAL warmup ], - [ BOOL rampup ], - [ ENUM {CHOSEN, IGNORE, ALL} healthy ], + [ INT alt=0 ], + [ REAL warmup=-1 ], + [ BOOL rampup=1 ], + [ ENUM {CHOSEN, IGNORE, ALL} healthy=CHOSEN ], [ BLOB param ], [ ENUM {NOW, LAZY} resolve] ) @@ -436,8 +433,6 @@ is _not_ the order given when backends are added. * `by` how to determine the sharding key - default: `HASH` - * `HASH`: * when called in backend context: Use the varnish hash value as @@ -451,21 +446,19 @@ is _not_ the order given when backends are added. * `BLOB`: use the `key_blob` argument - * `key` lookup key with `by=KEY` +* `key` lookup key with `by=KEY` - the `shard.key()` function may come handy to generate a sharding - key from custom strings. + the `shard.key()` function may come handy to generate a sharding + key from custom strings. - * `key_blob` lookup key with `by=BLOB` +* `key_blob` lookup key with `by=BLOB` - Currently, this uses the first 4 bytes from the given blob in - network byte order (big endian), left-padded with zeros for blobs - smaller than 4 bytes. + Currently, this uses the first 4 bytes from the given blob in + network byte order (big endian), left-padded with zeros for blobs + smaller than 4 bytes. * `alt` alternative backend selection - default: `0` - Select the `alt`-th alternative backend for the given `key`. This is particularly useful for retries / restarts due to backend @@ -476,8 +469,6 @@ is _not_ the order given when backends are added. * `rampup` slow start for servers which just went healthy - default: `true` - If `alt==0` and the chosen backend is in its rampup period, with a probability proportional to the fraction of time since the backup became healthy to the rampup period, return the next alternative @@ -491,8 +482,6 @@ is _not_ the order given when backends are added. possible values: -1, 0..1 - default: `-1` - `-1`: use the warmup probability from the director definition Only used for `alt==0`: Sets the ratio of requests (0.0 to 1.0) that @@ -505,8 +494,6 @@ is _not_ the order given when backends are added. * `healthy` - default: `CHOSEN` - * CHOSEN: Return a healthy backend if possible. For `alt==0`, return the first healthy backend or none. From fgsch at lodoss.net Thu Mar 29 04:27:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 29 Mar 2018 04:27:12 +0000 (UTC) Subject: [master] 074d66a Tighten up tests Message-ID: <20180329042712.914BA9691A@lists.varnish-cache.org> commit 074d66ad49f7fa3b8fda6a23e9d1ad734842be5a Author: Federico G. Schwindt Date: Wed Mar 28 17:09:55 2018 -0300 Tighten up tests Prep work for upcoming change. diff --git a/bin/varnishtest/tests/b00012.vtc b/bin/varnishtest/tests/b00012.vtc index 224bd1a..6f81802 100644 --- a/bin/varnishtest/tests/b00012.vtc +++ b/bin/varnishtest/tests/b00012.vtc @@ -12,7 +12,7 @@ server s1 { varnish v1 -vcl+backend {} -start client c1 { - send "GET /foo HTTP/1.1\n\nGET /bar HTTP/1.1\n\nGET /bar HTTP/1.1\n\n" + send "GET /foo HTTP/1.1\nHost: foo\n\nGET /bar HTTP/1.1\nHost: foo\n\nGET /bar HTTP/1.1\nHost: foo\n\n" rxresp expect resp.status == 200 expect resp.bodylen == 3 diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc index ffae946..46f5641 100644 --- a/bin/varnishtest/tests/b00013.vtc +++ b/bin/varnishtest/tests/b00013.vtc @@ -16,17 +16,17 @@ varnish v1 -cli "param.set accept_filter false" varnish v1 -start client c1 { - send "GET /foo HTTP/1.1\r\n\r\nGET " + send "GET /foo HTTP/1.1\r\nHost: foo\r\n\r\nGET " rxresp expect resp.status == 200 expect resp.bodylen == 3 expect resp.http.x-varnish == "1001" - send "/bar HTTP/1.1\n\nGET /bar " + send "/bar HTTP/1.1\nHost: foo\n\nGET /bar " rxresp expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1003" - send "HTTP/1.1\n\n" + send "HTTP/1.1\nHost: foo\n\n" rxresp expect resp.status == 200 expect resp.bodylen == 6 diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc index 228aaa9..8d5db15 100644 --- a/bin/varnishtest/tests/b00040.vtc +++ b/bin/varnishtest/tests/b00040.vtc @@ -46,28 +46,28 @@ delay .1 client c1 { send "GET /4 HTTP/1.1\r\n" - send "\r\n" + send "Host: foo\r\n\r\n" rxresp expect resp.status == 200 } -run delay .1 client c1 { - send "GET /5 HTTP/1.1\r\nHost: localhost\r\nBogo: Header\001More\r\n\r\n" + send "GET /5 HTTP/1.1\r\nHost: foo\r\nBogo: Header\001More\r\n\r\n" rxresp expect resp.status == 400 } -run delay .1 client c1 { - send "GET /6 HTTP/1.1\r\nHost: localhost\r\nBogo: Header\r\r\n\r\n" + send "GET /6 HTTP/1.1\r\nHost: foo\r\nBogo: Header\r\r\n\r\n" rxresp expect resp.status == 400 } -run delay .1 client c1 { - send "GET /7 HTTP/1.1\r\nHost: localhost\r\nBogo: Header\rMore\r\n\r\n" + send "GET /7 HTTP/1.1\r\nHost: foo\r\nBogo: Header\rMore\r\n\r\n" rxresp expect resp.status == 400 } -run diff --git a/bin/varnishtest/tests/b00055.vtc b/bin/varnishtest/tests/b00055.vtc index bb3a36d..076eb1b 100644 --- a/bin/varnishtest/tests/b00055.vtc +++ b/bin/varnishtest/tests/b00055.vtc @@ -12,7 +12,7 @@ server s1 { varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {} -start client c1 -connect "${tmpdir}/v1.sock" { - send "GET /foo HTTP/1.1\n\nGET /bar HTTP/1.1\n\nGET /bar HTTP/1.1\n\n" + send "GET /foo HTTP/1.1\nHost: foo\n\nGET /bar HTTP/1.1\nHost: foo\n\nGET /bar HTTP/1.1\nHost: foo\n\n" rxresp expect resp.status == 200 expect resp.bodylen == 3 diff --git a/bin/varnishtest/tests/b00056.vtc b/bin/varnishtest/tests/b00056.vtc index b875f0b..4da3c7c 100644 --- a/bin/varnishtest/tests/b00056.vtc +++ b/bin/varnishtest/tests/b00056.vtc @@ -16,17 +16,17 @@ varnish v1 -cli "param.set accept_filter false" varnish v1 -start client c1 -connect "${tmpdir}/v1.sock" { - send "GET /foo HTTP/1.1\r\n\r\nGET " + send "GET /foo HTTP/1.1\r\nHost: foo\r\n\r\nGET " rxresp expect resp.status == 200 expect resp.bodylen == 3 expect resp.http.x-varnish == "1001" - send "/bar HTTP/1.1\n\nGET /bar " + send "/bar HTTP/1.1\nHost: foo\n\nGET /bar " rxresp expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1003" - send "HTTP/1.1\n\n" + send "HTTP/1.1\nHost: foo\n\n" rxresp expect resp.status == 200 expect resp.bodylen == 6 diff --git a/bin/varnishtest/tests/c00018.vtc b/bin/varnishtest/tests/c00018.vtc index 96368b9..7a27755 100644 --- a/bin/varnishtest/tests/c00018.vtc +++ b/bin/varnishtest/tests/c00018.vtc @@ -55,9 +55,8 @@ logexpect l1 -v v1 -g raw { expect 0 1003 ReqMethod {^POST$} expect 0 1003 ReqURL {^/$} expect 0 1003 ReqProtocol {^HTTP/1.1$} - expect 0 1003 ReqHeader {^Content-Length: 20$} - expect 0 1003 ReqHeader {^X-Forwarded-For:} - expect 0 1003 VCL_call {^RECV$} + expect * 1003 ReqHeader {^Content-Length: 20$} + expect * 1003 VCL_call {^RECV$} expect 0 1003 VCL_return {^pass$} expect 0 1003 RespProtocol {^HTTP/1.1$} expect 0 1003 RespStatus {^100$} @@ -70,9 +69,8 @@ logexpect l1 -v v1 -g raw { expect 0 1005 ReqMethod {^POST$} expect 0 1005 ReqURL {^/$} expect 0 1005 ReqProtocol {^HTTP/1.1$} - expect 0 1005 ReqHeader {^Content-Length: 3$} - expect 0 1005 ReqHeader {^X-Forwarded-For:} - expect 0 1005 VCL_call {^RECV$} + expect * 1005 ReqHeader {^Content-Length: 3$} + expect * 1005 VCL_call {^RECV$} expect 0 1005 VCL_return {^pass$} expect 0 1005 VCL_call {^HASH$} @@ -82,9 +80,8 @@ logexpect l1 -v v1 -g raw { expect 0 1007 ReqMethod {^POST$} expect 0 1007 ReqURL {^/late$} expect 0 1007 ReqProtocol {^HTTP/1.1$} - expect 0 1007 ReqHeader {^Content-Length: 20$} - expect 0 1007 ReqHeader {^X-Forwarded-For:} - expect 0 1007 VCL_call {^RECV$} + expect * 1007 ReqHeader {^Content-Length: 20$} + expect * 1007 VCL_call {^RECV$} expect 0 1007 VCL_return {^pass$} expect 0 1007 VCL_call {^HASH$} expect 0 1007 VCL_return {^lookup$} @@ -104,9 +101,8 @@ logexpect l1 -v v1 -g raw { expect 0 1009 ReqMethod {^POST$} expect 0 1009 ReqURL {^/latecache$} expect 0 1009 ReqProtocol {^HTTP/1.1$} - expect 0 1009 ReqHeader {^Content-Length: 20$} - expect 0 1009 ReqHeader {^X-Forwarded-For:} - expect 0 1009 VCL_call {^RECV$} + expect * 1009 ReqHeader {^Content-Length: 20$} + expect * 1009 VCL_call {^RECV$} expect 0 1009 Storage {^(malloc|umem) Transient$} expect 0 1009 RespProtocol {^HTTP/1.1$} expect 0 1009 RespStatus {^100$} @@ -121,9 +117,8 @@ logexpect l1 -v v1 -g raw { expect 0 1011 ReqMethod {^POST$} expect 0 1011 ReqURL {^/err$} expect 0 1011 ReqProtocol {^HTTP/1.1$} - expect 0 1011 ReqHeader {^Content-Length: 20$} - expect 0 1011 ReqHeader {^X-Forwarded-For:} - expect 0 1011 VCL_call {^RECV$} + expect * 1011 ReqHeader {^Content-Length: 20$} + expect * 1011 VCL_call {^RECV$} expect 0 1011 VCL_return {^synth$} expect 0 1011 VCL_call {^HASH$} expect 0 1011 VCL_return {^lookup$} diff --git a/bin/varnishtest/tests/c00039.vtc b/bin/varnishtest/tests/c00039.vtc index 370234f..4c36c2a 100644 --- a/bin/varnishtest/tests/c00039.vtc +++ b/bin/varnishtest/tests/c00039.vtc @@ -40,7 +40,7 @@ client c1 { client c1 { # Each line is 32 bytes. Total: 32 * 8 == 256 - send "GET /....0....5....0. HTTP/1.1\r\n" + send "GET /..... HTTP/1.1\r\nHost: foo\r\n" send "1...5: ..0....5....0....5....0\r\n" send "1...5: ..0....5....0....5....0\r\n" send "1...5: ..0....5....0....5....0\r\n" @@ -52,8 +52,7 @@ client c1 { expect resp.status == 200 # Each line is 32 except last, which is 33. Total: 32 * 7 + 33 == 257 - send "GET /....0....5....0. HTTP/1.1\r\n" - send "1...5: ..0....5....0....5....0\r\n" + send "GET /..... HTTP/1.1\r\nHost: foo\r\n" send "1...5: ..0....5....0....5....0\r\n" send "1...5: ..0....5....0....5....0\r\n" send "1...5: ..0....5....0....5....0\r\n" diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index 71c5c63..a541596 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -33,7 +33,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" # ReqAcct body counts include chunked overhead - expect * = ReqAcct "^18 0 18 187 104 291$" + expect * = ReqAcct "^29 0 29 187 104 291$" expect 0 = End } -start @@ -57,19 +57,19 @@ logexpect l5 -v v1 -g request { expect * 1005 Begin "^req .* rxreq" # ReqAcct body counts include chunked overhead # Header bytes is 5 larger than in l1 due to two item X-Varnish hdr - expect * = ReqAcct "^18 0 18 192 104 296$" + expect * = ReqAcct "^29 0 29 192 104 296$" expect 0 = End } -start client c1 { - txreq + txreq -hdr "Host: foo" rxresp expect resp.bodylen == 75 expect resp.status == 200 delay .1 # test that there is no difference on miss/hit - txreq + txreq -hdr "Host: foo" rxresp expect resp.bodylen == 75 expect resp.status == 200 diff --git a/bin/varnishtest/tests/f00001.vtc b/bin/varnishtest/tests/f00001.vtc index dc9fd9b..2c756c9 100644 --- a/bin/varnishtest/tests/f00001.vtc +++ b/bin/varnishtest/tests/f00001.vtc @@ -12,6 +12,7 @@ varnish v1 -vcl+backend { client c1 { send "POST / HTTP/1.1\r\n" + send "Host: foo\r\n" send "Transfer-Encoding: chunked\r\n\r\n" send "FFFFFFFFFFFFFFED\r\n" send "0\r\n\r\n" @@ -32,6 +33,7 @@ varnish v1 -vcl+backend { client c1 { send "POST / HTTP/1.1\r\n" + send "Host: foo\r\n" send "Transfer-Encoding: chunked\r\n\r\n" send "FFFFFFFFFFFFFFED\r\n" diff --git a/bin/varnishtest/tests/l00002.vtc b/bin/varnishtest/tests/l00002.vtc index 942c8b5..82296bc 100644 --- a/bin/varnishtest/tests/l00002.vtc +++ b/bin/varnishtest/tests/l00002.vtc @@ -24,9 +24,10 @@ varnish v1 -vcl+backend { # Request (1001): # GET /1 HTTP/1.1\r\n 17 bytes +# Host: foo\r\n 11 bytes # Content-Length: 4\r\n 19 bytes # \r\n 2 bytes -# Total: 38 bytes +# Total: 49 bytes # Response: # HTTP/1.1 200 OK\r\n 17 bytes # Content-Length: 1000\r\n 22 bytes @@ -37,8 +38,9 @@ varnish v1 -vcl+backend { # Request (1003): # GET /2 HTTP/1.1\r\n 17 bytes +# Host: foo\r\n 11 bytes # \r\n 2 bytes -# Total: 19 bytes +# Total: 30 bytes # Response: # HTTP/1.1 200 OK\r\n 17 bytes # Content-Length: 2000\r\n 22 bytes @@ -49,8 +51,9 @@ varnish v1 -vcl+backend { # Request (1005): # GET /2 HTTP/1.1\r\n 17 bytes +# Host: foo\r\n 11 bytes # \r\n 2 bytes -# Total: 19 bytes +# Total: 30 bytes # Response: # HTTP/1.1 200 OK\r\n 17 bytes # Content-Length: 2000\r\n 22 bytes @@ -69,13 +72,13 @@ varnish v1 -vcl+backend { # Total: 28 bytes logexpect l1 -v v1 -g session { expect * 1001 Begin "^req .* rxreq" - expect * = ReqAcct "^38 4 42 87 1000 1087$" + expect * = ReqAcct "^49 4 53 87 1000 1087$" expect 0 = End expect * 1003 Begin "^req .* rxreq" - expect * = ReqAcct "^19 0 19 87 2000 2087$" + expect * = ReqAcct "^30 0 30 87 2000 2087$" expect 0 = End expect * 1005 Begin "^req .* rxreq" - expect * = ReqAcct "^19 0 19 87 2000 2087$" + expect * = ReqAcct "^30 0 30 87 2000 2087$" expect 0 = End expect * 1006 Begin "^req .* rxreq" expect * = ReqAcct "^7 0 7 28 0 28$" @@ -84,11 +87,11 @@ logexpect l1 -v v1 -g session { # Request 1001 client c1 { - txreq -url "/1" -body "asdf" + txreq -url "/1" -hdr "Host: foo" -body "asdf" rxresp expect resp.status == 200 - send "GET /2 HTTP/1.1\r\n\r\nGET /2 HTTP/1.1\r\n\r\n" + send "GET /2 HTTP/1.1\r\nHost: foo\r\n\r\nGET /2 HTTP/1.1\r\nHost: foo\r\n\r\n" rxresp expect resp.status == 200 rxresp @@ -101,7 +104,7 @@ client c1 { logexpect l1 -wait -varnish v1 -expect s_req_hdrbytes == 83 +varnish v1 -expect s_req_hdrbytes == 116 varnish v1 -expect s_req_bodybytes == 4 varnish v1 -expect s_resp_hdrbytes == 289 varnish v1 -expect s_resp_bodybytes == 5000 diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc index 6ef90a3..bdc2333 100644 --- a/bin/varnishtest/tests/l00003.vtc +++ b/bin/varnishtest/tests/l00003.vtc @@ -30,8 +30,9 @@ varnish v1 -vcl+backend { # Request (1001): # GET / HTTP/1.1\r\n 16 bytes +# Host: foo\r\n 11 bytes # \r\n 2 bytes -# Total: 18 bytes +# Total: 29 bytes # Reponse: # HTTP/1.1 200 OK\r\n 17 bytes @@ -62,7 +63,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - expect * = ReqAcct "^18 0 18 93 55 148$" + expect * = ReqAcct "^29 0 29 93 55 148$" expect 0 = End expect * 1003 Begin "^req .* esi" expect * = ReqAcct "^0 0 0 0 12 12$" @@ -76,7 +77,7 @@ logexpect l1 -v v1 -g request { } -start client c1 { - txreq -url "/" + txreq -url "/" -hdr "Host: foo" rxresp expect resp.status == 200 expect resp.body == "123abc123defghi" @@ -84,7 +85,7 @@ client c1 { logexpect l1 -wait -varnish v1 -expect s_req_hdrbytes == 18 +varnish v1 -expect s_req_hdrbytes == 29 varnish v1 -expect s_req_bodybytes == 0 varnish v1 -expect s_resp_hdrbytes == 93 varnish v1 -expect s_resp_bodybytes == 55 diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc index 22e5ea3..95ad361 100644 --- a/bin/varnishtest/tests/l00004.vtc +++ b/bin/varnishtest/tests/l00004.vtc @@ -22,9 +22,10 @@ varnish v1 -vcl+backend { # req: # POST / HTTP/1.1\r\n 17 bytes +# Host: foo\r\n 11 bytes # Content-Length: 4\r\n 19 bytes # \r\n 2 bytes -# Total: 38 bytes +# Total: 49 bytes # bereq: # POST / HTTP/1.1\r\n 17 bytes @@ -45,12 +46,12 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - expect * = PipeAcct "^38 49 4 42$" + expect * = PipeAcct "^49 60 4 42$" expect 0 = End } -start client c1 { - txreq -req "POST" -url "/" -hdr "Content-Length: 4" + txreq -req "POST" -url "/" -hdr "Content-Length: 4" -hdr "Host: foo" send "asdf" rxresp expect resp.status == 200 @@ -58,6 +59,6 @@ client c1 { logexpect l1 -wait -varnish v1 -expect s_pipe_hdrbytes == 38 +varnish v1 -expect s_pipe_hdrbytes == 49 varnish v1 -expect s_pipe_in == 4 varnish v1 -expect s_pipe_out == 42 diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc index 3bb7cd5..1e5dcab 100644 --- a/bin/varnishtest/tests/o00000.vtc +++ b/bin/varnishtest/tests/o00000.vtc @@ -199,7 +199,7 @@ delay .1 # Try with appended request (See also: #1728) client c2 { - send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHdr1: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n\r\n" + send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHost: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n\r\n" rxresp expect resp.http.url == "/3" } -run diff --git a/bin/varnishtest/tests/o00001.vtc b/bin/varnishtest/tests/o00001.vtc index ae6e388..ef78b2d 100644 --- a/bin/varnishtest/tests/o00001.vtc +++ b/bin/varnishtest/tests/o00001.vtc @@ -1,17 +1,40 @@ varnishtest "PROXY v2 test" server s1 { - # The server address is part of the hash-key - # so we need three responses rxreq + expect req.url == "/1" expect req.http.x-forwarded-for == "${localhost}" - txresp -hdr "Obj: 1" + txresp + + rxreq + expect req.url == "/2" + expect req.http.x-forwarded-for == "${localhost}" + txresp + rxreq + expect req.url == "/3" + expect req.http.x-forwarded-for == "${localhost}" + txresp + + rxreq + expect req.url == "/4" + expect req.http.x-forwarded-for == "${localhost}" + txresp + + rxreq + expect req.url == "/5" + expect req.http.x-forwarded-for == "${localhost}" + txresp + + rxreq + expect req.url == "/6" expect req.http.x-forwarded-for == "1.2.3.4" - txresp -hdr "Obj: 2" + txresp + rxreq + expect req.url == "/7" expect req.http.x-forwarded-for == "102:304:506::d0e:f10" - txresp -hdr "Obj: 3" + txresp } -start varnish v1 -proto "PROXY" -vcl+backend { @@ -21,6 +44,7 @@ varnish v1 -proto "PROXY" -vcl+backend { "1.2.3.4"; "102:304:506::d0e:f10"; } + acl fwd_server { "5.6.7.8"; "8182:8384:8586::8d8e:8f80"; @@ -44,18 +68,18 @@ logexpect l1 -v v1 -g raw { expect * 1000 Proxy "2 local local local local" expect * 1003 ProxyGarbage "PROXY2: bad command \\(2\\)" expect * 1004 ProxyGarbage "PROXY2: Ignoring UNSPEC\\|UNSPEC addresses" - expect * 1006 ProxyGarbage "PROXY2: Ignoring unsupported protocol \\(0x99\\)" - expect * 1008 ProxyGarbage "PROXY2: Ignoring short IPv4 addresses \\(11\\)" - expect * 1010 ProxyGarbage "PROXY2: Ignoring short IPv6 addresses \\(35\\)" - expect * 1012 Proxy "2 1.2.3.4 2314 5.6.7.8 2828" - expect * 1015 Proxy "2 102:304:506::d0e:f10 2314 8182:8384:8586::8d8e:8f80 2828" + expect * 1007 ProxyGarbage "PROXY2: Ignoring unsupported protocol \\(0x99\\)" + expect * 1010 ProxyGarbage "PROXY2: Ignoring short IPv4 addresses \\(11\\)" + expect * 1013 ProxyGarbage "PROXY2: Ignoring short IPv6 addresses \\(35\\)" + expect * 1016 Proxy "2 1.2.3.4 2314 5.6.7.8 2828" + expect * 1019 Proxy "2 102:304:506::d0e:f10 2314 8182:8384:8586::8d8e:8f80 2828" } -start client c1 { # LOCAL command sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" sendhex "20 00 00 00" - txreq + txreq -url /1 rxresp expect resp.status == 200 expect resp.http.si == "${v1_addr}" @@ -79,7 +103,7 @@ client c1 { # UNSPEC proto sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" sendhex "21 00 00 00" - txreq + txreq -url /2 rxresp expect resp.status == 200 expect resp.http.si == "${v1_addr}" @@ -93,7 +117,7 @@ client c1 { # unknown proto sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" sendhex "21 99 00 00" - txreq + txreq -url /3 rxresp expect resp.status == 200 expect resp.http.si == "${v1_addr}" @@ -108,7 +132,7 @@ client c1 { sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" sendhex "21 11 00 0b" sendhex "01 02 03 04 05 06 07 08 09 0a 0b" - txreq + txreq -url /4 rxresp expect resp.status == 200 expect resp.http.si == "${v1_addr}" @@ -125,7 +149,7 @@ client c1 { sendhex "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" sendhex "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f" sendhex "01 02 03" - txreq + txreq -url /5 rxresp expect resp.status == 200 expect resp.http.fs == false @@ -138,10 +162,9 @@ delay .1 # good IPv4 client c1 -proxy2 "1.2.3.4:2314 5.6.7.8:2828" { - txreq + txreq -url /6 rxresp expect resp.status == 200 - expect resp.http.obj == 2 expect resp.http.fs == true expect resp.http.fc == true expect resp.http.ci == "1.2.3.4" @@ -158,10 +181,9 @@ delay .1 # good IPv6 client c1 \ -proxy2 "[102:304:506::d0e:f10]:2314 [8182:8384:8586::8d8e:8f80]:2828" { - txreq + txreq -url /7 rxresp expect resp.status == 200 - expect resp.http.obj == 3 expect resp.http.fs == true expect resp.http.fc == true expect resp.http.ci == "102:304:506::d0e:f10" diff --git a/bin/varnishtest/tests/r00262.vtc b/bin/varnishtest/tests/r00262.vtc index b5a9efc..698a0b0 100644 --- a/bin/varnishtest/tests/r00262.vtc +++ b/bin/varnishtest/tests/r00262.vtc @@ -10,13 +10,13 @@ server s1 { varnish v1 -arg "-p timeout_linger=20" -vcl+backend { } -start client c1 { - send "GET / HTTP/1.1\r\n\r\n\r\n" + send "GET / HTTP/1.1\r\nHost: foo\r\n\r\n\r\n" rxresp expect resp.status == 200 expect resp.http.X-Varnish == "1001" - send "GET / HTTP/1.1\r\n\r\n" + send "GET / HTTP/1.1\r\nHost: foo\r\n\r\n" rxresp expect resp.status == 200 expect resp.http.X-Varnish == "1003 1002" diff --git a/bin/varnishtest/tests/r01729.vtc b/bin/varnishtest/tests/r01729.vtc index 1a68f5b..883a60c 100644 --- a/bin/varnishtest/tests/r01729.vtc +++ b/bin/varnishtest/tests/r01729.vtc @@ -23,6 +23,7 @@ client c1 { non_fatal send "PUT /1 HTTP/1.1\r\n" + send "Host: foo\r\n" send "Content-Length: 31\r\n" send "Transfer-Encoding: chunked\r\n" send "\r\n" @@ -39,6 +40,7 @@ client c1 { client c1 { fatal send "PUT /2 HTTP/1.1\r\n" + send "Host: foo\r\n" send "Transfer-Encoding: chunked\r\n" send "\r\n" send "14\r\n" diff --git a/bin/varnishtest/tests/r01847.vtc b/bin/varnishtest/tests/r01847.vtc index a163fe3..ac764ee 100644 --- a/bin/varnishtest/tests/r01847.vtc +++ b/bin/varnishtest/tests/r01847.vtc @@ -24,9 +24,9 @@ client c1 { expect resp.http.rxhost == www.example.com expect resp.http.rxurl == /bar - txreq -url https://www.example.com/bar + txreq -url https://www.example.com/bar -hdr "Host: ${localhost}" rxresp - expect resp.http.rxhost == "" + expect resp.http.rxhost == "${localhost}" expect resp.http.rxurl == https://www.example.com/bar } -run diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index f73530d..197e5a8 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -32,9 +32,9 @@ shell { delay 1 client c1 { - txreq -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz" + txreq -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz" -hdr "Host: ${localhost}" rxresp - txreq -url /1?foo=bar -hdr "baz: qux" + txreq -url /1?foo=bar -hdr "baz: qux" -hdr "Host: ${localhost}" rxresp } -run @@ -44,7 +44,7 @@ shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log" shell "kill -HUP `cat ${tmpdir}/ncsa.pid`" client c1 { - txreq -url /2 + txreq -url /2 -hdr "Host: ${localhost}" rxresp } -run @@ -53,10 +53,10 @@ delay 1 shell "kill `cat ${tmpdir}/ncsa.pid`" # default formatter and rotation -shell -match {^\d+.\d+.\d+.\d+ - user \[../.../20[1-9][0-9]:..:..:.. (?# -)[+-]....\] "GET http://localhost/1\?foo=bar HTTP/1.1" 200 100 "-" "-" -\d+.\d+.\d+.\d+ - - \[../.../20[1-9][0-9]:..:..:.. [+-]....\] (?# -)"GET http://localhost/1\?foo=bar HTTP/1.1" 404 \d+ "-" "-"$} \ +shell -match {${localhost} - user \[../.../20[1-9][0-9]:..:..:.. (?# +)[+-]....\] "GET http://${localhost}/1\?foo=bar HTTP/1.1" 200 100 "-" "-" +${localhost} - - \[../.../20[1-9][0-9]:..:..:.. [+-]....\] (?# +)"GET http://${localhost}/1\?foo=bar HTTP/1.1" 404 \d+ "-" "-"$} \ "cat ${tmpdir}/ncsa.old.log" shell "grep -q /2 ${tmpdir}/ncsa.log" @@ -111,13 +111,13 @@ shell -err -match "Usage: .*varnishncsa " \ {varnishncsa extra} # -b -shell -match {^\d+.\d+.\d+.\d+ 100 -- 0$} \ +shell -match {^${localhost} 100 +${localhost} 0$} \ {varnishncsa -n ${v1_name} -b -d -F "%{Host}i %b"} # -c -shell -match {^- 100 -- \d+ -- -$} \ +shell -match {^${localhost} 100 +${localhost} \d+ +${localhost} -$} \ {varnishncsa -n ${v1_name} -c -d -F "%{Host}i %b"} # -f and standard formatters diff --git a/bin/varnishtest/tests/u00013.vtc b/bin/varnishtest/tests/u00013.vtc index 411bba8..133d999 100644 --- a/bin/varnishtest/tests/u00013.vtc +++ b/bin/varnishtest/tests/u00013.vtc @@ -3,8 +3,8 @@ varnishtest "varnishncsa outputs when UDS addresses are in use" # The %h formatter gets its value from ReqStart or BackendStart, # which now may be a UDS address. -# For UDS backends without a .hosthdr setting, the Host header is -# set to "0.0.0.0", which may appear in %r output. +# For UDS backends without a .host_header spec, the Host header is +# set to "0.0.0.0" if the header is missing in the request. server s1 -listen "${tmpdir}/s1.sock" { rxreq @@ -14,7 +14,7 @@ server s1 -listen "${tmpdir}/s1.sock" { varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {} -start client c1 -connect "${tmpdir}/v1.sock" { - txreq + txreq -proto HTTP/1.0 rxresp } -run diff --git a/bin/varnishtest/tests/v00008.vtc b/bin/varnishtest/tests/v00008.vtc index 95a870f..42798f2 100644 --- a/bin/varnishtest/tests/v00008.vtc +++ b/bin/varnishtest/tests/v00008.vtc @@ -17,7 +17,7 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -url "/foo" -hdr "Host: snafu" rxresp - txreq -url "/bar" + txreq -url "/bar" -proto HTTP/1.0 rxresp } -run @@ -37,6 +37,6 @@ varnish v1 -vcl { } client c1 { - txreq -url "/barf" + txreq -url "/barf" -proto HTTP/1.0 rxresp } -run diff --git a/bin/varnishtest/tests/v00057.vtc b/bin/varnishtest/tests/v00057.vtc index 58d8de3..01a177b 100644 --- a/bin/varnishtest/tests/v00057.vtc +++ b/bin/varnishtest/tests/v00057.vtc @@ -1,4 +1,4 @@ -varnishtest "Test backend .hosthdr with UDS backends" +varnishtest "Test host header specification with UDS backends" server s1 -listen "${tmpdir}/s1.sock" { rxreq @@ -17,7 +17,7 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -url "/foo" -hdr "Host: snafu" rxresp - txreq -url "/bar" + txreq -url "/bar" -proto HTTP/1.0 rxresp } -run @@ -36,6 +36,6 @@ varnish v1 -vcl { } client c1 { - txreq -url "/barf" + txreq -url "/barf" -proto HTTP/1.0 rxresp } -run From fgsch at lodoss.net Thu Mar 29 04:27:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 29 Mar 2018 04:27:12 +0000 (UTC) Subject: [master] c65bf8e Include the Host header if not present Message-ID: <20180329042712.ABF8B9691D@lists.varnish-cache.org> commit c65bf8e335e30a6e2dd5d720c62c0d457ebbe634 Author: Federico G. Schwindt Date: Thu Mar 29 00:09:16 2018 -0300 Include the Host header if not present Enforced for HTTP/1.1 requests. To disable it use -nohost. diff --git a/bin/varnishtest/tests/r01847.vtc b/bin/varnishtest/tests/r01847.vtc index ac764ee..8249cd2 100644 --- a/bin/varnishtest/tests/r01847.vtc +++ b/bin/varnishtest/tests/r01847.vtc @@ -24,7 +24,7 @@ client c1 { expect resp.http.rxhost == www.example.com expect resp.http.rxurl == /bar - txreq -url https://www.example.com/bar -hdr "Host: ${localhost}" + txreq -url https://www.example.com/bar rxresp expect resp.http.rxhost == "${localhost}" expect resp.http.rxurl == https://www.example.com/bar diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index 197e5a8..e18d9b9 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -32,9 +32,9 @@ shell { delay 1 client c1 { - txreq -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz" -hdr "Host: ${localhost}" + txreq -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz" rxresp - txreq -url /1?foo=bar -hdr "baz: qux" -hdr "Host: ${localhost}" + txreq -url /1?foo=bar -hdr "baz: qux" rxresp } -run @@ -44,7 +44,7 @@ shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log" shell "kill -HUP `cat ${tmpdir}/ncsa.pid`" client c1 { - txreq -url /2 -hdr "Host: ${localhost}" + txreq -url /2 rxresp } -run diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 1f72492..ab0ff6e 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -172,7 +172,7 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name) AZ(pthread_mutex_unlock(¯o_mtx)); } -static char * +char * macro_get(const char *b, const char *e) { struct macro *m; diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 8878e1d..f5b8e35 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -114,6 +114,7 @@ void macro_undef(struct vtclog *vl, const char *instance, const char *name); void macro_def(struct vtclog *vl, const char *instance, const char *name, const char *fmt, ...) v_printflike_(4, 5); +char *macro_get(const char *, const char *); struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) v_printflike_(2, 3); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index e5857e7..0634d66 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -832,11 +832,12 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen) static char* const * http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, - char* body) + char *body, unsigned nohost) { int bodylen = 0; char *b, *c; char *nullbody; + char *m; int nolen = 0; int l; @@ -846,7 +847,11 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, for (; *av != NULL; av++) { if (!strcmp(*av, "-nolen")) { nolen = 1; + } else if (!strcmp(*av, "-nohost")) { + nohost = 1; } else if (!strcmp(*av, "-hdr")) { + if (!strncasecmp(av[1], "Host:", 5)) + nohost = 1; VSB_printf(hp->vsb, "%s%s", av[1], nl); av++; } else if (!strcmp(*av, "-hdrlen")) { @@ -904,6 +909,12 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, } else break; } + if (!nohost) { + m = macro_get("localhost", NULL); + AN(m); + VSB_printf(hp->vsb, "Host: %s%s", m, nl); + free(m); + } if (body != NULL && !nolen) VSB_printf(hp->vsb, "Content-Length: %d%s", bodylen, nl); VSB_cat(hp->vsb, nl); @@ -944,8 +955,11 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * These three switches can appear in any order but must come before the * following ones. * + * \-nohost + * Don't include a Host header in the request. + * * \-nolen - * Don't include a Content-Length header in the response. + * Don't include a Content-Length header. * * \-hdr STRING * Add STRING as a header, it must follow this format: @@ -1019,7 +1033,7 @@ cmd_http_txresp(CMD_ARGS) /* send a "Content-Length: 0" header unless something else happens */ REPLACE(body, ""); - av = http_tx_parse_args(av, vl, hp, body); + av = http_tx_parse_args(av, vl, hp, body, 1); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txresp spec: %s\n", *av); @@ -1207,6 +1221,7 @@ cmd_http_txreq(CMD_ARGS) const char *url = "/"; const char *proto = "HTTP/1.1"; const char *up = NULL; + unsigned nohost; (void)cmd; (void)vl; @@ -1240,7 +1255,8 @@ cmd_http_txreq(CMD_ARGS) "Upgrade: h2c%s" "HTTP2-Settings: %s%s", nl, nl, up, nl); - av = http_tx_parse_args(av, vl, hp, NULL); + nohost = strcasecmp(proto, "HTTP/1.1") != 0; + av = http_tx_parse_args(av, vl, hp, NULL, nohost); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txreq spec: %s\n", *av); http_write(hp, 4, "txreq"); From phk at FreeBSD.org Thu Mar 29 11:13:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 11:13:11 +0000 (UTC) Subject: [master] 3c4281b Flexelinting pointer constness Message-ID: <20180329111311.58E44A36C3@lists.varnish-cache.org> commit 3c4281b1e9cbb82d546085d338075a1eb7f63bce Author: Poul-Henning Kamp Date: Thu Mar 29 11:03:55 2018 +0000 Flexelinting pointer constness diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index ab0ff6e..d4e45c7 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -460,7 +460,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir, { FILE *f; struct vsb *vsb; - char *p; + const char *p; + char *q; (void)signal(SIGPIPE, SIG_IGN); @@ -475,10 +476,10 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vsb = VSB_new_auto(); AN(vsb); if (*fn != '/') { - p = macro_get("pwd", NULL); - AN(p); - VSB_cat(vsb, p); - free(p); + q = macro_get("pwd", NULL); + AN(q); + VSB_cat(vsb, q); + free(q); } p = strrchr(fn, '/'); if (p != NULL) { From phk at FreeBSD.org Thu Mar 29 11:13:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 11:13:11 +0000 (UTC) Subject: [master] 96ba7a4 Add function to produce HAproxy backend confs Message-ID: <20180329111311.7067DA36C7@lists.varnish-cache.org> commit 96ba7a4a20cbeb468d32a17958bc127bf45272d8 Author: Poul-Henning Kamp Date: Thu Mar 29 11:05:02 2018 +0000 Add function to produce HAproxy backend confs diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index f5b8e35..5776e9b 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -91,7 +91,8 @@ int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, char * synth_body(const char *len, int rnd); -void cmd_server_genvcl(struct vsb *vsb); +void cmd_server_gen_vcl(struct vsb *vsb); +void cmd_server_gen_haproxy_conf(struct vsb *vsb); void vtc_loginit(char *buf, unsigned buflen); struct vtclog *vtc_logopen(const char *id); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 9496046..a339a7f 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -386,7 +386,7 @@ server_wait(struct server *s) */ void -cmd_server_genvcl(struct vsb *vsb) +cmd_server_gen_vcl(struct vsb *vsb) { struct server *s; @@ -407,6 +407,39 @@ cmd_server_genvcl(struct vsb *vsb) /********************************************************************** + * Generate VCL backend decls for our servers + */ + +void +cmd_server_gen_haproxy_conf(struct vsb *vsb) +{ + struct server *s; + + AZ(pthread_mutex_lock(&server_mtx)); + VTAILQ_FOREACH(s, &servers, list) { + if (*s->listen != '/') + VSB_printf(vsb, + "\n backend be%s\n" + "\tserver srv%s %s:%s\n", + s->name + 1, s->name + 1, s->aaddr, s->aport); + else + INCOMPL(); + } + VTAILQ_FOREACH(s, &servers, list) { + if (*s->listen != '/') + VSB_printf(vsb, + "\n frontend http%s\n" + "\tuse_backend be%s\n" + "\tbind \"fd@${fe%s}\"\n", + s->name + 1, s->name + 1, s->name + 1); + else + INCOMPL(); + } + AZ(pthread_mutex_unlock(&server_mtx)); +} + + +/********************************************************************** * Server command dispatch */ diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index d09d7cf..4c2e529 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -754,7 +754,7 @@ varnish_vclbackend(struct varnish *v, const char *vcl) VSB_printf(vsb2, "vcl %.1f;\n", v->syntax); - cmd_server_genvcl(vsb2); + cmd_server_gen_vcl(vsb2); AZ(VSB_finish(vsb2)); From phk at FreeBSD.org Thu Mar 29 11:13:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 11:13:11 +0000 (UTC) Subject: [master] 820e38d Move -f argument later in argv Message-ID: <20180329111311.88788A36CC@lists.varnish-cache.org> commit 820e38de58b46e3be8d2d02fee60a51a85b3ca33 Author: Poul-Henning Kamp Date: Thu Mar 29 11:05:28 2018 +0000 Move -f argument later in argv Add -conf+backend to automatically produce backend confs from list of servers diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index e3ea8d2..0cb2dad 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -200,8 +200,6 @@ haproxy_new(const char *name) h->cfg_fn = strdup(buf); AN(h->cfg_fn); - VSB_printf(h->args, "-f %s ", h->cfg_fn); - bprintf(buf, "rm -rf %s ; mkdir -p %s", h->workdir, h->workdir); AZ(system(buf)); @@ -285,6 +283,8 @@ haproxy_start(struct haproxy *h) AN(vsb); VSB_printf(vsb, "exec %s %s", h->filename, VSB_data(h->args)); + VSB_printf(vsb, " -f %s ", h->cfg_fn); + if (h->opt_worker || h->opt_daemon) { bprintf(buf, "%s/pid", h->workdir); h->pid_fn = strdup(buf); @@ -459,7 +459,7 @@ haproxy_build_backends(const struct haproxy *h, const char *vsb_data) */ static void -haproxy_write_conf(const struct haproxy *h, const char *cfg) +haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be) { struct vsb *vsb, *vsb2; @@ -472,6 +472,10 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg) VSB_printf(vsb, " global\n\tstats socket %s " "level admin mode 600\n", h->cli_fn); AZ(VSB_cat(vsb, cfg)); + + if (auto_be) + cmd_server_gen_haproxy_conf(vsb); + AZ(VSB_finish(vsb)); AZ(haproxy_build_backends(h, VSB_data(vsb))); @@ -494,7 +498,7 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg) * * To define a haproxy server, you'll use this syntax:: * - * haproxy hNAME [-arg STRING] [-conf STRING] + * haproxy hNAME [-arg STRING] [-conf[+vcl] STRING] * * The first ``haproxy hNAME`` invocation will start the haproxy master * process in the background, waiting for the ``-start`` switch to actually @@ -511,6 +515,10 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg) * \-conf STRING * Specify the configuration to be loaded by this HAProxy instance. * + * \-conf+backend STRING + * Specify the configuration to be loaded by this HAProxy instance, + * all server instances will be automatically appended + * * You can decide to start the HAProxy instance and/or wait for several events:: * * haproxy hNAME [-start] [-wait-running] [-wait-stopped] @@ -587,7 +595,13 @@ cmd_haproxy(CMD_ARGS) } if (!strcmp(*av, "-conf")) { AN(av[1]); - haproxy_write_conf(h, av[1]); + haproxy_write_conf(h, av[1], 0); + av++; + continue; + } + if (!strcmp(*av, "-conf+backend")) { + AN(av[1]); + haproxy_write_conf(h, av[1], 1); av++; continue; } From phk at FreeBSD.org Thu Mar 29 11:13:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 11:13:11 +0000 (UTC) Subject: [master] 6643c5f Add test-case for -conf+backend Message-ID: <20180329111311.A10BAA36D0@lists.varnish-cache.org> commit 6643c5f41059613106a827eedcc5499cdbd6f99d Author: Poul-Henning Kamp Date: Thu Mar 29 11:06:26 2018 +0000 Add test-case for -conf+backend diff --git a/bin/varnishtest/tests/h00003.vtc b/bin/varnishtest/tests/h00003.vtc new file mode 100644 index 0000000..c894495 --- /dev/null +++ b/bin/varnishtest/tests/h00003.vtc @@ -0,0 +1,26 @@ +varnishtest "Test -conf+backend" + +feature ignore_unknown_macro + +feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'} + +server s1 { + rxreq + txresp -body "s1 >>> Hello world!" +} -start + +haproxy h1 -arg "-d" -conf+backend { + defaults + mode http + timeout connect 5s + timeout server 30s + timeout client 30s + +} -start + +client c1 -connect ${h1_fe1_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.body == "s1 >>> Hello world!" +} -run From phk at FreeBSD.org Thu Mar 29 13:55:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 13:55:12 +0000 (UTC) Subject: [master] 665794f Give vtc_record() the ability to collect the bits in a VSB Message-ID: <20180329135512.F1765A6791@lists.varnish-cache.org> commit 665794fa22c7648fd813fcff871b89ceb24feae5 Author: Poul-Henning Kamp Date: Thu Mar 29 13:05:16 2018 +0000 Give vtc_record() the ability to collect the bits in a VSB diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 5776e9b..94dbf14 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -134,7 +134,7 @@ struct vsb *vtc_hex_to_bin(struct vtclog *vl, const char *arg); void vtc_expect(struct vtclog *, const char *, const char *, const char *, const char *, const char *); void vtc_wait4(struct vtclog *, long, int, int, int); -void *vtc_record(struct vtclog *, int); +void *vtc_record(struct vtclog *, int, struct vsb *); /* vtc_term.c */ struct term *Term_New(struct vtclog *, int, int); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 0cb2dad..4094ec0 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -243,7 +243,7 @@ haproxy_thread(void *priv) struct haproxy *h; CAST_OBJ_NOTNULL(h, priv, HAPROXY_MAGIC); - return (vtc_record(h->vl, h->fds[0])); + return (vtc_record(h->vl, h->fds[0], NULL)); } /********************************************************************** diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index e232f95..b3584a9 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -187,7 +187,7 @@ vtc_wait4(struct vtclog *vl, long pid, } void * -vtc_record(struct vtclog *vl, int fd) +vtc_record(struct vtclog *vl, int fd, struct vsb *vsb) { char buf[65536]; struct pollfd fds[1]; @@ -204,6 +204,8 @@ vtc_record(struct vtclog *vl, int fd) if (fds->revents & POLLIN) { i = read(fd, buf, sizeof buf - 1); if (i > 0) { + if (vsb != NULL) + VSB_bcat(vsb, buf, i); buf[i] = '\0'; vtc_dump(vl, 3, "debug", buf, -2); } diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 4c2e529..a688b62 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -376,7 +376,7 @@ varnish_thread(void *priv) struct varnish *v; CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC); - return (vtc_record(v->vl, v->fds[0])); + return (vtc_record(v->vl, v->fds[0], NULL)); } /********************************************************************** From phk at FreeBSD.org Thu Mar 29 13:55:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 13:55:13 +0000 (UTC) Subject: [master] 2abd431 Eliminate arguments inherited from vtc_varnish's two-process handling. Message-ID: <20180329135513.1109DA6794@lists.varnish-cache.org> commit 2abd4310090304d79c9be0a5919c227071ba9809 Author: Poul-Henning Kamp Date: Thu Mar 29 13:06:38 2018 +0000 Eliminate arguments inherited from vtc_varnish's two-process handling. diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 4094ec0..d9c0350 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -588,11 +588,6 @@ cmd_haproxy(CMD_ARGS) av++; continue; } - if (!strcmp(*av, "-cleanup")) { - AZ(av[1]); - haproxy_cleanup(h, 0); - continue; - } if (!strcmp(*av, "-conf")) { AN(av[1]); haproxy_write_conf(h, av[1], 0); @@ -614,18 +609,10 @@ cmd_haproxy(CMD_ARGS) haproxy_start(h); continue; } - if (!strcmp(*av, "-stop")) { - haproxy_stop(h); - continue; - } - if (!strcmp(*av, "-wait-stopped")) { + if (!strcmp(*av, "-wait")) { wait_stopped(h); continue; } - if (!strcmp(*av, "-wait-running")) { - wait_running(h); - continue; - } vtc_fatal(h->vl, "Unknown haproxy argument: %s", *av); } } From phk at FreeBSD.org Thu Mar 29 13:55:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 29 Mar 2018 13:55:13 +0000 (UTC) Subject: [master] 1c8fed6 More work on vtc_haproxy. Message-ID: <20180329135513.2A79BA6798@lists.varnish-cache.org> commit 1c8fed623298ec14a3ef5f0cb33b89b6eedec9f7 Author: Poul-Henning Kamp Date: Thu Mar 29 13:52:35 2018 +0000 More work on vtc_haproxy. I hadn't realized how unsuitable vtc_varnish was to copy from, because of the dual process complications, this simplifiles things back again. Move -D and -W out of -arg, it makes no sense to have to pull them out of there again. Add -conf-OK and -conf-BAD arguments. diff --git a/bin/varnishtest/tests/h00001.vtc b/bin/varnishtest/tests/h00001.vtc index 3c599a5..1b86d81 100644 --- a/bin/varnishtest/tests/h00001.vtc +++ b/bin/varnishtest/tests/h00001.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body "s1 >>> Hello world!" } -start -haproxy h1 -arg "-d" -conf { +haproxy h1 -conf { defaults mode http timeout connect 5s diff --git a/bin/varnishtest/tests/h00002.vtc b/bin/varnishtest/tests/h00002.vtc index 6e2af66..935f233 100644 --- a/bin/varnishtest/tests/h00002.vtc +++ b/bin/varnishtest/tests/h00002.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body "s1 >>> Hello world!" } -start -haproxy h1 -arg "-D" -conf { +haproxy h1 -D -conf { defaults mode http timeout connect 5s diff --git a/bin/varnishtest/tests/h00003.vtc b/bin/varnishtest/tests/h00003.vtc index c894495..404b1ee 100644 --- a/bin/varnishtest/tests/h00003.vtc +++ b/bin/varnishtest/tests/h00003.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body "s1 >>> Hello world!" } -start -haproxy h1 -arg "-d" -conf+backend { +haproxy h1 -conf+backend { defaults mode http timeout connect 5s diff --git a/bin/varnishtest/tests/h00004.vtc b/bin/varnishtest/tests/h00004.vtc new file mode 100644 index 0000000..122e041 --- /dev/null +++ b/bin/varnishtest/tests/h00004.vtc @@ -0,0 +1,18 @@ +varnishtest "Test -conf+backend" + +feature ignore_unknown_macro + +feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'} + +haproxy h1 -conf-OK { + defaults + mode http + timeout connect 5s + timeout server 30s + timeout client 30s + +} + +haproxy h2 -conf-BAD {unknown keyword 'FOOBAR' in 'global' section} { + FOOBAR +} diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index d9c0350..f8eec12 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -37,7 +37,6 @@ #include "vtc.h" -#include "vav.h" #include "vfil.h" #include "vpf.h" #include "vtcp.h" @@ -46,9 +45,9 @@ #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" #define HAPROXY_OPT_WORKER "-W" #define HAPROXY_OPT_DAEMON "-D" -#define HAPROXY_OPT_CHECK_MODE "-c" #define HAPROXY_SIGNAL SIGINT #define HAPROXY_EXPECT_EXIT (128 + HAPROXY_SIGNAL) +#define HAPROXY_GOOD_CONF "Configuration file is valid" struct haproxy { unsigned magic; @@ -77,10 +76,9 @@ struct haproxy { const char *cli_fn; char *workdir; + struct vsb *msgs; }; -static void haproxy_cleanup(struct haproxy *h, int is_haproxy_listener); - static VTAILQ_HEAD(, haproxy) haproxies = VTAILQ_HEAD_INITIALIZER(haproxies); @@ -89,42 +87,14 @@ static VTAILQ_HEAD(, haproxy) haproxies = */ static void -wait_stopped(struct haproxy *h) -{ - vtc_log(h->vl, 3, "wait-stopped"); - while (1) { - if (h->kill_status >= 0 || h->kill_errno == ESRCH) - break; - - usleep(1000); - h->kill_status = kill(h->pid, 0); - h->kill_errno = errno; - } -} - -static void -wait_running(struct haproxy *h) +haproxy_wait_pidfile(struct haproxy *h) { char buf_err[1024] = {0}; int usleep_time = 1000; double t0; pid_t pid; - if (h->opt_check_mode) - return; - - if (h->pid_fn == NULL) { - /* - * If we use no pid-file, check that the process - * we started is still running. - */ - if (kill(h->pid, 0) < 0) - vtc_fatal(h->vl, "Process %ld not there: %s", - (long)h->pid, strerror(errno)); - return; - } - - vtc_log(h->vl, 3, "wait-running"); + vtc_log(h->vl, 3, "wait-pid-file"); for (t0 = VTIM_mono(); VTIM_mono() - t0 < 3;) { if (vtc_error) return; @@ -243,7 +213,7 @@ haproxy_thread(void *priv) struct haproxy *h; CAST_OBJ_NOTNULL(h, priv, HAPROXY_MAGIC); - return (vtc_record(h->vl, h->fds[0], NULL)); + return (vtc_record(h->vl, h->fds[0], h->msgs)); } /********************************************************************** @@ -253,35 +223,30 @@ haproxy_thread(void *priv) static void haproxy_start(struct haproxy *h) { - char **argv; - int i; char buf[PATH_MAX]; struct vsb *vsb; vtc_log(h->vl, 2, "%s", __func__); AZ(VSB_finish(h->args)); - argv = VAV_Parse(VSB_data(h->args), NULL, 0); - AN(argv); - if (argv[0] != NULL) - vtc_fatal(h->vl, "Could not parse argv-string: %s", argv[0]); - for (i = 1; argv[i] != NULL; i++) { - /* Check if HAPROXY_OPT_WORKER option was provided. */ - if (!strcmp(argv[i], HAPROXY_OPT_WORKER)) { - h->opt_worker = 1; - } else if (!strcmp(argv[i], HAPROXY_OPT_DAEMON)) { - h->opt_daemon = 1; - } else if (!strcmp(argv[i], HAPROXY_OPT_CHECK_MODE)) { - h->opt_check_mode = 1; - } - } - VAV_Free(argv); vtc_log(h->vl, 4, "opt_worker %d opt_daemon %d opt_check_mode %d", h->opt_worker, h->opt_daemon, h->opt_check_mode); vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "exec %s %s", h->filename, VSB_data(h->args)); + + VSB_printf(vsb, "exec %s", h->filename); + if (h->opt_check_mode) + VSB_printf(vsb, " -c"); + else if (h->opt_daemon) + VSB_printf(vsb, " -D"); + else + VSB_printf(vsb, " -d"); + + if (h->opt_worker) + VSB_printf(vsb, " -W"); + + VSB_printf(vsb, " %s", VSB_data(h->args)); VSB_printf(vsb, " -f %s ", h->cfg_fn); @@ -301,11 +266,10 @@ haproxy_start(struct haproxy *h) * if signaled by signal. */ h->expect_exit = HAPROXY_EXPECT_EXIT; - } else { - h->expect_exit = 0; } AZ(pipe(&h->fds[0])); + vtc_log(h->vl, 4, "XXX %d @%d", h->fds[1], __LINE__); AZ(pipe(&h->fds[2])); h->pid = h->ppid = fork(); assert(h->pid >= 0); @@ -334,73 +298,44 @@ haproxy_start(struct haproxy *h) AZ(pthread_create(&h->tp, NULL, haproxy_thread, h)); - wait_running(h); + if (h->pid_fn != NULL) + haproxy_wait_pidfile(h); } -/********************************************************************** - * Stop a HAProxy - */ - -static void -haproxy_stop(struct haproxy *h) -{ - if (h->opt_check_mode) - return; - - if (h->pid < 0) - haproxy_start(h); - - vtc_log(h->vl, 2, "Stop"); - h->kill_status = kill(h->pid, HAPROXY_SIGNAL); - h->kill_errno = errno; - h->expect_signal = -HAPROXY_SIGNAL; - - wait_stopped(h); -} /********************************************************************** - * Cleanup + * Wait for a HAProxy instance. */ static void -haproxy_cleanup(struct haproxy *h, int is_haproxy_listener) +haproxy_wait(struct haproxy *h) { void *p; - vtc_log(h->vl, 2, "%s (%d)", __func__, is_haproxy_listener); - if (!is_haproxy_listener) { - /* Close the STDIN connection. */ - closefd(&h->fds[1]); - - /* Wait until STDOUT+STDERR closes */ - AZ(pthread_join(h->tp, &p)); - closefd(&h->fds[0]); - if (h->opt_daemon) - return; - } - - if (h->ppid == -1) - return; - - vtc_wait4(h->vl, h->ppid, h->expect_exit, h->expect_signal, 0); - h->ppid = -1; -} - -/********************************************************************** - * Wait for a HAProxy instance. - */ + vtc_log(h->vl, 2, "Wait"); -static void -haproxy_wait(struct haproxy *h) -{ if (h->pid < 0) - return; + haproxy_start(h); - vtc_log(h->vl, 2, "Wait"); + closefd(&h->fds[1]); - haproxy_stop(h); + if (!h->opt_check_mode) { + assert(h->pid > 0); + vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); + h->kill_status = kill(h->pid, HAPROXY_SIGNAL); + h->kill_errno = errno; + h->expect_signal = -HAPROXY_SIGNAL; + // XXX: loop over kills to ESRCH ? + } - haproxy_cleanup(h, 0); + AZ(pthread_join(h->tp, &p)); + AZ(p); + closefd(&h->fds[0]); + if (!h->opt_daemon) { + vtc_wait4(h->vl, h->ppid, h->expect_exit, h->expect_signal, 0); + h->ppid = -1; + } + h->pid = -1; } #define HAPROXY_BE_FD_STR "fd@${" @@ -454,6 +389,22 @@ haproxy_build_backends(const struct haproxy *h, const char *vsb_data) return (0); } +static void +haproxy_check_conf(struct haproxy *h, const char *expect) +{ + + h->msgs = VSB_new_auto(); + AN(h->msgs); + h->opt_check_mode = 1; + haproxy_start(h); + haproxy_wait(h); + AZ(VSB_finish(h->msgs)); + if (strstr(VSB_data(h->msgs), expect) == NULL) + vtc_fatal(h->vl, "Did not find expected string '%s'", expect); + vtc_log(h->vl, 2, "Found expected '%s'", expect); + VSB_destroy(&h->msgs); +} + /********************************************************************** * Write a configuration for HAProxy instance. */ @@ -498,7 +449,9 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be) * * To define a haproxy server, you'll use this syntax:: * - * haproxy hNAME [-arg STRING] [-conf[+vcl] STRING] + * haproxy hNAME -conf-OK CONFIG + * haproxy hNAME -conf-BAD ERROR CONFIG + * haproxy hNAME [-D] [-W] [-arg STRING] [-conf[+vcl] STRING] * * The first ``haproxy hNAME`` invocation will start the haproxy master * process in the background, waiting for the ``-start`` switch to actually @@ -509,6 +462,22 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be) * hNAME * Identify the HAProxy server with a string, it must starts with 'h'. * + * \-conf-OK CONFIG + * Run haproxy in '-c' mode to check config is OK + * stdout/stderr should contain 'Configuration file is valid' + * The exit code should be 0. + * + * \-conf-BAD ERROR CONFIG + * Run haproxy in '-c' mode to check config is BAD. + * "ERROR" should be part of the diagnostics on stdout/stderr. + * The exit code should be 1. + * + * \-D + * Run HAproxy in daemon mode. If not given '-d' mode used. + * + * \-W + * Enable HAproxy in Worker mode. + * * \-arg STRING * Pass an argument to haproxy, for example "-h simple_list". * @@ -519,26 +488,12 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be) * Specify the configuration to be loaded by this HAProxy instance, * all server instances will be automatically appended * - * You can decide to start the HAProxy instance and/or wait for several events:: - * - * haproxy hNAME [-start] [-wait-running] [-wait-stopped] - * * \-start * Start this HAProxy instance. * - * \-stop + * \-wait * Stop this HAProxy instance. * - * \-wait-running - * Wait for that instance to terminate. - * - * \-wait-stopped - * Wait for that instance to terminate. - * - * \-cleanup - * Once HAProxy is stopped, clean everything after it. This is only used - * in very few tests and you should never need it. - * * \-expectexit NUMBER * Expect haproxy to exit(3) with this value * @@ -580,6 +535,33 @@ cmd_haproxy(CMD_ARGS) for (; *av != NULL; av++) { if (vtc_error) break; + + if (!strcmp(*av, "-conf-OK")) { + AN(av[1]); + haproxy_write_conf(h, av[1], 0); + av++; + haproxy_check_conf(h, HAPROXY_GOOD_CONF); + continue; + } + if (!strcmp(*av, "-conf-BAD")) { + AN(av[1]); + AN(av[2]); + haproxy_write_conf(h, av[2], 0); + h->expect_exit = 1; + haproxy_check_conf(h, av[1]); + av += 2; + continue; + } + + if (!strcmp(*av, HAPROXY_OPT_DAEMON)) { + h->opt_daemon = 1; + continue; + } + if (!strcmp(*av, HAPROXY_OPT_WORKER)) { + h->opt_worker = 1; + continue; + } + if (!strcmp(*av, "-arg")) { AN(av[1]); AZ(h->pid); @@ -588,6 +570,7 @@ cmd_haproxy(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-conf")) { AN(av[1]); haproxy_write_conf(h, av[1], 0); @@ -600,6 +583,7 @@ cmd_haproxy(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-expectexit")) { h->expect_exit = strtoul(av[1], NULL, 0); av++; @@ -610,7 +594,7 @@ cmd_haproxy(CMD_ARGS) continue; } if (!strcmp(*av, "-wait")) { - wait_stopped(h); + haproxy_wait(h); continue; } vtc_fatal(h->vl, "Unknown haproxy argument: %s", *av); From fgsch at lodoss.net Thu Mar 29 17:22:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 29 Mar 2018 17:22:11 +0000 (UTC) Subject: [master] a3f94bd Don't call VTCP_hisname() on UDS connections Message-ID: <20180329172211.E9CAEAC6A7@lists.varnish-cache.org> commit a3f94bde6c004cb97ec35fd6813859ce0cc09e73 Author: Federico G. Schwindt Date: Thu Mar 29 14:19:37 2018 -0300 Don't call VTCP_hisname() on UDS connections diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index a339a7f..1307cd5 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -242,8 +242,11 @@ server_thread(void *priv) fd = accept(s->sock, addr, &l); if (fd < 0) vtc_fatal(vl, "Accept failed: %s", strerror(errno)); - VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf); - vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf); + if (*s->listen != '/') { + VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf); + vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf); + } else + vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd); fd = http_process(vl, s->spec, fd, &s->sock, s->listen); vtc_log(vl, 3, "shutting fd %d", fd); j = shutdown(fd, SHUT_WR); From fgsch at lodoss.net Fri Mar 30 13:37:13 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 13:37:13 +0000 (UTC) Subject: [master] 1ed2bb6 Correct assert Message-ID: <20180330133713.E1F4395FC9@lists.varnish-cache.org> commit 1ed2bb63a5bac9bf0843603f0e00948a38349e92 Author: Federico G. Schwindt Date: Fri Mar 30 10:25:11 2018 -0300 Correct assert diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 50cdd9a..799d109 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -212,7 +212,7 @@ h2_b64url_settings(struct h2_sess *h2, struct req *req) if (q == NULL) return (-1); i = q - s; - assert(i >= 0 && i <= 63); + assert(i >= 0 && i <= 64); x <<= 6; x |= i; n += 6; From fgsch at lodoss.net Fri Mar 30 14:39:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 14:39:08 +0000 (UTC) Subject: [master] a581aa8 Don't report on vsmw_newcluster for now Message-ID: <20180330143908.3728B9727C@lists.varnish-cache.org> commit a581aa81d3d06f14041f0b96be2efe842a94615b Author: Federico G. Schwindt Date: Fri Mar 30 11:33:21 2018 -0300 Don't report on vsmw_newcluster for now diff --git a/tools/lsan.suppr b/tools/lsan.suppr index 5cef2ee..67ea016 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -17,3 +17,5 @@ leak:hpack/vhp_gen_hufdec.c leak:binheap_new # ev leak:mct_callback +# +leak:vsmw_newcluster From fgsch at lodoss.net Fri Mar 30 14:39:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 14:39:08 +0000 (UTC) Subject: [master] aa69c9a Test persistent storage under Travis Message-ID: <20180330143908.4E2C09727F@lists.varnish-cache.org> commit aa69c9a4ecf45e19d855404451e4044d4d8801cf Author: Federico G. Schwindt Date: Fri Mar 30 11:35:26 2018 -0300 Test persistent storage under Travis diff --git a/.travis.yml b/.travis.yml index 4b310be..f716744 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ before_install: sudo apt-get update; sudo apt-get install -y clang-$CLANG llvm-$CLANG; export CC=clang-$CLANG; - export CONFIGURE_ARGS="--enable-developer-warnings --enable-debugging-symbols --disable-stack-protector ${SAN_FLAGS}"; + export CONFIGURE_ARGS="--enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage ${SAN_FLAGS}"; export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0; export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr; export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr; From fgsch at lodoss.net Fri Mar 30 14:39:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 14:39:08 +0000 (UTC) Subject: [master] 047a30e Plug tiny leak Message-ID: <20180330143908.687DE97283@lists.varnish-cache.org> commit 047a30e2b5025d4654f94b6f09c47c043679daf3 Author: Federico G. Schwindt Date: Fri Mar 30 11:36:24 2018 -0300 Plug tiny leak diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index fe16d89..981c5bf 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -414,6 +414,7 @@ mgt_launch_child(struct cli *cli) return; } + free(p); child_state = CH_RUNNING; } From fgsch at lodoss.net Fri Mar 30 18:58:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 18:58:11 +0000 (UTC) Subject: [master] 52e8fb1 Also exclude backtrace_symbols Message-ID: <20180330185811.C07D5A0600@lists.varnish-cache.org> commit 52e8fb15a6a39be0820c5449174f2441b29448ee Author: Federico G. Schwindt Date: Fri Mar 30 12:16:58 2018 -0300 Also exclude backtrace_symbols diff --git a/tools/lsan.suppr b/tools/lsan.suppr index 67ea016..89bc6c7 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -18,4 +18,6 @@ leak:binheap_new # ev leak:mct_callback # +leak:backtrace_symbols +# leak:vsmw_newcluster From fgsch at lodoss.net Fri Mar 30 18:58:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Mar 2018 18:58:11 +0000 (UTC) Subject: [master] a3696e6 Plug another tiny leak Message-ID: <20180330185811.D392FA0603@lists.varnish-cache.org> commit a3696e650951f397b2952c62011acb73f59520fe Author: Federico G. Schwindt Date: Fri Mar 30 15:34:16 2018 -0300 Plug another tiny leak diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index db42d31..69f3195 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -282,6 +282,7 @@ interactive(int sock) } } } + free(answer); cli_write(sock, "banner\n"); while (1) { i = poll(fds, 2, -1); From fgsch at lodoss.net Sat Mar 31 13:38:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 31 Mar 2018 13:38:11 +0000 (UTC) Subject: [master] 71519e2 Tweak this parameter regardless of configure args Message-ID: <20180331133811.4AAC5B7951@lists.varnish-cache.org> commit 71519e2d9d99eb3f52af0ff1677988a3c4cfbf40 Author: Federico G. Schwindt Date: Sat Mar 31 10:32:01 2018 -0300 Tweak this parameter regardless of configure args diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 2d262e9..0797bfc 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -510,7 +510,8 @@ MCF_InitParams(struct cli *cli) low = sysconf(_SC_THREAD_STACK_MIN); MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low); -#if defined(__SANITIZER) +#if defined(__SANITIZER) || \ + (defined(__has_feature) && __has_feature(address_sanitizer)) def = 92 * 1024; #else def = 48 * 1024; From fgsch at lodoss.net Sat Mar 31 14:38:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 31 Mar 2018 14:38:08 +0000 (UTC) Subject: [master] 998d7b2 Rework to make gcc happy Message-ID: <20180331143808.1CD34B8B53@lists.varnish-cache.org> commit 998d7b2e5078b8ac7e68727b78a92dd4aa342535 Author: Federico G. Schwindt Date: Sat Mar 31 11:27:28 2018 -0300 Rework to make gcc happy diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 0797bfc..139814c 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -43,6 +43,10 @@ #include "vav.h" #include "vcli_serve.h" +#if !defined(__has_feature) +#define __has_feature(x) 0 +#endif + struct plist { unsigned magic; #define PLIST_MAGIC 0xbfc3ea16 @@ -510,8 +514,7 @@ MCF_InitParams(struct cli *cli) low = sysconf(_SC_THREAD_STACK_MIN); MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low); -#if defined(__SANITIZER) || \ - (defined(__has_feature) && __has_feature(address_sanitizer)) +#if defined(__SANITIZER) || __has_feature(address_sanitizer) def = 92 * 1024; #else def = 48 * 1024; From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] 9260dbe Parse proxy-protocol-v2 tlv and ssl sub tlv. Message-ID: <20180314161008.E373DA7E54@lists.varnish-cache.org> commit 9260dbe0e3f90c79268cd3a073da063d3e8e1fd8 Author: Emmanuel Hocdet Date: Thu Feb 8 16:30:07 2018 +0100 Parse proxy-protocol-v2 tlv and ssl sub tlv. Parse and check tlv and ssl sub tlv as described in: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt diff --git a/bin/varnishd/proxy/cache_proxy.h b/bin/varnishd/proxy/cache_proxy.h new file mode 100644 index 0000000..c3cfcb1 --- /dev/null +++ b/bin/varnishd/proxy/cache_proxy.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2018 GANDI SAS + * All rights reserved. + * + * Author: Emmanuel Hocdet + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define PP2_TYPE_ALPN 0x01 +#define PP2_TYPE_AUTHORITY 0x02 +#define PP2_TYPE_CRC32C 0x03 +#define PP2_TYPE_NOOP 0x04 +#define PP2_TYPE_SSL 0x20 +#define PP2_SUBTYPE_SSL_VERSION 0x21 +#define PP2_SUBTYPE_SSL_CN 0x22 +#define PP2_SUBTYPE_SSL_CIPHER 0x23 +#define PP2_SUBTYPE_SSL_SIG_ALG 0x24 +#define PP2_SUBTYPE_SSL_KEY_ALG 0x25 +#define PP2_SUBTYPE_SSL_MAX 0x25 diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index a0f76fe..b16f971 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -1,8 +1,10 @@ /*- * Copyright (c) 2015 Varnish Software AS + * Copyright (c) 2018 GANDI SAS * All rights reserved. * - * Author: Poul-Henning Kamp + * Authors: Poul-Henning Kamp + * Emmanuel Hocdet * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,6 +36,7 @@ #include "cache/cache_varnishd.h" #include "cache/cache_transport.h" +#include "proxy/cache_proxy.h" #include "vend.h" #include "vsa.h" @@ -153,6 +156,19 @@ vpx_proto1(const struct worker *wrk, const struct req *req) * PROXY 2 protocol */ +struct pp2_tlv { + uint8_t type; + uint8_t length_hi; + uint8_t length_lo; + uint8_t value[0]; +}__attribute__((packed)); + +struct pp2_tlv_ssl { + uint8_t client; + uint32_t verify; + struct pp2_tlv sub_tlv[0]; +}__attribute__((packed)); + static const char vpx2_sig[] = { '\r', '\n', '\r', '\n', '\0', '\r', '\n', 'Q', 'U', 'I', 'T', '\n', @@ -161,8 +177,9 @@ static const char vpx2_sig[] = { static int vpx_proto2(const struct worker *wrk, struct req *req) { - int l; + int l, hdr_len; const uint8_t *p; + char *d; sa_family_t pfam = 0xff; struct sockaddr_in sin4; struct sockaddr_in6 sin6; @@ -178,10 +195,12 @@ vpx_proto2(const struct worker *wrk, struct req *req) assert(req->htc->rxbuf_e - req->htc->rxbuf_b >= 16L); l = vbe16dec(req->htc->rxbuf_b + 14); - assert(req->htc->rxbuf_e - req->htc->rxbuf_b >= 16L + l); - HTC_RxPipeline(req->htc, req->htc->rxbuf_b + 16L + l); + hdr_len = l + 16L; + assert(req->htc->rxbuf_e - req->htc->rxbuf_b >= hdr_len); + HTC_RxPipeline(req->htc, req->htc->rxbuf_b + hdr_len); WS_Reset(req->ws, 0); p = (const void *)req->htc->rxbuf_b; + d = req->htc->rxbuf_b + 16L; /* Version @12 top half */ if ((p[12] >> 4) != 2) { @@ -219,6 +238,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) "PROXY2: Ignoring short IPv4 addresses (%d)", l); return (0); } + l -= 12; + d += 12; break; case 0x21: /* IPv6|TCP */ @@ -228,6 +249,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) "PROXY2: Ignoring short IPv6 addresses (%d)", l); return (0); } + l -= 36; + d += 36; break; default: /* Ignore proxy header */ @@ -281,6 +304,39 @@ vpx_proto2(const struct worker *wrk, struct req *req) SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, pb); VSL(SLT_Proxy, req->sp->vxid, "2 %s %s %s %s", hb, pb, ha, pa); + + while (l > sizeof(struct pp2_tlv)) { + int el = vbe16dec(d + 1) + 3; + if (el > l) { + VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: Ignoring TLV"); + return (0); + } + switch(d[0]) { + case PP2_TYPE_SSL: + { + const char *sd; + int sl; + sd = d + sizeof(struct pp2_tlv) + sizeof(struct pp2_tlv_ssl); + sl = l - sizeof(struct pp2_tlv) - sizeof(struct pp2_tlv_ssl); + while (sl > sizeof(struct pp2_tlv)) { + int esl = vbe16dec(sd + 1) + 3; + if (esl > sl) { + VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: Ignoring SSL TLV"); + return (0); + } + sd += esl; + sl -= esl; + } + break; + } + } + d += el; + l -= el; + } + if (l) { + VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: header length mismatch"); + return (0); + } return (0); } From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] 405d376 Support proxy-protocol-v2 crc32c tlv. Message-ID: <20180314161009.038E0A7E57@lists.varnish-cache.org> commit 405d3767ca44197a78153388a2ca4e0e0dac61ab Author: Emmanuel Hocdet Date: Thu Feb 8 16:32:06 2018 +0100 Support proxy-protocol-v2 crc32c tlv. Compute checksum if crc32c tlv is send in proxy-protocol-v2 header. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index b16f971..63eb461 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -174,6 +174,82 @@ static const char vpx2_sig[] = { 'Q', 'U', 'I', 'T', '\n', }; +static uint32_t crctable[256] = { + 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, + 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, + 0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, + 0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L, + 0x105EC76FL, 0xE235446CL, 0xF165B798L, 0x030E349BL, + 0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L, + 0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L, 0x89D76C54L, + 0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL, + 0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL, + 0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L, + 0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L, + 0x6DFE410EL, 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL, + 0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L, + 0xF779DEAEL, 0x05125DADL, 0x1642AE59L, 0xE4292D5AL, + 0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL, + 0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L, 0x6EF07595L, + 0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L, + 0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L, + 0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L, + 0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L, + 0x5125DAD3L, 0xA34E59D0L, 0xB01EAA24L, 0x42752927L, + 0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L, + 0xDBFC821CL, 0x2997011FL, 0x3AC7F2EBL, 0xC8AC71E8L, + 0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L, + 0x61C69362L, 0x93AD1061L, 0x80FDE395L, 0x72966096L, + 0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L, + 0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L, + 0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L, + 0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L, + 0xB602C312L, 0x44694011L, 0x5739B3E5L, 0xA55230E6L, + 0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L, + 0x3CDB9BDDL, 0xCEB018DEL, 0xDDE0EB2AL, 0x2F8B6829L, + 0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL, + 0x456CAC67L, 0xB7072F64L, 0xA457DC90L, 0x563C5F93L, + 0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L, + 0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL, + 0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L, + 0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL, + 0x1871A4D8L, 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL, + 0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L, + 0xA24BB5A6L, 0x502036A5L, 0x4370C551L, 0xB11B4652L, + 0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL, + 0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL, 0x3BC21E9DL, + 0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L, + 0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL, + 0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L, + 0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L, + 0xFF56BD19L, 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL, + 0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L, + 0x0417B1DBL, 0xF67C32D8L, 0xE52CC12CL, 0x1747422FL, + 0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL, + 0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L, 0x9D9E1AE0L, + 0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL, + 0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L, + 0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L, + 0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL, + 0xE330A81AL, 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL, + 0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L, + 0x69E9F0D5L, 0x9B8273D6L, 0x88D28022L, 0x7AB90321L, + 0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL, + 0xF36E6F75L, 0x0105EC76L, 0x12551F82L, 0xE03E9C81L, + 0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL, + 0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL, + 0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L +}; + +static uint32_t crc32c(const uint8_t *buf, int len) +{ + uint32_t crc = 0xffffffff; + while (len-- > 0) { + crc = (crc >> 8) ^ crctable[(crc ^ (*buf++)) & 0xff]; + } + return (crc ^ 0xffffffff); +} + static int vpx_proto2(const struct worker *wrk, struct req *req) { @@ -312,6 +388,16 @@ vpx_proto2(const struct worker *wrk, struct req *req) return (0); } switch(d[0]) { + case PP2_TYPE_CRC32C: + { + uint32_t n_crc32c = vbe32dec(d+3); + *(d+3) = 0; *(d+4) = 0; *(d+5) = 0; *(d+6) = 0; + if (crc32c(p, hdr_len) != n_crc32c) { + VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: CRC error"); + return (-1); + } + break; + } case PP2_TYPE_SSL: { const char *sd; From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] cc4a99f Store PROXY TLV attributes in sess_attr xport_priv. Message-ID: <20180314161009.243FBA7E5B@lists.varnish-cache.org> commit cc4a99f421ca6054b581beef9d0436110eda1238 Author: Emmanuel Hocdet Date: Tue Feb 27 12:20:06 2018 +0100 Store PROXY TLV attributes in sess_attr xport_priv. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 63eb461..44083c2 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -254,8 +254,11 @@ static int vpx_proto2(const struct worker *wrk, struct req *req) { int l, hdr_len; + uintptr_t *up; + uint16_t *tlv_len_p; + uint16_t tlv_len = 0; const uint8_t *p; - char *d; + char *d, *tlv_start; sa_family_t pfam = 0xff; struct sockaddr_in sin4; struct sockaddr_in6 sin6; @@ -381,6 +384,9 @@ vpx_proto2(const struct worker *wrk, struct req *req) VSL(SLT_Proxy, req->sp->vxid, "2 %s %s %s %s", hb, pb, ha, pa); + tlv_start = d; + tlv_len = l; + while (l > sizeof(struct pp2_tlv)) { int el = vbe16dec(d + 1) + 3; if (el > l) { @@ -423,6 +429,14 @@ vpx_proto2(const struct worker *wrk, struct req *req) VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: header length mismatch"); return (0); } + if (tlv_len && WS_Reserve(req->sp->ws, 2 + tlv_len)) { + tlv_len_p = (uint16_t *)req->sp->ws->f; + *tlv_len_p = tlv_len; + memcpy(req->sp->ws->f + 2, tlv_start, tlv_len); + WS_Release(req->sp->ws, 2 + tlv_len); + SES_Reserve_xport_priv(req->sp, &up); + *up = (uintptr_t)tlv_len_p; + } return (0); } From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] 86a62bb Add VPX_tlv interface to extract PROXY tlv attributes. Message-ID: <20180314161009.56D97A7E60@lists.varnish-cache.org> commit 86a62bbe83d2c6ea4f247387b6a1c568c8c475ae Author: Emmanuel Hocdet Date: Tue Feb 27 12:21:22 2018 +0100 Add VPX_tlv interface to extract PROXY tlv attributes. diff --git a/bin/varnishd/proxy/cache_proxy.h b/bin/varnishd/proxy/cache_proxy.h index c3cfcb1..1e4dd25 100644 --- a/bin/varnishd/proxy/cache_proxy.h +++ b/bin/varnishd/proxy/cache_proxy.h @@ -37,3 +37,5 @@ #define PP2_SUBTYPE_SSL_SIG_ALG 0x24 #define PP2_SUBTYPE_SSL_KEY_ALG 0x25 #define PP2_SUBTYPE_SSL_MAX 0x25 + +int VPX_tlv(const struct req *req, int tlv, void **dst, int *len); diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 44083c2..fee5ac2 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -250,6 +250,60 @@ static uint32_t crc32c(const uint8_t *buf, int len) return (crc ^ 0xffffffff); } +int +VPX_tlv(const struct req *req, int tlv, void **dst, int *len) +{ + uintptr_t *p; + uint16_t l; + char *d; + int ssltlv = 0; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); + + if (SES_Get_xport_priv(req->sp, &p) != 0) { + *dst = NULL; + return (-1); + } + d = (char *)*p; + l = *(uint16_t *)d; + d += 2; + + if (tlv > PP2_TYPE_SSL && tlv <= PP2_SUBTYPE_SSL_MAX) { + ssltlv = tlv; + tlv = PP2_TYPE_SSL; + } + while (l > sizeof(struct pp2_tlv)) { + uint16_t v_len = vbe16dec(d + 1); + if (d[0] == tlv) { + if (ssltlv) { + char *sd; + int sl; + sd = d + sizeof(struct pp2_tlv) + sizeof(struct pp2_tlv_ssl); + sl = l - sizeof(struct pp2_tlv) - sizeof(struct pp2_tlv_ssl); + while (sl > sizeof(struct pp2_tlv)) { + uint16_t subv_len = vbe16dec(sd + 1); + if (sd[0] == ssltlv) { + *dst = sd + 3; + *len = subv_len; + return (0); + } + sd += (subv_len + 3); + sl -= (subv_len + 3); + } + } else { + *dst = d + 3; + *len = v_len; + return (0); + } + } + d += (v_len + 3); + l -= (v_len + 3); + } + *dst = NULL; + return (-1); +} + static int vpx_proto2(const struct worker *wrk, struct req *req) { From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] c8d7078 Add VMOD proxy Message-ID: <20180314161009.8D526A7E64@lists.varnish-cache.org> commit c8d7078fcc902445530f45120b1e6efbd4e93bae Author: Emmanuel Hocdet Date: Thu Mar 1 18:06:31 2018 +0100 Add VMOD proxy diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc new file mode 100644 index 0000000..49cb0ef --- /dev/null +++ b/bin/varnishtest/tests/o00005.vtc @@ -0,0 +1,41 @@ +varnishtest "PROXY v2 TLV test" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -proto "PROXY" -vcl+backend { + import proxy; + + sub vcl_deliver { + if (proxy.ssl_verify_result() == 0) { + set resp.http.verify = "ok"; + } + set resp.http.alpn = proxy.alpn(); + set resp.http.ssl-version = proxy.ssl_version(); + set resp.http.cipher = proxy.ssl_cipher(); + set resp.http.key = proxy.cert_key(); + set resp.http.sign = proxy.cert_sign(); + } +} -start + +client c1 { + sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 11 00 65" + sendhex "d9 46 b5 21 5f 8e a8 22 ed 96 01 bb 03 00 04 95" + sendhex "03 ee 75 01 00 02 68 32 02 00 0a 68 6f 63 64 65" + sendhex "74 2e 6e 65 74 20 00 3d 01 00 00 00 00 21 00 07" + sendhex "54 4c 53 76 31 2e 33 25 00 05 45 43 32 35 36 24" + sendhex "00 0a 52 53 41 2d 53 48 41 32 35 36 23 00 16 41" + sendhex "45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53" + sendhex "48 41 32 35 36" + txreq + rxresp + expect resp.status == 200 + expect resp.http.verify == ok + expect resp.http.alpn == h2 + expect resp.http.ssl-version == TLSv1.3 + expect resp.http.cipher == AEAD-AES128-GCM-SHA256 + expect resp.http.key == EC256 + expect resp.http.sign == RSA-SHA256 +} -run diff --git a/bin/varnishtest/vmods.h b/bin/varnishtest/vmods.h index fb8f68f..06ab9b8 100644 --- a/bin/varnishtest/vmods.h +++ b/bin/varnishtest/vmods.h @@ -34,3 +34,4 @@ VTC_VMOD(purge) VTC_VMOD(vtc) VTC_VMOD(blob) VTC_VMOD(unix) +VTC_VMOD(proxy) diff --git a/configure.ac b/configure.ac index 4323d1b..45bdb45 100644 --- a/configure.ac +++ b/configure.ac @@ -760,6 +760,7 @@ AC_CONFIG_FILES([ lib/libvmod_vtc/Makefile lib/libvmod_blob/Makefile lib/libvmod_unix/Makefile + lib/libvmod_proxy/Makefile man/Makefile varnishapi.pc varnishapi-uninstalled.pc diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 810e903..04fa9db 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -222,6 +222,10 @@ reference/vmod_unix.generated.rst: reference $(top_builddir)/lib/libvmod_unix/vm cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ || true BUILT_SOURCES += reference/vmod_unix.generated.rst +reference/vmod_proxy.generated.rst: reference $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst + cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@ || true +BUILT_SOURCES += reference/vmod_proxy.generated.rst + EXTRA_DIST += $(BUILT_SOURCES) MAINTAINERCLEANFILES = $(EXTRA_DIST) diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 9df60eb..1bf5435 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -26,6 +26,7 @@ The Varnish Reference Manual vmod_purge.generated.rst vmod_blob.generated.rst vmod_unix.generated.rst + vmod_proxy.generated.rst directors.rst varnish-counters.rst vsl.rst diff --git a/lib/Makefile.am b/lib/Makefile.am index 411bd80..907ee4a 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -11,4 +11,5 @@ SUBDIRS = \ libvmod_purge \ libvmod_vtc \ libvmod_blob \ - libvmod_unix + libvmod_unix \ + libvmod_proxy diff --git a/lib/libvmod_proxy/Makefile.am b/lib/libvmod_proxy/Makefile.am new file mode 100644 index 0000000..d8d5861 --- /dev/null +++ b/lib/libvmod_proxy/Makefile.am @@ -0,0 +1,7 @@ +# + +libvmod_proxy_la_SOURCES = \ + vmod_proxy.c + +# Use vmodtool.py generated automake boilerplate +include $(srcdir)/automake_boilerplate.am diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate.am new file mode 100644 index 0000000..64f3c5a --- /dev/null +++ b/lib/libvmod_proxy/automake_boilerplate.am @@ -0,0 +1,39 @@ + +# Boilerplate generated by vmodtool.py - changes will be overwritten + +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmoddir = $(pkglibdir)/vmods +vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py +vmodtoolargs = --strict --boilerplate + +vmod_LTLIBRARIES = libvmod_proxy.la + +libvmod_proxy_la_CFLAGS = \ + @SAN_CFLAGS@ + +libvmod_proxy_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + $(VMOD_LDFLAGS) \ + @SAN_LDFLAGS@ + +nodist_libvmod_proxy_la_SOURCES = vcc_if.c vcc_if.h + +$(libvmod_proxy_la_OBJECTS): vcc_if.h + +vcc_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_if.c + +vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc + +EXTRA_DIST = vmod.vcc automake_boilerplate.am + +CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \ + $(builddir)/vmod_proxy.rst \ + $(builddir)/vmod_proxy.man.rst + diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod.vcc new file mode 100644 index 0000000..c3774db --- /dev/null +++ b/lib/libvmod_proxy/vmod.vcc @@ -0,0 +1,125 @@ +#- +# Copyright (c) 2018 GANDI SAS +# All rights reserved. +# +# Author: Emmanuel Hocdet +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +$Module proxy 3 Varnish Standard Module +$ABI strict + +DESCRIPTION +=========== + +`vmod_proxy` contains functions to extract proxy-protocol-v2 TLV attributes +as described in https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt. + + +$Function STRING alpn() + +Description + Extract alpn attribute. +Example + set req.http.alpn = proxy.alpn(); + +$Function STRING authority() + +Description + Extract authority attribute. This corresponds to sni from a tls + connection. +Example + set req.http.authority = proxy.authority(); + +$Function BOOL is_ssl() + +Description + Report if proxy-protocol-v2 has ssl tlv. + +Example + | if (proxy.is_ssl()) { + | set req.http.ssl-version = proxy.ssl_version(); + | } + +$Function BOOL client_has_cert_sess() + +Description + Report if the client provided a certificate at least once over the TLS + session this connection belongs to. + +$Function BOOL client_has_cert_conn() + +Description + Report if the client provided a certificate over the current connection. + +$Function INT ssl_verify_result() + +Description + Report the SSL_get_verify_result from a TLS session. It only matters + if client_has_cert_sess() is true. Per default, value is set to 0 + (X509_V_OK). + +Example + | if (proxy.client_has_cert_sess() && proxy.ssl_verify_result() == 0) { + | set req.http.ssl-verify = "ok"; + | } + +$Function STRING ssl_version() + +Description + Extract ssl version attribute. +Example + set req.http.ssl-version = proxy.ssl_version(); + +$Function STRING client_cert_cn() + +Description + Extract the common name attribute of the client certificate's. +Example + set req.http.cert-cn = proxy.client_cert_cn(); + +$Function STRING ssl_cipher() + +Description + Extract the ssl cipher attribute. +Example + set req.http.ssl-cipher = proxy.ssl_cipher(); + +$Function STRING cert_sign() + +Description + Extract the certificate signature algorithm attribute. +Example + set req.http.cert-sign = proxy.cert_sign(); + +$Function STRING cert_key() + +Description + Extract the certificate key algorithm attribute. +Example + set req.http.cert-key = proxy.cert_key(); + +SEE ALSO +======== + +* :ref:`varnishd(1)` +* :ref:`vsl(7)` diff --git a/lib/libvmod_proxy/vmod_proxy.c b/lib/libvmod_proxy/vmod_proxy.c new file mode 100644 index 0000000..8c6973c --- /dev/null +++ b/lib/libvmod_proxy/vmod_proxy.c @@ -0,0 +1,158 @@ +/*- + * Copyright (c) 2018 GANDI SAS + * All rights reserved. + * + * Author: Emmanuel Hocdet + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "cache/cache.h" + +#include "vend.h" +#include "vcl.h" + +#include "proxy/cache_proxy.h" + +#include "vcc_if.h" + + +struct pp2_tlv_ssl { + uint8_t client; + uint32_t verify; +}__attribute__((packed)); + +#define PP2_CLIENT_SSL 0x01 +#define PP2_CLIENT_CERT_CONN 0x02 +#define PP2_CLIENT_CERT_SESS 0x04 + +static VCL_BOOL +tlv_ssl_flag(VRT_CTX, int flag) +{ + struct pp2_tlv_ssl *dst; + int len; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (VPX_tlv(ctx->req, PP2_TYPE_SSL, (void **)&dst, &len)) + return (0); + + return ((dst->client & flag) == flag); +} + +VCL_BOOL v_matchproto_(td_proxy_is_ssl) +vmod_is_ssl(VRT_CTX) +{ + return tlv_ssl_flag(ctx, PP2_CLIENT_SSL); +} + +VCL_BOOL v_matchproto_(td_proxy_client_has_cert_sess) +vmod_client_has_cert_sess(VRT_CTX) +{ + return tlv_ssl_flag(ctx, PP2_CLIENT_CERT_SESS); +} + +VCL_BOOL v_matchproto_(td_proxy_client_has_cert_conn) +vmod_client_has_cert_conn(VRT_CTX) +{ + return tlv_ssl_flag(ctx, PP2_CLIENT_CERT_CONN); +} + +/* return come from SSL_get_verify_result */ +VCL_INT v_matchproto_(td_proxy_ssl_verify_result) +vmod_ssl_verify_result(VRT_CTX) +{ + struct pp2_tlv_ssl *dst; + int len; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (VPX_tlv(ctx->req, PP2_TYPE_SSL, (void **)&dst, &len)) + return (0); /* X509_V_OK */ + + return (vbe32dec(&dst->verify)); +} + +static VCL_STRING +tlv_string(VRT_CTX, int tlv) +{ + char *dst, *d; + int len; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (VPX_tlv(ctx->req, tlv, (void **)&dst, &len)) + return (NULL); + if (!WS_Reserve(ctx->ws, len+1)) + return (NULL); + d = ctx->ws->f; + memcpy(d, dst, len); + d[len] = '\0'; + WS_Release(ctx->ws, len+1); + return (d); +} + +VCL_STRING v_matchproto_(td_proxy_alpn) +vmod_alpn(VRT_CTX) +{ + return tlv_string(ctx, PP2_TYPE_ALPN); +} + +VCL_STRING v_matchproto_(td_proxy_authority) +vmod_authority(VRT_CTX) +{ + return tlv_string(ctx, PP2_TYPE_AUTHORITY); +} + +VCL_STRING v_matchproto_(td_proxy_ssl_version) +vmod_ssl_version(VRT_CTX) +{ + return tlv_string(ctx, PP2_SUBTYPE_SSL_VERSION); +} + +VCL_STRING v_matchproto_(td_proxy_ssl_cipher) +vmod_ssl_cipher(VRT_CTX) +{ + return tlv_string(ctx, PP2_SUBTYPE_SSL_CIPHER); +} + +VCL_STRING v_matchproto_(td_proxy_cert_sign) +vmod_cert_sign(VRT_CTX) +{ + return tlv_string(ctx, PP2_SUBTYPE_SSL_SIG_ALG); +} + +VCL_STRING v_matchproto_(td_proxy_cert_key) +vmod_cert_key(VRT_CTX) +{ + return tlv_string(ctx, PP2_SUBTYPE_SSL_KEY_ALG); +} + +VCL_STRING v_matchproto_(td_proxy_client_cert_cn) +vmod_client_cert_cn(VRT_CTX) +{ + return tlv_string(ctx, PP2_SUBTYPE_SSL_CN); +} diff --git a/man/Makefile.am b/man/Makefile.am index e7c00df..0e35dc5 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -20,7 +20,8 @@ dist_man_MANS = \ vmod_std.3 \ vmod_vtc.3 \ vmod_blob.3 \ - vmod_unix.3 + vmod_unix.3 \ + vmod_proxy.3 CLEANFILES = $(dist_man_MANS) @@ -105,4 +106,7 @@ vmod_blob.3: $(top_builddir)/lib/libvmod_blob/vmod_blob.man.rst vmod_unix.3: $(top_builddir)/lib/libvmod_unix/vmod_unix.man.rst ${RST2MAN} $(RST2ANY_FLAGS) $? $@ +vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst + ${RST2MAN} $(RST2ANY_FLAGS) $? $@ + .NOPATH: $(dist_man_MANS) From manu at gandi.net Wed Mar 14 16:10:09 2018 From: manu at gandi.net (Emmanuel Hocdet) Date: Wed, 14 Mar 2018 16:10:09 -0000 Subject: [master] 5147961 Fix compilation warning for tlv_len_p in PROXY. Message-ID: <20180314161009.A8C1BA7E67@lists.varnish-cache.org> commit 5147961c55a703783323c8c4787eb145f3214cf0 Author: Emmanuel Hocdet Date: Wed Mar 14 16:10:15 2018 +0100 Fix compilation warning for tlv_len_p in PROXY. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index fee5ac2..196a273 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -254,6 +254,7 @@ int VPX_tlv(const struct req *req, int tlv, void **dst, int *len) { uintptr_t *p; + uint16_t *tlv_len_p; uint16_t l; char *d; int ssltlv = 0; @@ -265,9 +266,9 @@ VPX_tlv(const struct req *req, int tlv, void **dst, int *len) *dst = NULL; return (-1); } - d = (char *)*p; - l = *(uint16_t *)d; - d += 2; + tlv_len_p = (void *)(*p); + l = *tlv_len_p; + d = (char *)(tlv_len_p + 1); if (tlv > PP2_TYPE_SSL && tlv <= PP2_SUBTYPE_SSL_MAX) { ssltlv = tlv; @@ -484,7 +485,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) return (0); } if (tlv_len && WS_Reserve(req->sp->ws, 2 + tlv_len)) { - tlv_len_p = (uint16_t *)req->sp->ws->f; + tlv_len_p = (void *)req->sp->ws->f; *tlv_len_p = tlv_len; memcpy(req->sp->ws->f + 2, tlv_start, tlv_len); WS_Release(req->sp->ws, 2 + tlv_len);