[master] 9a267ea Introduce vmod-purge

Dridi Boukelmoune dridi.boukelmoune at gmail.com
Mon Sep 4 13:50:07 CEST 2017


commit 9a267ea8303f79ebca2aa0047f9c11ad3703f2b2
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 29 10:49:28 2017 +0200

    Introduce vmod-purge
    
    This module offers a finer-grained control over purges, in the old
    pre-vcl 4.0 style. For that, the signatures of VRT_purge and HSH_Purge
    were updated to inform about the number of rearmed objects.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 3549867..2a2435c 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -586,7 +586,7 @@ hsh_rush2(struct worker *wrk, struct rush *r)
  * Purge an entire objhead
  */
 
-void
+unsigned
 HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
 double keep)
 {
@@ -667,6 +667,7 @@ double keep)
 	} while (more);
 	WS_Release(wrk->aws, 0);
 	Pool_PurgeStat(n_tot);
+	return (n_tot);
 }
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e1e521b..ecd2374 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -538,7 +538,7 @@ VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize)
  * purges
  */
 
-void
+unsigned
 VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 {
 
@@ -548,12 +548,13 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		    "purge can only happen in vcl_hit{} or vcl_miss{}");
 		VRT_handling(ctx, VCL_RET_FAIL);
-		return;
+		return (0);
 	}
 
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
-	HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead, ttl, grace, keep);
+	return (HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
+	    ttl, grace, keep));
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 92f9a72..3d86c20 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -69,7 +69,7 @@ void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(struct req *, void *ctx, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *,
     struct ban *);
-void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
+unsigned HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
     double keep);
 void HSH_config(const char *h_arg);
 struct boc *HSH_RefBoc(const struct objcore *);
diff --git a/bin/varnishtest/tests/m00031.vtc b/bin/varnishtest/tests/m00031.vtc
new file mode 100644
index 0000000..2d40cb7
--- /dev/null
+++ b/bin/varnishtest/tests/m00031.vtc
@@ -0,0 +1,116 @@
+varnishtest "Test purge vmod"
+
+server s1 {
+	rxreq
+	expect req.http.Accept-Language == fr
+	txresp -hdr "Vary: Accept-Language" -body fr
+
+	rxreq
+	expect req.http.Accept-Language == fr
+	txresp -hdr "Vary: Accept-Language" -body fr
+
+	rxreq
+	expect req.http.Accept-Language == en
+	txresp -hdr "Vary: Accept-Language" -body en
+} -start
+
+varnish v1 -vcl+backend {
+	import purge;
+
+	sub vcl_recv {
+		if (req.method ~ "PURGE") {
+			return (hash);
+		}
+	}
+
+	sub my_purge {
+		if (req.method == "PURGE") {
+			set req.http.purged = purge.hard();
+		}
+		else {
+			set req.http.purged = purge.soft();
+		}
+		if (req.http.purged == "0") {
+			return (synth(404));
+		}
+		else {
+			return (synth(200));
+		}
+	}
+
+	sub vcl_hit {
+		if (req.method ~ "PURGE") { call my_purge; }
+	}
+
+	sub vcl_miss {
+		if (req.method ~ "PURGE") { call my_purge; }
+	}
+
+	sub vcl_synth {
+		if (req.method ~ "PURGE") {
+			set resp.http.purged = req.http.purged;
+			return (deliver);
+		}
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.X-Varnish-Be = bereq.xid;
+	}
+} -start
+
+logexpect l1 -v v1 -q Hit -i Hit {
+	expect * * Hit "^1002 -.+ 10.000000 0.000000$"
+} -start
+
+logexpect l2 -v v1 -q "Begin ~ bgfetch" {
+	expect * * BerespHeader "X-Varnish-Be: 1005"
+} -start
+
+logexpect l3 -v v1 -g raw -q "vxid == 0" {
+	expect * * ExpKill x=1008
+} -start
+
+client c1 {
+	txreq -hdr "Accept-Language: fr"
+	rxresp
+	expect resp.body == fr
+	expect resp.http.X-Varnish-Be == 1002
+
+	txreq -req SOFTPURGE
+	rxresp
+	expect resp.status == 200
+	expect resp.http.purged == 1
+
+	txreq -hdr "Accept-Language: fr"
+	rxresp
+	expect resp.body == fr
+	expect resp.http.X-Varnish == "1004 1002"
+	expect resp.http.X-Varnish-Be == 1002
+} -run
+
+# Wait until bgfetch triggered by 1004 is done
+logexpect l2 -wait
+
+client c1 {
+	txreq -hdr "Accept-Language: en"
+	rxresp
+	expect resp.body == en
+	expect resp.http.X-Varnish-Be == 1008
+
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 200
+	expect resp.http.purged == 2
+} -run
+
+# Wait until the expire thread kicks in
+logexpect l3 -wait
+
+client c1 {
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 404
+	expect resp.http.purged == 0
+} -run
+
+logexpect l1 -wait
diff --git a/bin/varnishtest/vmods.h b/bin/varnishtest/vmods.h
index 6f51d9e..a994bef 100644
--- a/bin/varnishtest/vmods.h
+++ b/bin/varnishtest/vmods.h
@@ -30,4 +30,5 @@
 VTC_VMOD(std)
 VTC_VMOD(debug)
 VTC_VMOD(directors)
+VTC_VMOD(purge)
 VTC_VMOD(vtc)
diff --git a/configure.ac b/configure.ac
index 4c58ea1..2743139 100644
--- a/configure.ac
+++ b/configure.ac
@@ -732,6 +732,7 @@ AC_CONFIG_FILES([
     lib/libvmod_debug/Makefile
     lib/libvmod_std/Makefile
     lib/libvmod_directors/Makefile
+    lib/libvmod_purge/Makefile
     lib/libvmod_vtc/Makefile
     man/Makefile
     varnishapi.pc
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 3978423..0de2f28 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -205,6 +205,10 @@ reference/vmod_directors.generated.rst: reference $(top_builddir)/lib/libvmod_di
 	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ || true
 BUILT_SOURCES += reference/vmod_directors.generated.rst
 
+reference/vmod_purge.generated.rst: reference $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
+	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@ || true
+BUILT_SOURCES += reference/vmod_purge.generated.rst
+
 reference/vmod_vtc.generated.rst: reference $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
 	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@ || true
 BUILT_SOURCES += reference/vmod_vtc.generated.rst
diff --git a/include/vrt.h b/include/vrt.h
index 3308634..9f1ad5e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -41,7 +41,7 @@
  *
  * 6.1 (unreleased):
  *	http_CollectHdrSep added
- *	VRT_purge modified (may fail a transaction)
+ *	VRT_purge modified (may fail a transaction, signature changed)
  * 6.0 (2017-03-15):
  *	VRT_hit_for_pass added
  *	VRT_ipcmp added
@@ -300,7 +300,7 @@ int VRT_re_match(VRT_CTX, const char *, void *re);
 const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *);
 
 void VRT_ban_string(VRT_CTX, const char *);
-void VRT_purge(VRT_CTX, double ttl, double grace, double keep);
+unsigned VRT_purge(VRT_CTX, double ttl, double grace, double keep);
 
 void VRT_count(VRT_CTX, unsigned);
 void VRT_synth(VRT_CTX, unsigned, const char *);
diff --git a/lib/Makefile.am b/lib/Makefile.am
index dbb4ad9..8981c65 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -8,4 +8,5 @@ SUBDIRS = \
 	libvmod_debug \
 	libvmod_std \
 	libvmod_directors \
+	libvmod_purge \
 	libvmod_vtc
diff --git a/lib/libvmod_purge/Makefile.am b/lib/libvmod_purge/Makefile.am
new file mode 100644
index 0000000..4a1098b
--- /dev/null
+++ b/lib/libvmod_purge/Makefile.am
@@ -0,0 +1,40 @@
+#
+
+AM_LDFLAGS  = $(AM_LT_LDFLAGS)
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/bin/varnishd \
+	-I$(top_builddir)/include
+
+vmoddir = $(pkglibdir)/vmods
+vmod_srcdir = $(top_srcdir)/lib/libvmod_purge
+vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
+vmod_LTLIBRARIES = libvmod_purge.la
+
+libvmod_purge_la_CFLAGS = \
+	@SAN_CFLAGS@
+
+libvmod_purge_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared \
+	@SAN_LDFLAGS@
+
+libvmod_purge_la_SOURCES = vmod_purge.c
+
+nodist_libvmod_purge_la_SOURCES = \
+	vcc_if.c \
+	vcc_if.h
+
+# BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
+$(libvmod_purge_la_OBJECTS): vcc_if.h
+
+vcc_if.h vmod_purge.rst vmod_purge.man.rst: vcc_if.c
+
+vcc_if.c: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
+EXTRA_DIST = vmod.vcc
+
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+	$(builddir)/vmod_purge.rst \
+	$(builddir)/vmod_purge.man.rst
diff --git a/lib/libvmod_purge/flint.lnt b/lib/libvmod_purge/flint.lnt
new file mode 100644
index 0000000..e69de29
diff --git a/lib/libvmod_purge/flint.sh b/lib/libvmod_purge/flint.sh
new file mode 100755
index 0000000..522e30d
--- /dev/null
+++ b/lib/libvmod_purge/flint.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+FLOPS='
+	-I../../bin/varnishd
+	*.c
+'
+
+. ../../tools/flint_skel.sh
diff --git a/lib/libvmod_purge/vmod.vcc b/lib/libvmod_purge/vmod.vcc
new file mode 100644
index 0000000..a935732
--- /dev/null
+++ b/lib/libvmod_purge/vmod.vcc
@@ -0,0 +1,104 @@
+#-
+# Copyright (c) 2017 Varnish Software AS
+# All rights reserved.
+#
+# Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+$Module purge 3 Varnish Purge Module
+$ABI strict
+
+DESCRIPTION
+===========
+
+`vmod_purge` contains functions that offer a finer-grained control than the
+``purge`` transition in ``vcl_recv``. The functions can only be called from
+``vcl_hit`` or ``vcl_miss`` and they should in general be used in both to
+ensure that all variants of a same object are taken care of.
+
+EXAMPLE
+=======
+
+::
+
+    sub vcl_recv {
+        if (req.method == "PURGE") {
+            if (client.ip !~ purge_acl) {
+                return (synth(405));
+            }
+            return (hash);
+        }
+    }
+
+    sub my_purge {
+        set req.http.purged = purge.hard();
+        if (req.http.purged == "0") {
+            return (synth(404));
+        }
+        else {
+            return (synth(200));
+        }
+    }
+
+    sub vcl_hit {
+        if (req.method == "PURGE") {
+            call my_purge;
+        }
+    }
+
+    sub vcl_miss {
+        if (req.method == "PURGE") {
+            call my_purge;
+        }
+    }
+
+    sub vcl_synth {
+        if (req.method == "PURGE") {
+            if (req.http.purged) {
+                set resp.http.purged = req.http.purged;
+            }
+            return (deliver);
+        }
+    }
+
+$Function INT hard()
+
+Description
+	This is equivalent to ``return(purge)`` but explicitly called from
+        ``vcl_hit`` and ``vcl_miss``. It returns the number of purged objects.
+Example
+	set req.http.purged = purge.hard();
+
+$Function INT soft(DURATION ttl = 0, DURATION grace = -1, DURATION keep = -1)
+
+Description
+        Sets the TTL, grace and keep. By default, TTL is set to 0 with grace
+        and keep periods left untouched. Setting a negative value for any of
+        the parameters leaves them untouched. Setting all three parameters to
+        0 is equivalent to a hard purge. It can only be called from ``vcl_hit``
+        or ``vcl_miss``. It returns the number of soft-purged objects.
+
+SEE ALSO
+========
+
+* :ref:`vcl(7)`
diff --git a/lib/libvmod_purge/vmod_purge.c b/lib/libvmod_purge/vmod_purge.c
new file mode 100644
index 0000000..7fba6b2
--- /dev/null
+++ b/lib/libvmod_purge/vmod_purge.c
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2017 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "math.h"
+
+#include "cache/cache.h"
+
+#include "vrt.h"
+
+#include "vcc_if.h"
+
+VCL_INT __match_proto__(td_purge_hard)
+vmod_hard(VRT_CTX)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	return (VRT_purge(ctx, 0, 0, 0));
+}
+
+VCL_INT __match_proto__(td_purge_soft)
+vmod_soft(VRT_CTX, VCL_DURATION ttl, VCL_DURATION grace, VCL_DURATION keep)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	if (ttl < 0)
+		ttl = NAN;
+	if (grace < 0)
+		grace = NAN;
+	if (keep < 0)
+		keep = NAN;
+	return (VRT_purge(ctx, ttl, grace, keep));
+}
diff --git a/man/Makefile.am b/man/Makefile.am
index fd7ee1a..73f6049 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -16,6 +16,7 @@ dist_man_MANS = \
 	vtc.7 \
 	varnishtop.1 \
 	vmod_directors.3 \
+	vmod_purge.3 \
 	vmod_std.3 \
 	vmod_vtc.3
 
@@ -87,6 +88,9 @@ varnishhist.1: \
 vmod_directors.3: $(top_builddir)/lib/libvmod_directors/vmod_directors.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
+vmod_purge.3: $(top_builddir)/lib/libvmod_purge/vmod_purge.man.rst
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@
+
 vmod_std.3: $(top_builddir)/lib/libvmod_std/vmod_std.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 



More information about the varnish-commit mailing list