r4119 - in branches/sky/esi_last_modified/varnish-cache/bin: varnishd varnishtest/tests
sky at projects.linpro.no
sky at projects.linpro.no
Wed Jun 17 16:16:59 CEST 2009
Author: sky
Date: 2009-06-17 16:16:59 +0200 (Wed, 17 Jun 2009)
New Revision: 4119
Added:
branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00017.vtc
branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00018.vtc
branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00019.vtc
branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00020.vtc
Modified:
branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache.h
branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_center.c
branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_esi.c
branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_response.c
Log:
Implement if-modified-since correctly for ESI
We don't actually know the last-modified value of an ESI
document until we have assembled it. At this point it is
too late to give a correct value.
This patch starts off by giving the first requestor
of an object the last-modified of the container.
This is safe because the worse that can happen is that
a fragment has a higher last-modified and we waste
some bandwidth. (Boy, trailers would be nice)
As we assemble the fragments, we keep track of the highest
last-modified value and store that in the object.
Next time the object is requested, we return that
last-modified.
If we get an IMS and we have never assembled the object
we assemble the object and then return it.
(A future optimisation is to assemble it and then
send not-modified if it isn't, purely to save bandwidth)
If we have a cached value then we base the IMS of it.
There is a race here that if a fragment is updated
and noone has requested the entire object the cached
value is stale and we will return not modified.
I think this is fine for now since otherwise we have to
a) keep reverse dependencies or
b) pull all fragments together if it isn't needed
A user can always get the latest version by doing
a force reload, since the browser doesn't send
if-modified-since on those.
An improvement would be to keep track of the time
any fragment expires an only cache the last-modified
that long. Which would solve the problem for anyone
who doesn't purge fragments to get them to update.
This should probably have a parameter
Modified: branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache.h
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache.h 2009-06-17 13:55:49 UTC (rev 4118)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache.h 2009-06-17 14:16:59 UTC (rev 4119)
@@ -324,6 +324,8 @@
double grace;
double last_modified;
+ double cache_last_modified;
+
double last_lru;
struct http http[1];
@@ -387,6 +389,9 @@
/* Acceptable grace period */
double grace;
+ /* Cache last-modified for ESI purposes */
+ double cache_last_modified;
+
enum step step;
unsigned cur_method;
unsigned handling;
Modified: branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_center.c 2009-06-17 13:55:49 UTC (rev 4118)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_center.c 2009-06-17 14:16:59 UTC (rev 4119)
@@ -976,6 +976,8 @@
AN(sp->director);
sp->disable_esi = 0;
+ if(!sp->esis)
+ sp->cache_last_modified = 0;
VCL_recv_method(sp);
if (sp->restarts >= params->max_restarts) {
Modified: branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_esi.c 2009-06-17 13:55:49 UTC (rev 4118)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_esi.c 2009-06-17 14:16:59 UTC (rev 4119)
@@ -836,6 +836,8 @@
sp->esis--;
sp->obj = obj;
+ sp->obj->cache_last_modified = sp->cache_last_modified;
+
/* Reset the workspace */
WS_Reset(sp->ws, ws_wm);
Modified: branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_response.c 2009-06-17 13:55:49 UTC (rev 4118)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishd/cache_response.c 2009-06-17 14:16:59 UTC (rev 4119)
@@ -57,7 +57,9 @@
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", lm);
http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish");
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "X-Varnish: %u", sp->xid);
- TIM_format(sp->obj->last_modified, lm);
+ TIM_format(sp->obj->cache_last_modified ?
+ sp->obj->cache_last_modified :
+ sp->obj->last_modified, lm);
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Last-Modified: %s", lm);
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
sp->doclose ? "close" : "keep-alive");
@@ -77,6 +79,13 @@
ims = TIM_parse(p);
if (ims > sp->t_req) /* [RFC2616 14.25] */
return (0);
+
+ /* If we don't know the correct last-modified it is better to just send
+ back the full page -- else we risk stale displays on the client side */
+ if (!sp->disable_esi && !VTAILQ_EMPTY(&sp->obj->esibits) && !sp->obj->cache_last_modified)
+ return (0);
+ if (sp->obj->cache_last_modified > ims)
+ return (0);
if (sp->obj->last_modified > ims) {
return (0);
}
@@ -137,12 +146,32 @@
struct storage *st;
unsigned u = 0;
char lenbuf[20];
+ char lm[64];
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
WRW_Reserve(sp->wrk, &sp->fd);
+ /* Assemble the highest modified in ESI chains */
+ if (sp->obj->last_modified > sp->cache_last_modified)
+ sp->cache_last_modified = sp->obj->last_modified;
+
+ /* We don't actually always have the correct last-modified
+ * If we have a cached version, then we use it
+ * Otherwise the worst case is we send a older header
+ * Which is fine
+ */
+
+ if (!sp->disable_esi &&
+ !VTAILQ_EMPTY(&sp->obj->esibits) &&
+ sp->esis == 0 &&
+ sp->obj->cache_last_modified) {
+ TIM_format(sp->obj->cache_last_modified, lm);
+ http_Unset(sp->wrk->resp, H_Last_Modified);
+ http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Last-Modified: %s", lm);
+ }
+
if (sp->disable_esi || !sp->esis)
sp->acct_req.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
Added: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00017.vtc
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00017.vtc (rev 0)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00017.vtc 2009-06-17 14:16:59 UTC (rev 4119)
@@ -0,0 +1,44 @@
+# $Id$
+
+test "Cache the last-modified of ESI pages so we can do the right thing with them"
+
+server s1 {
+ rxreq
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:33:01 GMT" -body {
+ <html>
+ Before include
+ <esi:include src="/body"/>
+ After include
+ }
+ rxreq
+ expect req.url == "/body"
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:34:01 GMT" -body {
+ Inside ESI
+ }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_fetch {
+ set beresp.ttl = 600s;
+ esi;
+ }
+} -start
+
+client c1 {
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:33:01 GMT"
+ delay 1
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:34:01 GMT"
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:33:01 GMT"
+ rxresp
+ expect resp.status == 200
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:34:01 GMT"
+ rxresp
+ expect resp.status == 304
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
Property changes on: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00017.vtc
___________________________________________________________________
Name: svn:keywords
+ Id
Added: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00018.vtc
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00018.vtc (rev 0)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00018.vtc 2009-06-17 14:16:59 UTC (rev 4119)
@@ -0,0 +1,37 @@
+# $Id: e00015.vtc 4068 2009-05-11 08:50:45Z sky $
+
+test "Test last-modified with intial ESI payload where there is no cached-last modified in which case we can't 304"
+
+server s1 {
+ rxreq
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:33:01 GMT" -body {
+ <html>
+ Before include
+ <esi:include src="/body"/>
+ After include
+ }
+ rxreq
+ expect req.url == "/body"
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:34:01 GMT" -body {
+ Inside ESI
+ }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_fetch {
+ set beresp.ttl = 600s;
+ esi;
+ }
+} -start
+
+client c1 {
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:33:01 GMT"
+ rxresp
+ expect resp.status == 200
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:34:01 GMT"
+ rxresp
+ expect resp.status == 304
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
Added: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00019.vtc
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00019.vtc (rev 0)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00019.vtc 2009-06-17 14:16:59 UTC (rev 4119)
@@ -0,0 +1,44 @@
+# $Id$
+
+test "Cache the last-modified of ESI pages so we can do the right thing with them"
+
+server s1 {
+ rxreq
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:34:01 GMT" -body {
+ <html>
+ Before include
+ <esi:include src="/body"/>
+ After include
+ }
+ rxreq
+ expect req.url == "/body"
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:33:01 GMT" -body {
+ Inside ESI
+ }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_fetch {
+ set beresp.ttl = 600s;
+ esi;
+ }
+} -start
+
+client c1 {
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:34:01 GMT"
+ delay 1
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:34:01 GMT"
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:33:01 GMT"
+ rxresp
+ expect resp.status == 200
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:34:01 GMT"
+ rxresp
+ expect resp.status == 304
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
Property changes on: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00019.vtc
___________________________________________________________________
Name: svn:keywords
+ Id
Added: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00020.vtc
===================================================================
--- branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00020.vtc (rev 0)
+++ branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00020.vtc 2009-06-17 14:16:59 UTC (rev 4119)
@@ -0,0 +1,50 @@
+# $Id$
+
+test "Nested last-modified with ESI"
+
+server s1 {
+ rxreq
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:34:01 GMT" -body {
+ <html>
+ Before include
+ <esi:include src="/body"/>
+ After include
+ }
+ rxreq
+ expect req.url == "/body"
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:33:01 GMT" -body {
+ <foo>Inside ESI
+ <esi:include src="/body2"/>
+ }
+ rxreq
+ expect req.url == "/body2"
+ txresp -hdr "Last-Modified: Mon, 18 May 2009 23:35:01 GMT" -body {
+ Inside ESI2
+ }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_fetch {
+ set beresp.ttl = 600s;
+ esi;
+ }
+} -start
+
+client c1 {
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:34:01 GMT"
+ delay 1
+ txreq
+ rxresp
+ expect resp.http.last-modified == "Mon, 18 May 2009 23:35:01 GMT"
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:34:01 GMT"
+ rxresp
+ expect resp.status == 200
+ txreq -hdr "If-Modified-Since: Mon, 18 May 2009 23:35:01 GMT"
+ rxresp
+ expect resp.status == 304
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
Property changes on: branches/sky/esi_last_modified/varnish-cache/bin/varnishtest/tests/e00020.vtc
___________________________________________________________________
Name: svn:keywords
+ Id
More information about the varnish-commit
mailing list