[experimental-ims] ffaedcc Merged conditional backend request feature

Geoff Simmons geoff at varnish-cache.org
Fri Jul 8 11:47:53 CEST 2011


commit ffaedccbbb526e54edbb092ebc50ce8d0dd62642
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Jun 2 21:47:07 2011 +0200

    Merged conditional backend request feature

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a875071..8515c5e 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -596,6 +596,7 @@ struct sess {
 	struct objcore		*objcore;
 	struct VCL_conf		*vcl;
 
+	struct object		*stale_obj;
 	/* The busy objhead we sleep on */
 	struct objhead		*hash_objhead;
 
@@ -753,6 +754,12 @@ void http_SetResp(struct http *to, const char *proto, int status,
 void http_FilterFields(struct worker *w, int fd, struct http *to,
     const struct http *fm, unsigned how);
 void http_FilterHeader(const struct sess *sp, unsigned how);
+
+/* Check if a refresh should be done */
+void http_CheckRefresh(struct sess *sp);
+/* Check if we got 304 response */
+void http_Check304(struct sess *sp);
+
 void http_PutProtocol(struct worker *w, int fd, const struct http *to,
     const char *protocol);
 void http_PutStatus(struct http *to, int status);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 23ad92f..6455985 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -558,6 +558,7 @@ cnt_fetch(struct sess *sp)
 		sp->wrk->age = 0;
 		EXP_Clr(&sp->wrk->exp);
 		sp->wrk->exp.ttl = RFC2616_Ttl(sp);
+		sp->wrk->exp.keep = params->default_keep;
 
 		/* pass from vclrecv{} has negative TTL */
 		if (sp->objcore == NULL)
@@ -638,7 +639,7 @@ cnt_fetchbody(struct sess *sp)
 	int i;
 	struct http *hp, *hp2;
 	char *b;
-	unsigned l, nhttp;
+	unsigned l, nhttp, stale_nhttp;
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
 
@@ -724,6 +725,10 @@ cnt_fetchbody(struct sess *sp)
 
 	l = http_EstimateWS(sp->wrk->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+        if (sp->stale_obj) {
+            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
+            nhttp += stale_nhttp;
+        }
 
 	/* Create Vary instructions */
 	if (sp->objcore != NULL) {
@@ -763,6 +768,7 @@ cnt_fetchbody(struct sess *sp)
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+        sp->obj->exp.keep = sp->wrk->exp.keep;
 
 	sp->wrk->storage_hint = NULL;
 
@@ -789,11 +795,29 @@ cnt_fetchbody(struct sess *sp)
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
-	http_FilterFields(sp->wrk, sp->fd, hp2, hp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(sp->wrk, sp->fd, hp2);
-
-	if (http_GetHdr(hp, H_Last_Modified, &b))
+        
+        http_FilterFields(sp->wrk, sp->fd, hp2, hp,
+            	pass ? HTTPH_R_PASS : HTTPH_A_INS);
+
+        /*
+	 * If we found a candidate for conditional backend request, attempt it
+         * now. If backend responds with 304, http_Check304() merges stale_obj
+         * into sp->obj, any other response is handled as usual. In either case,
+         * the stale_obj is no longer needed in the cache, so discard it.
+         */
+        if (sp->stale_obj) {
+            http_Check304(sp);
+            if (sp->wrk->beresp->status == 304)
+                assert(sp->obj->http->status == 200);
+	    EXP_Clr(&sp->stale_obj->exp);
+	    EXP_Rearm(sp->stale_obj);
+	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
+	    AZ(sp->stale_obj);
+        }
+        http_CopyHome(sp->wrk, sp->fd, hp2);
+
+	if (http_GetHdr(hp, H_Last_Modified, &b)
+            || http_GetHdr(sp->obj->http, H_Last_Modified, &b))
 		sp->obj->last_modified = TIM_parse(b);
 	else
 		sp->obj->last_modified = floor(sp->wrk->entered);
@@ -1089,6 +1113,8 @@ cnt_lookup(struct sess *sp)
 		sp->wrk->stats.cache_hitpass++;
 		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
 		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+                if (sp->stale_obj != NULL)
+                    (void)HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
 		sp->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
@@ -1149,6 +1175,13 @@ cnt_miss(struct sess *sp)
 	sp->wrk->connect_timeout = 0;
 	sp->wrk->first_byte_timeout = 0;
 	sp->wrk->between_bytes_timeout = 0;
+
+        /* If a candidate for a conditional backend request was found,
+         * add If-Modified-Since and/or If-None-Match to the bereq.
+         */
+        if (sp->stale_obj)
+                http_CheckRefresh(sp);
+
 	VCL_miss_method(sp);
 	switch(sp->handling) {
 	case VCL_RET_ERROR:
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 1063c6b..b3704b4 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -498,8 +498,13 @@ FetchBody(struct sess *sp)
 	AN(sp->director);
 	AssertObjPassOrBusy(sp->obj);
 
+        /* If we've freshened from another object and got a "Not Modified"
+         * response, then we have already duped the other object's body.
+         */
+        if (sp->wrk->beresp->status != 304)
+        	AZ(VTAILQ_FIRST(&sp->obj->store));
+
 	AZ(sp->wrk->vgz_rx);
-	AZ(VTAILQ_FIRST(&sp->obj->store));
 	switch (sp->wrk->body_status) {
 	case BS_NONE:
 		cls = 0;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 6fdff15..7f9eacd 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -305,13 +305,16 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	struct objcore *oc;
 	struct objcore *busy_oc, *grace_oc;
 	struct object *o;
-	double grace_ttl;
+	struct object *stale_o;       /* for freshness check */
+	double grace_ttl, stale_ttl;
+        char *p;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
 	AN(hash);
+        AZ(sp->stale_obj);
 	w = sp->wrk;
 
 	HSH_Prealloc(sp);
@@ -339,7 +342,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	assert(oh->refcnt > 0);
 	busy_oc = NULL;
 	grace_oc = NULL;
+	stale_o = NULL;       /* for freshness check */
 	grace_ttl = NAN;
+        stale_ttl = NAN;
 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
 		/* Must be at least our own ref + the objcore we examine */
 		assert(oh->refcnt > 1);
@@ -362,7 +367,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		o = oc_getobj(sp->wrk, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
-		if (o->exp.ttl <= 0.)
+		if (o->exp.ttl <= 0. && o->exp.grace <= 0.
+		    && o->exp.keep <= 0.)
 			continue;
 		if (BAN_CheckObject(o, sp))
 			continue;
@@ -384,6 +390,24 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 				grace_ttl = o->entered + o->exp.ttl;
 			}
 		}
+
+		/* At this point we know:
+		 * - o's TTL has elapsed
+		 * - o is not busy or banned,
+                 * - o is not a Vary match.
+                 * The object may be used for a conditional backend request if
+                 * - the keep time has not elapsed, and
+                 * - it has a Last-Modified and/or an ETag header.
+                 * If there are several, use the least expired one.
+                 */
+               if (EXP_Keep(sp, o) >= sp->t_req
+                   && (http_GetHdr(o->http, H_Last_Modified, &p)
+                       || http_GetHdr(o->http, H_ETag, &p)))
+                   if (stale_o == NULL || stale_ttl < o->entered + o->exp.ttl) {
+                       stale_o = o;
+                       stale_ttl = o->entered + o->exp.ttl;
+                   }
+
 	}
 
 	/*
@@ -451,6 +475,18 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		return (NULL);
 	}
 
+        /* If we're not serving a valid or graced object and we saved stale_o,
+	 * it is a candidate for the conditional backend request. */
+        AZ(oc && !sp->hash_always_miss);
+        AZ(busy_oc);
+        if (stale_o != NULL) {
+                AZ(stale_o->objcore->flags & OC_F_BUSY);
+		CHECK_OBJ_NOTNULL(stale_o->objcore, OBJCORE_MAGIC);
+                Lck_AssertHeld(&oh->mtx);
+                stale_o->objcore->refcnt++;
+                sp->stale_obj = stale_o;
+        }
+
 	/* Insert (precreated) objcore in objecthead */
 	oc = w->nobjcore;
 	w->nobjcore = NULL;
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 5d8050a..f0e1756 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -40,6 +40,8 @@
 
 #include "vct.h"
 #include "cache.h"
+#include "hash_slinger.h"
+#include "stevedore.h"
 
 #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
 #include "http_headers.h"
@@ -65,6 +67,10 @@ static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = {
 /*lint -restore */
 
 static enum VSL_tag_e
+
+void http_FilterMissingFields(struct worker *w, int fd, struct http *to,
+    const struct http *fm);
+
 http2shmlog(const struct http *hp, int t)
 {
 
@@ -848,6 +854,36 @@ http_FilterFields(struct worker *w, int fd, struct http *to,
 	}
 }
 
+/*---------------------------------------------------------------------
+ * Same as http_FilterFields but keep any existing hdrs in fm.
+ * Furthermore, before copy, check if fm already has that hdr, and if so
+ * do not copy.  Used for 304 refresh processing.
+ */
+
+/* XXX: uplex/GS: Also, don't filter according to the "how" bitmap in
+ *      http_headers.h. We only use this to copy from one cached object to
+ *      another, so if a header made into the first object, we want it.
+ */
+
+void
+http_FilterMissingFields(struct worker *w, int fd, struct http *to,
+    const struct http *fm)
+{
+	unsigned u;
+	unsigned hdrlen;
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) {
+		if (fm->hd[u].b == NULL)
+			continue;
+                hdrlen = strchr(fm->hd[u].b, ':') - fm->hd[u].b;
+                if (http_findhdr(to, hdrlen, fm->hd[u].b))
+                    continue;
+                http_copyheader(w, fd, to, fm, u);
+	}
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -869,6 +905,89 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 	http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Varnish: %u", sp->xid);
 }
 
+/*-------------------------------------------------------------------
+ * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
+ * found an expired object that qualifies for a refresh check,
+ * so add the appropriate headers.
+ */
+
+void
+http_CheckRefresh(struct sess *sp)
+{
+	struct object *freshen_obj;
+	struct http *obj_hp, *bereq_hp;
+	char *p;
+
+	freshen_obj = sp->stale_obj;
+	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
+	bereq_hp = sp->wrk->bereq;
+	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
+	obj_hp = freshen_obj->http;
+	CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC);
+
+	if(http_GetHdr(obj_hp, H_ETag, &p))
+		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p);
+
+	if(http_GetHdr(obj_hp, H_Last_Modified, &p))
+		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p);
+}
+
+/*-------------------------------------------------------------------
+ * Called after fetch and sp->freshen_obj present.  Check
+ * response and handle as needed.
+ */
+
+void
+http_Check304(struct sess *sp)
+{
+	struct object *o, *o_stale;
+	char *p;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	o_stale = sp->stale_obj;
+	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
+	o = sp->obj;
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+
+	if (sp->wrk->beresp->status != 304) {
+	    /*
+	     * IMS/INM headers may have been removed in VCL, so only count a
+	     * non-validating response if they were present in the request.
+	     */
+	    if (http_GetHdr(sp->wrk->bereq, H_If_Modified_Since, &p)
+		|| http_GetHdr(sp->wrk->bereq, H_If_None_Match, &p))
+		sp->wrk->stats.cond_not_validated++;
+	    return;
+	}
+
+	/* 
+	 * Copy headers we need from the stale object into the 304 response
+	 */
+	http_FilterMissingFields(sp->wrk, sp->fd, sp->obj->http,
+				 sp->stale_obj->http);
+
+	/*
+	 * Dup the stale object's storage in to the new object
+	 * and reset Content-Length from the size of the storage.
+	 */
+	STV_dup(sp, o_stale, o);
+	http_Unset(o->http, H_Content_Length);
+	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len);
+
+	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
+	http_SetH(o->http, HTTP_HDR_REQ, "GET");
+	http_copyh(o->http, sp->wrk->bereq, HTTP_HDR_URL);
+
+	/*
+	 * XXX: Are we copying all the necessary fields from stale_obj?
+	 *	Should we copy o_stale->hits into o->hits?
+	 */
+	o->response = 200;
+	o->gziped = o_stale->gziped;
+
+        AZ(o_stale->objcore->flags & OC_F_BUSY);
+}
+
 /*--------------------------------------------------------------------
  * This function copies any header fields which reference foreign
  * storage into our own WS.
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 65b7522..07adeac 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -112,6 +112,10 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 		hp = sp->obj->http;
 		break;
+        case HDR_STALE_OBJ:
+		CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC);
+		hp = sp->stale_obj->http;
+		break;
 	default:
 		INCOMPL();
 	}
@@ -126,6 +130,11 @@ VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
 	struct http *hp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+        if (where == HDR_STALE_OBJ && sp->stale_obj == NULL) {
+	    WSP(sp, SLT_VCL_error,
+		"stale_obj does not exist (reading header %s)", n);
+            return NULL;
+	}
 	hp = vrt_selecthttp(sp, where);
 	if (!http_GetHdr(hp, n, &p))
 		return (NULL);
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 297151c..3396726 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -40,6 +40,9 @@
 #include "cache_backend.h"
 #include "hash_slinger.h"
 
+#define ILLEGAL_R(sess, obj, field)                                         \
+WSP(sess, SLT_VCL_error, "%s does not exist (reading field %s)", obj, field)
+
 static char vrt_hostname[255] = "";
 
 /*--------------------------------------------------------------------*/
@@ -61,7 +64,7 @@ vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld,
 	va_end(ap);
 }
 
-#define VRT_DO_HDR(obj, hdr, http, fld)				\
+#define VRT_DO_HDR_l(obj, hdr, cont, http, fld)			\
 void								\
 VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...)	\
 {								\
@@ -69,53 +72,74 @@ VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...)	\
 								\
 	va_start(ap, p);					\
 	vrt_do_string(sp->wrk, sp->fd,				\
-	    http, fld, #obj "." #hdr, p, ap);			\
+	    cont->http, fld, #obj "." #hdr, p, ap);		\
 	va_end(ap);						\
-}								\
-								\
+}
+
+#define VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)	\
 const char *							\
 VRT_r_##obj##_##hdr(const struct sess *sp)			\
 {								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
-	return (http->hd[fld].b);				\
-}
-
-VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
-VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
-VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
-VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
-VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   proto,	sp->obj->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   response,	sp->obj->http,		HTTP_HDR_RESPONSE)
-VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
-VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
-VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp,  response,	sp->wrk->beresp,	HTTP_HDR_RESPONSE)
+        if (!nullable || cont != NULL) {			\
+            CHECK_OBJ_NOTNULL(cont->http, HTTP_MAGIC);		\
+            return (cont->http->hd[fld].b);			\
+        }							\
+        ILLEGAL_R(sp, #obj, #hdr);				\
+        return(NULL);                                           \
+}                                                               \
+
+#define VRT_DO_HDR(obj, hdr, cont, http, fld, nullable)		\
+VRT_DO_HDR_l(obj, hdr, cont, http, fld)				\
+VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)		\
+
+VRT_DO_HDR(req,         request,    sp,             http,	HTTP_HDR_REQ,	    0)
+VRT_DO_HDR(req,         url,        sp,             http,	HTTP_HDR_URL,	    0)
+VRT_DO_HDR(req,         proto,      sp,             http,	HTTP_HDR_PROTO,	    0)
+VRT_DO_HDR(bereq,       request,    sp->wrk,        bereq,	HTTP_HDR_REQ,	    0)
+VRT_DO_HDR(bereq,       url,        sp->wrk,        bereq,	HTTP_HDR_URL,	    0)
+VRT_DO_HDR(bereq,       proto,      sp->wrk,        bereq,	HTTP_HDR_PROTO,	    0)
+VRT_DO_HDR(obj,         proto,      sp->obj,        http,	HTTP_HDR_PROTO,	    0)
+VRT_DO_HDR(obj,         response,   sp->obj,        http,	HTTP_HDR_RESPONSE,  0)
+VRT_DO_HDR(resp,        proto,      sp->wrk,        resp,	HTTP_HDR_PROTO,	    0)
+VRT_DO_HDR(resp,        response,   sp->wrk,        resp,	HTTP_HDR_RESPONSE,  0)
+VRT_DO_HDR(beresp,      proto,      sp->wrk,        beresp,	HTTP_HDR_PROTO,	    0)
+VRT_DO_HDR(beresp,      response,   sp->wrk,        beresp,	HTTP_HDR_RESPONSE,  0)
+VRT_DO_HDR_r(stale_obj,   proto,      sp->stale_obj,  http,	HTTP_HDR_PROTO,	    1)
+VRT_DO_HDR_r(stale_obj,   response,   sp->stale_obj,  http,	HTTP_HDR_RESPONSE,  1)
 
 /*--------------------------------------------------------------------*/
 
-#define VRT_DO_STATUS(obj, http)				\
+#define VRT_DO_STATUS_l(obj, cont, http)			\
 void								\
 VRT_l_##obj##_status(const struct sess *sp, int num)		\
 {								\
 								\
 	assert(num >= 100 && num <= 999);			\
-	http->status = num;					\
-}								\
-								\
+	cont->http->status = num;				\
+}
+
+#define VRT_DO_STATUS_r(obj, cont, http, nullable)		\
 int								\
 VRT_r_##obj##_status(const struct sess *sp)			\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(http->status);					\
+        if (nullable && cont == NULL) {				\
+            ILLEGAL_R(sp, #obj, "status");			\
+            return (503);                                       \
+        }                                                       \
+	return(cont->http->status);				\
 }
 
-VRT_DO_STATUS(obj, sp->obj->http)
-VRT_DO_STATUS(beresp, sp->wrk->beresp)
-VRT_DO_STATUS(resp, sp->wrk->resp)
+#define VRT_DO_STATUS(obj, cont, http, nullable)		\
+VRT_DO_STATUS_l(obj, cont, http)				\
+VRT_DO_STATUS_r(obj, cont, http, nullable)			\
+
+VRT_DO_STATUS(obj,          sp->obj,        http,	0)
+VRT_DO_STATUS(beresp,       sp->wrk,        beresp,	0)
+VRT_DO_STATUS(resp,	    sp->wrk,        resp,	0)
+VRT_DO_STATUS_r(stale_obj,  sp->stale_obj,  http,	1)
 
 /*--------------------------------------------------------------------*/
 
@@ -362,37 +386,48 @@ VRT_r_req_restarts(const struct sess *sp)
 
 /*--------------------------------------------------------------------*/
 
-#define VRT_DO_EXP(which, exp, fld, extra)			\
-								\
-void __match_proto__()						\
-VRT_l_##which##_##fld(struct sess *sp, double a)		\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	EXP_Set_##fld(&exp, a);					\
-	extra;							\
-}								\
-								\
-double __match_proto__()					\
-VRT_r_##which##_##fld(struct sess *sp)				\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(EXP_Get_##fld(&exp));				\
+#define VRT_DO_EXP_l(which, cont, fld, extra)			    \
+void __match_proto__()						    \
+VRT_l_##which##_##fld(struct sess *sp, double a)		    \
+{								    \
+								    \
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			    \
+	EXP_Set_##fld(&cont->exp, a);				    \
+	extra;							    \
+}
+
+#define VRT_DO_EXP_r(which, cont, fld, nullable)		    \
+double __match_proto__()					    \
+VRT_r_##which##_##fld(struct sess *sp)				    \
+{								    \
+								    \
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			    \
+	if (nullable && cont == NULL) {				    \
+	    ILLEGAL_R(sp, #which, #fld);			    \
+	    return (-1);					    \
+	}							    \
+	return(EXP_Get_##fld(&cont->exp));			    \
 }
 
-VRT_DO_EXP(req, sp->exp, ttl, )
-VRT_DO_EXP(req, sp->exp, grace, )
-VRT_DO_EXP(req, sp->exp, keep, )
-VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj))
-VRT_DO_EXP(obj, sp->obj->exp, ttl,
+#define VRT_DO_EXP(which, cont, fld, nullable, extra)		    \
+VRT_DO_EXP_l(which, cont, fld, extra)				    \
+VRT_DO_EXP_r(which, cont, fld, nullable)			    \
+
+VRT_DO_EXP(req, sp, ttl, 0, )
+VRT_DO_EXP(req, sp, grace, 0, )
+VRT_DO_EXP(req, sp, keep, 0, )
+VRT_DO_EXP(obj, sp->obj, grace, 0, EXP_Rearm(sp->obj))
+VRT_DO_EXP(obj, sp->obj, ttl, 0,
 	   EXP_Rearm(sp->obj);
 	   WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req))
-VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj))
-VRT_DO_EXP(beresp, sp->wrk->exp, grace, )
-VRT_DO_EXP(beresp, sp->wrk->exp, ttl,
+VRT_DO_EXP(obj, sp->obj, keep, 0, EXP_Rearm(sp->obj))
+VRT_DO_EXP(beresp, sp->wrk, grace, 0, )
+VRT_DO_EXP(beresp, sp->wrk, ttl, 0,
 	   WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req))
-VRT_DO_EXP(beresp, sp->wrk->exp, keep, )
+VRT_DO_EXP(beresp, sp->wrk, keep, 0, )
+VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 1)
+VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 1)
+VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 1)
 
 /*--------------------------------------------------------------------
  * req.xid
@@ -493,6 +528,8 @@ VRT_r_server_port(struct sess *sp)
 
 /*--------------------------------------------------------------------*/
 
+/* XXX: uplex/GS: a nice macro would eliminate the repetition here ... */
+
 int
 VRT_r_obj_hits(const struct sess *sp)
 {
@@ -502,6 +539,19 @@ VRT_r_obj_hits(const struct sess *sp)
 	return (sp->obj->hits);
 }
 
+int
+VRT_r_stale_obj_hits(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+        if (sp->stale_obj == NULL) {
+            ILLEGAL_R(sp, "stale_obj", "hits");
+            return (0);
+        }
+	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
+	return (sp->stale_obj->hits);
+}
+
 double
 VRT_r_obj_lastuse(const struct sess *sp)
 {
@@ -511,6 +561,19 @@ VRT_r_obj_lastuse(const struct sess *sp)
 	return (TIM_real() - sp->obj->last_use);
 }
 
+double
+VRT_r_stale_obj_lastuse(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+        if (sp->stale_obj == NULL) {
+            ILLEGAL_R(sp, "stale_obj", "lastuse");
+            return (0);
+        }
+	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
+	return (TIM_real() - sp->stale_obj->last_use);
+}
+
 unsigned
 VRT_r_req_backend_healthy(const struct sess *sp)
 {
@@ -519,3 +582,9 @@ VRT_r_req_backend_healthy(const struct sess *sp)
 	return (VDI_Healthy(sp->director, sp));
 }
 
+unsigned
+VRT_r_stale_obj(const struct sess *sp)
+{
+        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+        return (sp->stale_obj != NULL);
+}
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 26fcb31..f811198 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -582,13 +582,13 @@ static const struct parspec input_parspec[] = {
 		"10", "seconds" },
 	{ "default_keep", tweak_timeout_double, &master.default_keep,
 		0, UINT_MAX,
-		"Default keep period.  We will keep a useless object "
+		"Default keep period.  We will keep a stale object "
 		"around this long, making it available for conditional "
 		"backend fetches.  "
 		"That means that the object will be removed from the "
-		"cache at the end of ttl+grace+keep.",
+		"cache at the end of ttl+max(grace,keep).",
 		DELAYED_EFFECT,
-		"0", "seconds" },
+		"10", "seconds" },
 	{ "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0,
 		"Idle timeout for persistent sessions. "
 		"If a HTTP request has not been received in this many "
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 413e0b6..7c1665e 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -370,6 +370,30 @@ STV_trim(struct storage *st, size_t size)
 		st->stevedore->trim(st, size);
 }
 
+/*
+ * Duplicate the object storage (HTML body) from src into target, using a
+ * stevedore-specific dup method for src's stevedore.
+ *
+ * Currently, every method just copies storage from one object to the other,
+ * but this method of encapsulation opens the path to future techniques of
+ * sharing storage together with reference counting.
+ */
+void
+STV_dup(const struct sess *sp, struct object *src, struct object *target)
+{
+        struct stevedore *stv;
+
+        CHECK_OBJ_NOTNULL(src, OBJECT_MAGIC);
+        CHECK_OBJ_NOTNULL(target, OBJECT_MAGIC);
+        CHECK_OBJ_NOTNULL(src->objstore, STORAGE_MAGIC);
+        CHECK_OBJ_NOTNULL(src->objstore->stevedore, STEVEDORE_MAGIC);
+
+        stv = src->objstore->stevedore;
+        AN(stv->dup);
+        
+        stv->dup(sp, src, target);
+}
+
 void
 STV_free(struct storage *st)
 {
@@ -586,3 +610,24 @@ VRT_Stv_##nm(const char *nm)			\
 
 #include "vrt_stv_var.h"
 #undef VRTSTVVAR
+
+/*
+ * Default object store dup just copies the storage.
+ */
+void
+default_dup(const struct sess *sp, struct object *src, struct object *target)
+{
+        struct storage *st, *st2;
+        unsigned cl;
+
+        VTAILQ_FOREACH(st2, &src->store, list) {
+		cl = st2->len;
+		st = STV_alloc(sp, cl);
+		XXXAN(st);
+                assert(st->space >= cl);
+		VTAILQ_INSERT_TAIL(&target->store, st, list);
+		st->len = cl;
+		target->len += cl;
+		memcpy(st->ptr, st2->ptr, cl);
+	}
+}
diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h
index 5a5cc4b..1c0d2ee 100644
--- a/bin/varnishd/stevedore.h
+++ b/bin/varnishd/stevedore.h
@@ -39,6 +39,7 @@ struct stv_objsecrets;
 typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
 typedef void storage_open_f(const struct stevedore *);
 typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
+typedef void storage_dup_f(const struct sess *sp, struct object *src, struct object *target);
 typedef void storage_trim_f(struct storage *, size_t size);
 typedef void storage_free_f(struct storage *);
 typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
@@ -70,6 +71,7 @@ struct stevedore {
 	storage_open_f		*open;		/* called by cache process */
 	storage_alloc_f		*alloc;		/* --//-- */
 	storage_trim_f		*trim;		/* --//-- */
+        storage_dup_f           *dup;           /* --//-- */
 	storage_free_f		*free;		/* --//-- */
 	storage_close_f		*close;		/* --//-- */
 	storage_allocobj_f	*allocobj;	/* --//-- */
@@ -93,6 +95,7 @@ struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
 struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
     struct exp *, unsigned nhttp);
 struct storage *STV_alloc(const struct sess *sp, size_t size);
+void STV_dup(const struct sess *sp, struct object *src, struct object *target);
 void STV_trim(struct storage *st, size_t size);
 void STV_free(struct storage *st);
 void STV_open(void);
@@ -117,3 +120,6 @@ extern const struct stevedore smp_stevedore;
 #ifdef HAVE_LIBUMEM
 extern const struct stevedore smu_stevedore;
 #endif
+
+/* Default dup method */
+void default_dup(const struct sess *sp, struct object *src, struct object *target);
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 27cc86f..84e56ce 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -551,6 +551,7 @@ const struct stevedore smf_stevedore = {
 	.alloc	=	smf_alloc,
 	.trim	=	smf_trim,
 	.free	=	smf_free,
+        .dup    =       default_dup,
 };
 
 #ifdef INCLUDE_TEST_DRIVER
diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c
index 12a390f..2e9b81f 100644
--- a/bin/varnishd/storage_malloc.c
+++ b/bin/varnishd/storage_malloc.c
@@ -221,4 +221,5 @@ const struct stevedore sma_stevedore = {
 	.trim	=	sma_trim,
 	.var_free_space =	sma_free_space,
 	.var_used_space =	sma_used_space,
+        .dup    =       default_dup,
 };
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index a4ba598..4cc0eca 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -567,6 +567,7 @@ const struct stevedore smp_stevedore = {
 	.allocobj =	smp_allocobj,
 	.free	=	smp_free,
 	.trim	=	smp_trim,
+        .dup    =       default_dup,
 };
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c
index 24479e3..3ddab87 100644
--- a/bin/varnishd/storage_synth.c
+++ b/bin/varnishd/storage_synth.c
@@ -67,6 +67,7 @@ static struct stevedore sms_stevedore = {
 	.magic	=	STEVEDORE_MAGIC,
 	.name	=	"synth",
 	.free	=	sms_free,
+        .dup    =       default_dup,
 };
 
 struct vsb *
diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c
index 6e28a94..474f809 100644
--- a/bin/varnishd/storage_umem.c
+++ b/bin/varnishd/storage_umem.c
@@ -163,6 +163,7 @@ const struct stevedore smu_stevedore = {
 	.alloc	=	smu_alloc,
 	.free	=	smu_free,
 	.trim	=	smu_trim,
+        .dup    =       default_dup,
 };
 
 #endif /* HAVE_UMEM_H */
diff --git a/include/vrt.h b/include/vrt.h
index cfafe75..9d9ce16 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -159,7 +159,7 @@ int VRT_rewrite(const char *, const char *);
 void VRT_error(struct sess *, unsigned, const char *);
 int VRT_switch_config(const char *);
 
-enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
+enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_STALE_OBJ, HDR_BEREQ, HDR_BERESP };
 char *VRT_GetHdr(const struct sess *, enum gethdr_e where, const char *);
 void VRT_SetHdr(const struct sess *, enum gethdr_e where, const char *,
     const char *, ...);
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 0289b81..534501a 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -155,6 +155,8 @@ VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs")
 VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations")
 VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
 
+VSC_F(cond_not_validated,	uint64_t, 1, 'a', "Non-validating responses")
+
 #endif
 
 /**********************************************************************/
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 5407a51..28375fa 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -427,6 +427,66 @@ sp_variables = (
 		( ),
 		'const struct sess *'
 	),
+	('stale_obj',
+		'BOOL',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.proto',
+		'STRING',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.status',
+		'INT',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.response',
+		'STRING',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.hits',
+		'INT',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.http.',
+		'HDR_STALE_OBJ',
+ 		( 'error', 'miss', 'fetch',),
+		( ),		# XXX ?
+		'const struct sess *'
+	),
+	('stale_obj.ttl',
+		'DURATION',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'struct sess *'
+	),
+	('stale_obj.grace',
+		'DURATION',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'struct sess *'
+	),
+	('stale_obj.lastuse',
+		'DURATION',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'const struct sess *'
+	),
+	('stale_obj.keep',
+		'DURATION',
+ 		( 'error', 'miss', 'fetch',),
+		( ),
+		'struct sess *'
+	),
 	('resp.proto',
 		'STRING',
 		( 'deliver',),



More information about the varnish-commit mailing list