[master] 48821b4 Add the skeleton of a Conditional Fetch facility.
Poul-Henning Kamp
phk at varnish-cache.org
Wed Sep 11 16:44:44 CEST 2013
commit 48821b46b0714d23692174240042d38e88a4d69a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Wed Sep 11 14:40:22 2013 +0000
Add the skeleton of a Conditional Fetch facility.
This comes with a big apology to Geoff Simmons: I should have
tackled Conditional Fetch before even thinking about streaming
and therefore this has taken much longer than it should.
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 19c12c7..f23241c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -544,9 +544,10 @@ struct busyobj {
struct http *bereq0;
struct http *bereq;
struct http *beresp;
- struct objcore *ims_objcore;
+ struct object *ims_obj;
struct objcore *fetch_objcore;
struct object *fetch_obj;
+
struct exp exp;
struct http_conn htc;
@@ -918,7 +919,7 @@ enum vbf_fetch_mode_e {
VBF_BACKGROUND = 2,
};
void VBF_Fetch(struct worker *wrk, struct req *req,
- struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e);
+ struct objcore *oc, struct object *oldobj, enum vbf_fetch_mode_e);
/* cache_fetch_proc.c */
struct storage *VFP_GetStorage(struct busyobj *, ssize_t sz);
@@ -1127,6 +1128,7 @@ void VRY_Validate(const uint8_t *vary);
void VRY_Prep(struct req *);
enum vry_finish_flag { KEEP, DISCARD };
void VRY_Finish(struct req *req, enum vry_finish_flag);
+unsigned VRY_Len(const uint8_t *);
/* cache_vcl.c */
void VCL_Init(void);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b1219b5..a49ae95 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -60,6 +60,7 @@ vbf_release_req(struct busyobj *bo)
static enum fetch_step
vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
{
+ char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -86,6 +87,17 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
http_SetHeader(bo->bereq0, "Accept-Encoding: gzip");
}
}
+ if (bo->ims_obj != NULL) {
+ if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
+ http_PrintfHeader(bo->bereq0,
+ "If-Modified-Since: %s", p);
+ } else if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
+ http_PrintfHeader(bo->bereq0,
+ "If-None-Match: %s", p);
+ } else {
+ WRONG("Shouldn't have bo->ims_obj");
+ }
+ }
return (F_STP_STARTFETCH);
}
@@ -131,7 +143,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
static enum fetch_step
vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
{
- int i;
+ int i, do_ims;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -203,6 +215,14 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
AZ(bo->do_esi);
+ if (bo->ims_obj != NULL && bo->beresp->status == 304) {
+ bo->beresp->status = 200;
+ http_PrintfHeader(bo->beresp, "Content-Length: %jd",
+ bo->ims_obj->len);
+ do_ims = 1;
+ } else
+ do_ims = 0;
+
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (bo->do_esi)
@@ -211,7 +231,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
bo->fetch_objcore->flags |= OC_F_PASS;
if (wrk->handling == VCL_RET_DELIVER)
- return (F_STP_FETCH);
+ return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
if (wrk->handling == VCL_RET_RETRY) {
assert(bo->state == BOS_REQ_DONE);
bo->retries++;
@@ -239,7 +259,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
int varyl = 0;
struct object *obj;
-
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -441,6 +460,99 @@ vbf_stp_done(void)
/*--------------------------------------------------------------------
*/
+static enum fetch_step
+vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
+{
+ unsigned l;
+ uint16_t nhttp;
+ struct object *obj;
+ struct objiter *oi;
+ void *sp;
+ ssize_t sl, al, tl;
+ struct storage *st;
+
+ CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+ CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+ l = 0;
+ if (bo->ims_obj->vary != NULL)
+ l += VRY_Len(bo->ims_obj->vary);
+ l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
+
+ bo->stats = &wrk->stats;
+ obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
+ if (obj == NULL) {
+ (void)VFP_Error(bo, "Could not get storage");
+ VDI_CloseFd(&bo->vbc);
+ return (F_STP_DONE);
+ }
+ bo->stats = NULL;
+
+ AZ(bo->fetch_obj);
+ bo->fetch_obj = obj;
+
+ obj->gziped = bo->ims_obj->gziped;
+ obj->gzip_start = bo->ims_obj->gzip_start;
+ obj->gzip_last = bo->ims_obj->gzip_last;
+ obj->gzip_stop = bo->ims_obj->gzip_stop;
+
+ /* XXX: ESI */
+
+ if (bo->ims_obj->vary != NULL)
+ obj->vary = (void *)WS_Copy(obj->http->ws,
+ bo->ims_obj->vary, VRY_Len(bo->ims_obj->vary));
+
+ obj->vxid = bo->vsl->wid;
+
+ obj->http->logtag = HTTP_Obj;
+ /* XXX: we should have our own HTTP_A_CONDFETCH */
+ http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
+ http_CopyHome(obj->http);
+
+
+ if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
+ EXP_Insert(obj);
+ AN(obj->objcore->ban);
+ }
+
+ AZ(bo->ws_o->overflow);
+ VBO_setstate(bo, BOS_FETCHING);
+ HSH_Unbusy(&wrk->stats, obj->objcore);
+
+ st = NULL;
+ al = 0;
+
+ oi = ObjIterBegin(wrk, bo->ims_obj);
+ while (1 == ObjIter(oi, &sp, &sl)) {
+ while (sl > 0) {
+ if (st == NULL) {
+ st = VFP_GetStorage(bo, bo->ims_obj->len - al);
+ XXXAN(st);
+ }
+ tl = sl;
+ if (tl > st->space - st->len)
+ tl = st->space - st->len;
+ memcpy(st->ptr + st->len, sp, tl);
+ st->len += tl;
+ al += tl;
+ sp = (char *)sp + tl;
+ sl -= tl;
+ VBO_extend(bo, al);
+ if (st->len == st->space)
+ st = NULL;
+ }
+ }
+ assert(al == bo->ims_obj->len);
+ assert(obj->len == al);
+ if (bo->state != BOS_FAILED)
+ VBO_setstate(bo, BOS_FINISHED);
+ HSH_Complete(obj->objcore);
+ return (F_STP_DONE);
+}
+
+/*--------------------------------------------------------------------
+ */
+
static const char *
vbf_step_name(enum fetch_step stp)
{
@@ -490,8 +602,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
if (bo->state == BOS_FAILED)
assert(bo->fetch_objcore->flags & OC_F_FAILED);
- if (bo->ims_objcore != NULL)
- (void)HSH_DerefObjCore(&wrk->stats, &bo->ims_objcore);
+ if (bo->ims_obj != NULL)
+ (void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
VBO_DerefBusyObj(wrk, &bo);
THR_SetBusyobj(NULL);
@@ -502,14 +614,14 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
void
VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
- struct objcore *oldoc, enum vbf_fetch_mode_e mode)
+ struct object *oldobj, enum vbf_fetch_mode_e mode)
{
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
- CHECK_OBJ_ORNULL(oldoc, OBJCORE_MAGIC);
+ CHECK_OBJ_ORNULL(oldobj, OBJECT_MAGIC);
bo = VBO_GetBusyObj(wrk, req);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -528,9 +640,12 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
HSH_Ref(oc);
bo->fetch_objcore = oc;
- if (oldoc != NULL) {
- HSH_Ref(oldoc);
- bo->ims_objcore = oldoc;
+ if (oldobj != NULL) {
+ if (http_GetHdr(oldobj->http, H_Last_Modified, NULL) ||
+ http_GetHdr(oldobj->http, H_ETag, NULL)) {
+ HSH_Ref(oldobj->objcore);
+ bo->ims_obj = oldobj;
+ }
}
AZ(bo->req);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 02e7bcb..b0e7be7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -416,7 +416,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
switch (wrk->handling) {
case VCL_RET_DELIVER:
if (boc != NULL) {
- VBF_Fetch(wrk, req, boc, oc, VBF_BACKGROUND);
+ VBF_Fetch(wrk, req, boc, o, VBF_BACKGROUND);
} else {
(void)HTTP1_DiscardReqBody(req);// XXX: handle err
}
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 9dddf60..363e357 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -173,8 +173,8 @@ VRY_Create(struct busyobj *bo, struct vsb **psb)
/*
* Find length of a vary entry
*/
-static unsigned
-vry_len(const uint8_t *p)
+unsigned
+VRY_Len(const uint8_t *p)
{
unsigned l = vbe16dec(p);
@@ -189,7 +189,7 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
{
unsigned retval = 0;
- if (!memcmp(v1, v2, vry_len(v1))) {
+ if (!memcmp(v1, v2, VRY_Len(v1))) {
/* Same same */
retval = 0;
} else if (memcmp(v1 + 2, v2 + 2, v1[2] + 2)) {
@@ -331,8 +331,8 @@ VRY_Match(struct req *req, const uint8_t *vary)
}
if (i == 0) {
/* Same header, same contents*/
- vsp += vry_len(vsp);
- vary += vry_len(vary);
+ vsp += VRY_Len(vsp);
+ vary += VRY_Len(vary);
} else if (i == 2) {
/* Same header, different contents, cannot match */
return (0);
@@ -358,6 +358,6 @@ VRY_Validate(const uint8_t *vary)
while (vary[2] != 0) {
assert(strlen((const char*)vary+3) == vary[2]);
- vary += vry_len(vary);
+ vary += VRY_Len(vary);
}
}
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index b203503..e18b7f3 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -52,6 +52,7 @@ REQ_STEP(error, ERROR, (wrk, req))
FETCH_STEP(mkbereq, MKBEREQ, (wrk, bo))
FETCH_STEP(startfetch, STARTFETCH, (wrk, bo))
FETCH_STEP(fetchhdr, FETCHHDR, (wrk, bo))
+FETCH_STEP(condfetch, CONDFETCH, (wrk, bo))
FETCH_STEP(fetch, FETCH, (wrk, bo))
FETCH_STEP(done, DONE, ())
#endif
More information about the varnish-commit
mailing list