r4351 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
phk at projects.linpro.no
phk at projects.linpro.no
Mon Nov 16 13:44:03 CET 2009
Author: phk
Date: 2009-11-16 13:44:03 +0100 (Mon, 16 Nov 2009)
New Revision: 4351
Added:
trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc
Modified:
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_esi.c
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/cache_response.c
trunk/varnish-cache/bin/varnishd/stevedore.c
Log:
Rework ESI storage allocation.
Previously we stored the esi-metadata in the object workspace, but since we
are trying to make that a snug fit and we cannot preestimate how much space
ESI parsing will need, this no longer works.
Instead parse the ESI metadata into the workers workspace and when
done, allocate a storage object and move it all into that.
Beware that this may increase the memory cost for ESI objects by the stevedores
granularity.
Fixes #578
Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2009-11-16 10:58:30 UTC (rev 4350)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2009-11-16 12:44:03 UTC (rev 4351)
@@ -89,7 +89,7 @@
struct objhead;
struct objcore;
struct workreq;
-struct esi_bit;
+struct esidata;
struct vrt_backend;
struct cli_proto;
struct ban;
@@ -342,7 +342,7 @@
VTAILQ_HEAD(, storage) store;
- VTAILQ_HEAD(, esi_bit) esibits;
+ struct esidata *esidata;
double last_use;
Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-11-16 10:58:30 UTC (rev 4350)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-11-16 12:44:03 UTC (rev 4351)
@@ -54,9 +54,8 @@
#include "vrt.h"
#include "vcl.h"
#include "cache.h"
+#include "stevedore.h"
-#define NDEFELEM 10
-
/*--------------------------------------------------------------------*/
struct esi_bit {
@@ -65,11 +64,8 @@
txt verbatim;
txt host;
txt include;
- int free_this;
};
-VTAILQ_HEAD(esibithead, esi_bit);
-
struct esi_ptr {
const char *p;
const char *e;
@@ -87,14 +83,24 @@
txt t;
struct esi_bit *eb;
- struct esi_bit *ebl; /* list of */
- int neb;
int remflg; /* inside <esi:remove> </esi:remove> */
int incmt; /* inside <!--esi ... --> comment */
+
+ unsigned space; /* ... needed */
+
+ VTAILQ_HEAD(, esi_bit) esibits;
+
};
+struct esidata {
+ unsigned magic;
+#define ESIDATA_MAGIC 0x7255277f
+ VTAILQ_HEAD(, esi_bit) esibits;
+ struct storage *storage;
+};
+
/*--------------------------------------------------------------------
- * Move the parse-pointer forward.
+ * Move an esi_ptr one char forward
*/
static void
@@ -118,6 +124,10 @@
return;
}
+/*--------------------------------------------------------------------
+ * Consume one input character.
+ */
+
static void
N(struct esi_work *ew)
{
@@ -209,18 +219,12 @@
esi_addbit(struct esi_work *ew, const char *verbatim, unsigned len)
{
- if (ew->neb == 0) {
- ew->ebl = calloc(NDEFELEM, sizeof(struct esi_bit));
- XXXAN(ew->ebl);
- ew->neb = NDEFELEM;
- ew->ebl->free_this = 1;
- }
- ew->eb = ew->ebl;
- ew->ebl++;
- ew->neb--;
+ ew->space += sizeof(*ew->eb);
+ ew->eb = (void*)WS_Alloc(ew->sp->wrk->ws, sizeof *ew->eb);
+ AN(ew->eb);
+ memset(ew->eb, 0, sizeof *ew->eb);
-
- VTAILQ_INSERT_TAIL(&ew->sp->obj->esibits, ew->eb, list);
+ VTAILQ_INSERT_TAIL(&ew->esibits, ew->eb, list);
if (verbatim != NULL) {
ew->eb->verbatim.b = TRUST_ME(verbatim);
if (len > 0)
@@ -231,8 +235,10 @@
Tlen(ew->eb->verbatim),
Tlen(ew->eb->verbatim),
ew->eb->verbatim.b);
- } else
- ew->eb->verbatim.b = ew->eb->verbatim.e = (void*)ew->eb;
+ } else {
+ AN(ew->s.p);
+ ew->eb->verbatim.b = ew->eb->verbatim.e = TRUST_ME(ew->s.p);
+ }
}
/*--------------------------------------------------------------------*/
@@ -381,12 +387,12 @@
if ( val.b != val.e ) {
s = Tlen(val) + 1;
- c = WS_Alloc(ws, s);
+ c = WS_Alloc(ew->sp->wrk->ws, s);
XXXAN(c);
memcpy(c, val.b, Tlen(val));
val.b = c;
val.e = val.b + s;
- *val.e = '\0';
+ val.e[-1] = '\0';
}
if (Tlen(val) > 7 && !memcmp(val.b, "http://", 7)) {
@@ -428,7 +434,7 @@
if (q != NULL)
tag.e = q + 1;
- u = WS_Reserve(ws, 0);
+ u = WS_Reserve(ew->sp->wrk->ws, 0);
v = snprintf(ws->f, u - 1, "%.*s%.*s",
pdiff(tag.b, tag.e), tag.b,
pdiff(val.b, val.e), val.b);
@@ -436,8 +442,12 @@
xxxassert(v < u);
eb->include.b = ws->f;
eb->include.e = ws->f + v;
- WS_Release(ws, v);
+ WS_Release(ew->sp->wrk->ws, v);
}
+ if (eb->include.b != NULL)
+ ew->space += Tlen(eb->include);
+ if (eb->host.b != NULL)
+ ew->space += Tlen(eb->host);
}
}
@@ -654,6 +664,11 @@
ESI_Parse(struct sess *sp)
{
struct esi_work *ew, eww[1];
+ struct esi_bit *eb;
+ struct esidata *ed;
+ struct storage *st;
+ unsigned u;
+ char *hack;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
@@ -680,13 +695,19 @@
if (!contain_esi(sp->obj))
return;
+ /* XXX: debugging hack */
+ hack = sp->wrk->ws->f;
+
VSL_stats->esi_parse++;
/* XXX: only if GET ? */
ew = eww;
memset(eww, 0, sizeof eww);
+ VTAILQ_INIT(&ew->esibits);
ew->sp = sp;
ew->off = 1;
+ ew->space += sizeof(struct esidata);
+
ew->p.st = VTAILQ_FIRST(&sp->obj->store);
AN(ew->p.st);
ew->p.p = (char *)ew->p.st->ptr;
@@ -751,6 +772,48 @@
if (ew->incmt)
esi_error(ew, ew->t.e, -1,
"ESI 1.0 unterminated <!--esi comment");
+
+ st = STV_alloc(sp, ew->space, sp->obj->objcore);
+ AN(st);
+ assert(st->space >= ew->space);
+
+ ed = (void*)st->ptr;
+ st->len += sizeof(*ed);
+ memset(ed, 0, sizeof *ed);
+ ed->magic = ESIDATA_MAGIC;
+ ed->storage = st;
+ VTAILQ_INIT(&ed->esibits);
+
+ while (!VTAILQ_EMPTY(&ew->esibits)) {
+ eb = VTAILQ_FIRST(&ew->esibits);
+ VTAILQ_REMOVE(&ew->esibits, eb, list);
+ memcpy(st->ptr + st->len, eb, sizeof *eb);
+ eb = (void*)(st->ptr + st->len);
+ st->len += sizeof(*eb);
+
+ if (eb->include.b != NULL) {
+ u = Tlen(eb->include);
+ memcpy(st->ptr + st->len, eb->include.b, u);
+ eb->include.b = (void*)(st->ptr + st->len);
+ eb->include.e = eb->include.b + u;
+ st->len += u;
+ }
+ if (eb->host.b != NULL) {
+ u = Tlen(eb->host);
+ memcpy(st->ptr + st->len, eb->host.b, u);
+ eb->host.b = (void*)(st->ptr + st->len);
+ eb->host.e = eb->host.b + u;
+ st->len += u;
+ }
+
+ VTAILQ_INSERT_TAIL(&ed->esibits, eb, list);
+ }
+
+ assert(st->len < st->space);
+ assert(st->len == ew->space);
+ sp->obj->esidata = ed;
+
+ memset(hack, 0xaa, sp->wrk->ws->f - hack);
}
/*--------------------------------------------------------------------*/
@@ -763,11 +826,14 @@
struct worker *w;
char *ws_wm;
struct http http_save;
+ struct esidata *ed;
w = sp->wrk;
WRW_Reserve(w, &sp->fd);
http_save.magic = 0;
- VTAILQ_FOREACH(eb, &sp->obj->esibits, list) {
+ ed = sp->obj->esidata;
+ CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
+ VTAILQ_FOREACH(eb, &ed->esibits, list) {
if (Tlen(eb->verbatim)) {
if (sp->http->protover >= 1.1)
(void)WRW_Write(w, eb->chunk_length, -1);
@@ -857,16 +923,10 @@
void
ESI_Destroy(struct object *o)
{
- struct esi_bit *eb;
+ struct esidata *ed;
- /*
- * Delete esi_bits from behind and free(3) the ones that want to be.
- */
- while (!VTAILQ_EMPTY(&o->esibits)) {
- eb = VTAILQ_LAST(&o->esibits, esibithead);
- VTAILQ_REMOVE(&o->esibits, eb, list);
- if (eb->free_this)
- free(eb);
- }
+ ed = o->esidata;
+ o->esidata = NULL;
+ CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
+ STV_free(ed->storage);
}
-
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-11-16 10:58:30 UTC (rev 4350)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-11-16 12:44:03 UTC (rev 4351)
@@ -701,7 +701,8 @@
DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
o->xid, WS_Free(o->ws_o));
- ESI_Destroy(o);
+ if (o->esidata != NULL)
+ ESI_Destroy(o);
if (o->objcore != NULL && o->objcore->smp_seg != NULL) {
SMP_FreeObj(o);
} else {
Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c 2009-11-16 10:58:30 UTC (rev 4350)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c 2009-11-16 12:44:03 UTC (rev 4351)
@@ -139,7 +139,7 @@
HTTPH_A_DELIVER);
/* Only HTTP 1.1 can do Chunked encoding */
- if (!sp->disable_esi && !VTAILQ_EMPTY(&sp->obj->esibits)) {
+ if (!sp->disable_esi && sp->obj->esidata != NULL) {
http_Unset(sp->wrk->resp, H_Content_Length);
if(sp->http->protover >= 1.1)
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Transfer-Encoding: chunked");
@@ -178,7 +178,7 @@
if (sp->disable_esi || !sp->esis)
sp->acct_req.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
- if (!sp->disable_esi && sp->wantbody && !VTAILQ_EMPTY(&sp->obj->esibits)) {
+ if (!sp->disable_esi && sp->wantbody && sp->obj->esidata != NULL) {
if (WRW_FlushRelease(sp->wrk)) {
vca_close_session(sp, "remote closed");
return;
Modified: trunk/varnish-cache/bin/varnishd/stevedore.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-11-16 10:58:30 UTC (rev 4350)
+++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-11-16 12:44:03 UTC (rev 4351)
@@ -103,7 +103,6 @@
o->grace = NAN;
o->entered = NAN;
VTAILQ_INIT(&o->store);
- VTAILQ_INIT(&o->esibits);
sp->wrk->stats.n_object++;
}
Added: trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc 2009-11-16 12:44:03 UTC (rev 4351)
@@ -0,0 +1,88 @@
+# $Id: e00003.vtc 3742 2009-02-11 07:53:47Z tfheen $
+
+test "Aggressive use of ESI include"
+
+
+server s1 {
+ rxreq
+ txresp -body {
+ <html>
+ Before include
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/00"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/01"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/02"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/03"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/04"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/05"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/06"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/07"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/08"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/09"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/10"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/11"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/12"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/13"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/14"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/15"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/16"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/17"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/18"/>
+ <esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/19"/>
+ After include
+ }
+ rxreq
+ txresp -body { Included file 00 }
+ rxreq
+ txresp -body { Included file 01 }
+ rxreq
+ txresp -body { Included file 02 }
+ rxreq
+ txresp -body { Included file 03 }
+ rxreq
+ txresp -body { Included file 04 }
+ rxreq
+ txresp -body { Included file 05 }
+ rxreq
+ txresp -body { Included file 06 }
+ rxreq
+ txresp -body { Included file 07 }
+ rxreq
+ txresp -body { Included file 08 }
+ rxreq
+ txresp -body { Included file 09 }
+ rxreq
+ txresp -body { Included file 10 }
+ rxreq
+ txresp -body { Included file 11 }
+ rxreq
+ txresp -body { Included file 12 }
+ rxreq
+ txresp -body { Included file 13 }
+ rxreq
+ txresp -body { Included file 14 }
+ rxreq
+ txresp -body { Included file 15 }
+ rxreq
+ txresp -body { Included file 16 }
+ rxreq
+ txresp -body { Included file 17 }
+ rxreq
+ txresp -body { Included file 18 }
+ rxreq
+ txresp -body { Included file 19 }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_fetch {
+ esi;
+ }
+} -start
+
+client c1 {
+ txreq
+ rxresp
+ expect resp.status == 200
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0
More information about the varnish-commit
mailing list