[4.1] ae16ab8 Don't allow partial allocation of stevedore space for ESI
PÃ¥l Hermunn Johansen
hermunn at varnish-software.com
Tue May 31 13:19:05 CEST 2016
commit ae16ab8e3071e94e96bacc3af6f7964d12aca05c
Author: Pål Hermunn Johansen <hermunn at varnish-software.com>
Date: Tue May 31 11:32:37 2016 +0200
Don't allow partial allocation of stevedore space for ESI
When allocating ESI memory through objallocwithnuke, tell the sevedore
in storage_simple.c that getting less than requested, is not
interesting.
Fixes #1941
This is a back port of #1961
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f908af1..39bdef7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1051,7 +1051,8 @@ void RFC2616_Vary_AE(struct http *hp);
/* stevedore.c */
int STV_NewObject(struct objcore *, struct worker *,
const char *hint, unsigned len);
-struct storage *STV_alloc(const struct stevedore *, size_t size);
+struct storage *STV_alloc(const struct stevedore *, size_t size, int flags);
+#define LESS_MEM_ALLOCED_IS_OK 1
void STV_trim(const struct stevedore *, struct storage *, size_t size,
int move_ok);
void STV_free(const struct stevedore *, struct storage *st);
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 53fe080..edfd797 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -235,7 +235,8 @@ ObjIterEnd(struct objcore *oc, void **oix)
*/
static struct storage *
-objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
+objallocwithnuke(struct worker *wrk, const struct stevedore *stv,
+ size_t size, int flags)
{
struct storage *st = NULL;
unsigned fail;
@@ -243,15 +244,18 @@ objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
- if (size > cache_param->fetch_maxchunksize)
+ if (size > cache_param->fetch_maxchunksize) {
+ if (!(flags & LESS_MEM_ALLOCED_IS_OK))
+ return (NULL);
size = cache_param->fetch_maxchunksize;
+ }
assert(size <= UINT_MAX); /* field limit in struct storage */
for (fail = 0; fail <= cache_param->nuke_limit; fail++) {
/* try to allocate from it */
AN(stv->alloc);
- st = STV_alloc(stv, size);
+ st = STV_alloc(stv, size, flags);
if (st != NULL)
break;
@@ -299,7 +303,8 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr)
return (1);
}
- st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz);
+ st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz,
+ LESS_MEM_ALLOCED_IS_OK);
if (st == NULL)
return (0);
@@ -564,9 +569,11 @@ ObjSetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
st = o->objstore;
switch (attr) {
case OA_ESIDATA:
- o->esidata = objallocwithnuke(wrk, oc->stobj->stevedore, len);
+ o->esidata = objallocwithnuke(wrk, oc->stobj->stevedore,
+ len, 0);
if (o->esidata == NULL)
return (NULL);
+ assert(o->esidata->space >= len);
o->esidata->len = len;
retval = o->esidata->ptr;
break;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 77e54a3..4845dba 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -156,12 +156,18 @@ stv_pick_stevedore(struct vsl_log *vsl, const char **hint)
/*-------------------------------------------------------------------*/
struct storage *
-STV_alloc(const struct stevedore *stv, size_t size)
+STV_alloc(const struct stevedore *stv, size_t size, int flags)
{
struct storage *st;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+ if (!(flags & LESS_MEM_ALLOCED_IS_OK)) {
+ if (size > cache_param->fetch_maxchunksize)
+ return (NULL);
+ else
+ return (stv->alloc(stv, size));
+ }
if (size > cache_param->fetch_maxchunksize)
size = cache_param->fetch_maxchunksize;
diff --git a/bin/varnishtest/tests/r01941.vtc b/bin/varnishtest/tests/r01941.vtc
new file mode 100644
index 0000000..2531ded
--- /dev/null
+++ b/bin/varnishtest/tests/r01941.vtc
@@ -0,0 +1,126 @@
+
+varnishtest "ESI memory should only be fully allocated"
+
+# In this test case, the cache is almost filled. Then we force the
+# allocation of ESI memory that does not fit in the cache, is bigger
+# than fetch_chunksize, but where half the amount can be allocated by
+# the stevedore.
+
+server s1 {
+ rxreq
+ expect req.url == /big
+ txresp -bodylen 1037480
+ rxreq
+ txresp -body {<html>
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+</html>}
+ rxreq
+ expect req.url == /long1234567890123456789012345678901234567890.txt
+ txresp -body {fo}
+} -start
+
+varnish v1 -arg "-pfetch_chunksize=4k" \
+ -arg "-smalloc,1m" -vcl+backend {
+ sub vcl_backend_response {
+ set beresp.do_esi = true;
+ }
+} -start
+
+client c1 {
+ txreq -url /big
+ rxresp
+ expect resp.bodylen == 1037480
+ txreq -url /
+ rxresp
+} -run
More information about the varnish-commit
mailing list