[master] 11b91d1 Don't allow partial allocation of stevedore space for ESI
PÃ¥l Hermunn Johansen
hermunn at varnish-software.com
Tue May 31 13:11:07 CEST 2016
commit 11b91d198f828ef7f567464a990bb78f7e5e6893
Author: Pål Hermunn Johansen <hermunn at varnish-software.com>
Date: Tue May 10 11:37:05 2016 +0200
Don't allow partial allocation of stevedore space for ESI
When allocating ESI memory through objallocwithnuke, tell the sevedore
in storage_simple.c that getting less than requested, is not
interesting. At the same time, eliminate the possibility of getting
less memory than requested in sml_trimstore.
Fixes #1941
diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index f6b70e1..ef427cc 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -43,15 +43,25 @@
#include "vtim.h"
+/* Flags for allocating memory in sml_stv_alloc */
+#define LESS_MEM_ALLOCED_IS_OK 1
+
/*-------------------------------------------------------------------*/
static struct storage *
-sml_stv_alloc(const struct stevedore *stv, size_t size)
+sml_stv_alloc(const struct stevedore *stv, size_t size, int flags)
{
struct storage *st;
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+ if (!(flags & LESS_MEM_ALLOCED_IS_OK)) {
+ if (size > cache_param->fetch_maxchunksize)
+ return (NULL);
+ else
+ return (stv->sml_alloc(stv, size));
+ }
+
if (size > cache_param->fetch_maxchunksize)
size = cache_param->fetch_maxchunksize;
@@ -334,7 +344,8 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
*/
static struct storage *
-objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
+objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size,
+ int flags)
{
struct storage *st = NULL;
unsigned fail;
@@ -342,14 +353,17 @@ objallocwithnuke(struct worker *wrk, const struct stevedore *stv, size_t size)
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
- if (size > cache_param->fetch_maxchunksize)
+ if (size > cache_param->fetch_maxchunksize) {
+ if (!(flags & LESS_MEM_ALLOCED_IS_OK))
+ return (NULL);
size = cache_param->fetch_maxchunksize;
+ }
assert(size <= UINT_MAX); /* field limit in struct storage */
for (fail = 0; fail <= cache_param->nuke_limit; fail++) {
/* try to allocate from it */
- st = sml_stv_alloc(stv, size);
+ st = sml_stv_alloc(stv, size, flags);
if (st != NULL)
break;
@@ -388,7 +402,8 @@ sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz,
return (1);
}
- st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz);
+ st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz,
+ LESS_MEM_ALLOCED_IS_OK);
if (st == NULL)
return (0);
@@ -455,14 +470,10 @@ sml_trimstore(struct worker *wrk, struct objcore *oc)
if (st->space - st->len < 512)
return;
- st1 = sml_stv_alloc(stv, st->len);
+ st1 = sml_stv_alloc(stv, st->len, 0);
if (st1 == NULL)
return;
-
- if (st1->space < st->len) {
- sml_stv_free(stv, st1);
- return;
- }
+ assert(st1->space >= st->len);
memcpy(st1->ptr, st->ptr, st->len);
st1->len = st->len;
@@ -605,7 +616,8 @@ sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
} \
if (len == 0) \
break; \
- o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, len); \
+ o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, \
+ len, 0); \
if (o->aa_##l == NULL) \
break; \
CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC); \
diff --git a/bin/varnishtest/tests/r01941.vtc b/bin/varnishtest/tests/r01941.vtc
new file mode 100644
index 0000000..2531ded
--- /dev/null
+++ b/bin/varnishtest/tests/r01941.vtc
@@ -0,0 +1,126 @@
+
+varnishtest "ESI memory should only be fully allocated"
+
+# In this test case, the cache is almost filled. Then we force the
+# allocation of ESI memory that does not fit in the cache, is bigger
+# than fetch_chunksize, but where half the amount can be allocated by
+# the stevedore.
+
+server s1 {
+ rxreq
+ expect req.url == /big
+ txresp -bodylen 1037480
+ rxreq
+ txresp -body {<html>
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+<esi:include src="/long1234567890123456789012345678901234567890.txt" />
+</html>}
+ rxreq
+ expect req.url == /long1234567890123456789012345678901234567890.txt
+ txresp -body {fo}
+} -start
+
+varnish v1 -arg "-pfetch_chunksize=4k" \
+ -arg "-smalloc,1m" -vcl+backend {
+ sub vcl_backend_response {
+ set beresp.do_esi = true;
+ }
+} -start
+
+client c1 {
+ txreq -url /big
+ rxresp
+ expect resp.bodylen == 1037480
+ txreq -url /
+ rxresp
+} -run
More information about the varnish-commit
mailing list