[4.0] 535d44b Don't needlessly throw away Content-Length from backend, but assume it is sane and pass it to streaming clients, provided we don't munge the body along the way.
Martin Blix Grydeland
martin at varnish-software.com
Fri Nov 7 14:17:24 CET 2014
commit 535d44b8909ee88e8700a35bed79f9ca77e445a4
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date: Thu Nov 6 17:59:16 2014 +0100
Don't needlessly throw away Content-Length from backend, but assume
it is sane and pass it to streaming clients, provided we don't munge
the body along the way.
The ungzip'ed length stored as attribute has not been backported to
4.0, meaning on-the-fly ungzip will be delivered chunked.
Fixes: #1506
Conflicts:
bin/varnishd/cache/cache_fetch.c
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 699d9a8..a07d6e6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -360,6 +360,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
+ http_Unset(bo->beresp, H_Content_Length);
http_Merge(bo->ims_obj->http, bo->beresp,
bo->ims_obj->changed_gzip);
assert(bo->beresp->status == 200);
@@ -468,19 +469,23 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
+ http_Unset(bo->beresp, H_Content_Length);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
+ http_Unset(bo->beresp, H_Content_Length);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
+ http_Unset(bo->beresp, H_Content_Length);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
+ http_Unset(bo->beresp, H_Content_Length);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 3779238..623b07b 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -240,36 +240,23 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
req->res_mode = 0;
+ if (!req->disable_esi && req->obj->esidata != NULL)
+ req->res_mode |= RES_ESI;
+
if (req->esi_level > 0)
req->res_mode |= RES_ESI_CHILD;
- if (!req->disable_esi && req->obj->esidata != NULL) {
+ if (req->res_mode & (RES_ESI_CHILD|RES_ESI)) {
/* In ESI mode, we can't know the aggregate length */
- req->res_mode |= RES_ESI;
+ http_Unset(req->resp, H_Content_Length);
RFC2616_Weaken_Etag(req->resp);
- }
-
- if (req->res_mode & (RES_ESI_CHILD|RES_ESI)) {
- /* nothing */
} else if (req->resp->status == 304) {
- req->res_mode &= ~RES_LEN;
http_Unset(req->resp, H_Content_Length);
req->wantbody = 0;
- } else if (bo != NULL) {
- /* Streaming, decide CHUNKED/EOF later */
- } else if ((req->obj->objcore->flags & OC_F_PASS) && !req->wantbody) {
- /*
- * if we pass a HEAD the C-L header may already be in the
- * object and it will not match the actual storage length
- * which is zero.
- * Hand that C-L header back to client.
- */
- req->res_mode |= RES_LEN;
- } else {
- req->res_mode |= RES_LEN;
- http_Unset(req->resp, H_Content_Length);
- http_PrintfHeader(req->resp,
- "Content-Length: %zd", req->obj->len);
+ } else if (bo == NULL &&
+ !http_GetHdr(req->resp, H_Content_Length, NULL)) {
+ http_PrintfHeader(req->resp, "Content-Length: %zd",
+ req->obj->len);
}
if (cache_param->http_gzip_support && req->obj->gziped &&
@@ -279,31 +266,25 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
* XXX: we could cache that, but would still deliver
* XXX: with multiple writes because of the gunzip buffer
*/
- req->res_mode &= ~RES_LEN;
req->res_mode |= RES_GUNZIP;
+ http_Unset(req->resp, H_Content_Encoding);
+ http_Unset(req->resp, H_Content_Length);
}
- if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
- /* We havn't chosen yet, do so */
- if (!req->wantbody) {
- /* Nothing */
- } else if (req->http->protover >= 11) {
+ if (http_GetHdr(req->resp, H_Content_Length, NULL))
+ req->res_mode |= RES_LEN;
+
+ if (req->wantbody && !(req->res_mode & RES_LEN)) {
+ if (req->http->protover >= 11) {
+ http_SetHeader(req->resp, "Transfer-Encoding: chunked");
req->res_mode |= RES_CHUNKED;
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
- VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
-
- if (!(req->res_mode & RES_LEN))
- http_Unset(req->resp, H_Content_Length);
-
- if (req->res_mode & RES_GUNZIP)
- http_Unset(req->resp, H_Content_Encoding);
- if (req->res_mode & RES_CHUNKED)
- http_SetHeader(req->resp, "Transfer-Encoding: chunked");
+ VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
http_SetHeader(req->resp,
req->doclose ? "Connection: close" : "Connection: keep-alive");
@@ -380,44 +361,29 @@ V1D_Deliver_Synth(struct req *req)
AN(req->synth_body);
req->res_mode = 0;
- if (req->resp->status == 304) {
- req->res_mode &= ~RES_LEN;
- http_Unset(req->resp, H_Content_Length);
+
+ http_Unset(req->resp, H_Content_Length);
+ if (req->esi_level > 0) {
+ req->res_mode |= RES_ESI_CHILD;
+ } else if (req->resp->status == 304) {
req->wantbody = 0;
} else {
- req->res_mode |= RES_LEN;
- http_Unset(req->resp, H_Content_Length);
http_PrintfHeader(req->resp, "Content-Length: %zd",
VSB_len(req->synth_body));
+ req->res_mode |= RES_LEN;
}
- if (req->esi_level > 0) {
- /* Included ESI object, always CHUNKED or EOF */
- req->res_mode &= ~RES_LEN;
- req->res_mode |= RES_ESI_CHILD;
- }
-
- if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
- /* We havn't chosen yet, do so */
- if (!req->wantbody) {
- /* Nothing */
- } else if (req->http->protover >= 11) {
+ if (req->wantbody && !(req->res_mode & RES_LEN)) {
+ if (req->http->protover >= 11) {
+ http_SetHeader(req->resp, "Transfer-Encoding: chunked");
req->res_mode |= RES_CHUNKED;
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
- VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
- if (!(req->res_mode & RES_LEN))
- http_Unset(req->resp, H_Content_Length);
-
- if (req->res_mode & RES_GUNZIP)
- http_Unset(req->resp, H_Content_Encoding);
-
- if (req->res_mode & RES_CHUNKED)
- http_SetHeader(req->resp, "Transfer-Encoding: chunked");
+ VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
http_SetHeader(req->resp,
req->doclose ? "Connection: close" : "Connection: keep-alive");
diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index 587bf38..4df8ccd 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -25,8 +25,9 @@ varnish v1 \
sub vcl_backend_response {
set beresp.do_stream = false;
set beresp.storage_hint = "invalid";
- # Unset Date header to not change the object sizes
+ # Unset Date and CL header to not change the object sizes
unset beresp.http.Date;
+ unset beresp.http.Content-Length;
}
} -start
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 77425cb..4d715fa 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -17,8 +17,9 @@ varnish v1 \
sub vcl_backend_response {
set beresp.do_stream = false;
set beresp.storage_hint = "s0";
- # Unset Date header to not change the object sizes
+ # Unset Date and CL header to not change the object sizes
unset beresp.http.Date;
+ unset beresp.http.Content-Length;
}
} -start
diff --git a/bin/varnishtest/tests/r00801.vtc b/bin/varnishtest/tests/r00801.vtc
index f538d7f..ca1a567 100644
--- a/bin/varnishtest/tests/r00801.vtc
+++ b/bin/varnishtest/tests/r00801.vtc
@@ -13,6 +13,7 @@ varnish v1 -vcl+backend {
sub vcl_recv { return (pass); }
sub vcl_backend_response {
set beresp.do_stream = false;
+ unset beresp.http.content-length;
}
} -start
diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc
index 11e2a72..dc4728d 100644
--- a/bin/varnishtest/tests/r01140.vtc
+++ b/bin/varnishtest/tests/r01140.vtc
@@ -24,6 +24,7 @@ varnish v1 -arg "-p nuke_limit=0 -p shortlived=0" \
set beresp.do_stream = false;
# Unset Date header to not change the object sizes
unset beresp.http.Date;
+ unset beresp.http.Content-Length;
}
} -start
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index dc6336f..194de2d 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -16,8 +16,9 @@ varnish v1 \
sub vcl_backend_response {
set beresp.do_stream = false;
set beresp.storage_hint = "Transient";
- # Unset Date header to not change the object sizes
+ # Unset Date and CL header to not change the object sizes
unset beresp.http.Date;
+ unset beresp.http.Content-Length;
}
} -start
diff --git a/bin/varnishtest/tests/r01506.vtc b/bin/varnishtest/tests/r01506.vtc
new file mode 100644
index 0000000..fc5b841
--- /dev/null
+++ b/bin/varnishtest/tests/r01506.vtc
@@ -0,0 +1,41 @@
+varnishtest "Streaming of C-L header from backend"
+
+server s1 {
+ rxreq
+ txresp -nolen -hdr "Content-Length: 020"
+ send "0123456789"
+ sema r1 sync 2
+ send "0123456789"
+
+ rxreq
+ txresp -nolen -hdr "Content-Length: 010"
+ sema r2 sync 2
+ send "0123456789"
+} -start
+
+varnish v1 -vcl+backend {
+
+ sub vcl_backend_response {
+ if (bereq.url == "/2") {
+ set beresp.do_gzip = true;
+ }
+ }
+} -start -cliok "param.set debug +syncvsl" -cliok "param.set debug +flush_head"
+
+client c1 {
+ txreq
+ rxresphdrs
+ expect resp.http.content-length == "020"
+ sema r1 sync 2
+ rxrespbody
+ expect resp.body == "01234567890123456789"
+ delay .1
+
+ txreq -url "/2"
+ rxresphdrs
+ expect resp.http.content-length == "<undef>"
+ sema r2 sync 2
+ rxrespbody
+ expect resp.body == "0123456789"
+ delay .1
+} -run
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index 26443ca..0a5395f 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -57,7 +57,7 @@ HTTPH("Cache-Control", H_Cache_Control, HTTPH_R_FETCH ) /* RFC2616 1
HTTPH("Connection", H_Connection, HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS) /* RFC2616 14.10 */
HTTPH("Content-Encoding", H_Content_Encoding, 0 ) /* RFC2616 14.11 */
HTTPH("Content-Language", H_Content_Language, 0 ) /* RFC2616 14.12 */
-HTTPH("Content-Length", H_Content_Length, HTTPH_R_FETCH | HTTPH_A_INS) /* RFC2616 14.13 */
+HTTPH("Content-Length", H_Content_Length, HTTPH_R_FETCH ) /* RFC2616 14.13 */
HTTPH("Content-Location", H_Content_Location, 0 ) /* RFC2616 14.14 */
HTTPH("Content-MD5", H_Content_MD5, 0 ) /* RFC2616 14.15 */
HTTPH("Content-Range", H_Content_Range, HTTPH_R_FETCH | HTTPH_A_INS) /* RFC2616 14.16 */
More information about the varnish-commit
mailing list