[master] 7da66c9b0 Improve streaming range responses

[Poul-Henning Kamp]

commit 7da66c9b03f82b97e2476d5d0af62e58e0419216
Author: Gil Pedersen <git at gpost.dk>
Date:   Tue Nov 29 19:10:31 2022 +0100

    Improve streaming range responses
    
    This is an enhanced fix for #1777

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index b9aab9aed..695ca0211 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -124,7 +124,7 @@ vrg_dorange(struct req *req, void **priv)
 		high = req->resp_len - 1;
 	} else if (req->resp_len >= 0 && (high >= req->resp_len || high < 0))
 		high = req->resp_len - 1;
-	else if (high < 0 || req->resp_len < 0)
+	else if (high < 0)
 		return (NULL);			// Allow 200 response
 	/*
 	 * else (bo != NULL) {
@@ -136,13 +136,13 @@ vrg_dorange(struct req *req, void **priv)
 	if (req->resp_len >= 0 && low >= req->resp_len)
 		return ("low range beyond object");
 
-	if (req->resp_len >= 0)
+	if (req->resp_len >= 0) {
 		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
 		    (intmax_t)low, (intmax_t)high, (intmax_t)req->resp_len);
-	else
+		req->resp_len = (intmax_t)(1 + high - low);
+	} else
 		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/*",
 		    (intmax_t)low, (intmax_t)high);
-	req->resp_len = (intmax_t)(1 + high - low);
 
 	vrg_priv = WS_Alloc(req->ws, sizeof *vrg_priv);
 	if (vrg_priv == NULL)
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index bc2af742e..4858d3967 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -178,12 +178,10 @@ client c5 {
 		-hdr "Range: bytes=2-5" \
 		-hdr "Accept-encoding: gzip"
 	rxresp
-	expect resp.status == 200
-	expect resp.http.Content-Range == <undef>
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 2-5/*"
 	expect resp.http.Content-Length == <undef>
-	expect resp.http.Content-Encoding == gzip
-	gunzip
-	expect resp.bodylen == 100
+	expect resp.bodylen == 4
 } -run
 
 # Test partial range with http2
diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc
index 3edc19747..9f90bd3c2 100644
--- a/bin/varnishtest/tests/e00015.vtc
+++ b/bin/varnishtest/tests/e00015.vtc
@@ -87,8 +87,6 @@ varnish v1 -syntax 4.1 -vcl+backend {
 	}
 }
 
-# Note on Range requests: The range VDP is active, but as it cannot
-# reliably determine the size of the response, it falls back to a 200
 client c1 {
 	txreq -url /top2
 	rxresp
@@ -106,11 +104,10 @@ client c1 {
 	expect resp.http.filter0 == "esi"
 	expect resp.http.filters == "esi"
 
-	# see Note on Range above
 	txreq -url "/esi" -hdr "Range: bytes=1-2"
 	rxresp
-	expect resp.bodylen == 76
-	expect resp.status == 200
+	expect resp.bodylen == 2
+	expect resp.status == 206
 	expect resp.http.was == true
 	expect resp.http.filters == "esi range"
 
@@ -121,11 +118,10 @@ client c1 {
 	expect resp.http.was == true
 	expect resp.http.filters == "esi gunzip"
 
-	# see Note on Range above
 	txreq -url "/recurse" -hdr "Range: bytes=1-2"
 	rxresp
-	expect resp.bodylen == 120
-	expect resp.status == 200
+	expect resp.bodylen == 2
+	expect resp.status == 206
 	expect resp.http.was == true
 	expect resp.http.filters == "esi gunzip range"
 
@@ -136,4 +132,4 @@ client c1 {
 } -run
 
 varnish v1 -expect esi_errors == 0
-varnish v1 -expect MAIN.s_resp_bodybytes == 865
+varnish v1 -expect MAIN.s_resp_bodybytes == 673
diff --git a/bin/varnishtest/tests/g00005.vtc b/bin/varnishtest/tests/g00005.vtc
index 74b21de5b..1240c7a09 100644
--- a/bin/varnishtest/tests/g00005.vtc
+++ b/bin/varnishtest/tests/g00005.vtc
@@ -25,12 +25,13 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
 } -start
 
 client c1 {
-	# no range support with streaming cache miss and gunzip
+	# cache miss
 	txreq -hdr "Range: bytes=3-5"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == "10"
+	expect resp.status == 206
+	expect resp.bodylen == "3"
 	expect resp.http.content-encoding == <undef>
+	expect resp.body == "BAR"
 } -run
 
 varnish v1 -vsl_catchup
diff --git a/bin/varnishtest/tests.disabled/r01506.vtc b/bin/varnishtest/tests/r01506.vtc
similarity index 92%
rename from bin/varnishtest/tests.disabled/r01506.vtc
rename to bin/varnishtest/tests/r01506.vtc
index 1d0f0bb3c..92192653a 100644
--- a/bin/varnishtest/tests.disabled/r01506.vtc
+++ b/bin/varnishtest/tests/r01506.vtc
@@ -33,7 +33,6 @@ client c1 {
 	txreq -url /1 -hdr "Range: bytes=17-101"
 	rxresphdrs
 	expect resp.status == 206
-	expect resp.http.content-length == 85
 	barrier b1 sync
 	rxrespbody
 	expect resp.bodylen == 85
@@ -59,15 +58,13 @@ client c1 {
 	expect resp.bodylen == 136
 	delay .1
 
-	# Invalid range
+	# Handles out of bounds range
 	txreq -url /4 -hdr "Range: bytes=102-200"
 	rxresphdrs
 	expect resp.status == 206
-	expect resp.http.content-length == 99
 	barrier b1 sync
-	recv 34
-	delay .3
-	expect_close
+	rxrespbody
+	expect resp.bodylen == 34
 } -run
 
 varnish v1 -expect sc_range_short == 1
diff --git a/bin/varnishtest/tests.disabled/r01732.vtc b/bin/varnishtest/tests/r01732.vtc
similarity index 100%
rename from bin/varnishtest/tests.disabled/r01732.vtc
rename to bin/varnishtest/tests/r01732.vtc
diff --git a/bin/varnishtest/tests/r01777.vtc b/bin/varnishtest/tests/r01777.vtc
index 6808023c7..ef8871153 100644
--- a/bin/varnishtest/tests/r01777.vtc
+++ b/bin/varnishtest/tests/r01777.vtc
@@ -14,5 +14,8 @@ varnish v1 -vcl+backend { } -start
 client c1 {
 	txreq -hdr "Range: bytes=0-129"
 	rxresp
-	expect resp.status == 200
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 0-129/*"
+	expect resp.http.Content-Length == <undef>
+	expect resp.bodylen == 128
 } -run