[master] 08403b1dd Incremental freeing of storage for hfm/hpf and proper vtc

[Nils Goroll]

commit 08403b1dd03d392c22713b9b9c72e51235f95caf
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Feb 22 14:53:50 2019 +0100

    Incremental freeing of storage for hfm/hpf and proper vtc
    
    Previously, we did not test for the storage being actually freed and
    freeing only happened for passes, either explicit or a hit on hfm or
    hfp, but not the for the object becoming the hfm/hfp.
    
    We now handle incremental freeing for hfm/hfp as for private objects and
    add a test which actually checks that storage is being freed on the go.
    
    Fixes #2653

diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index c956c93a0..1bd65a66c 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -150,11 +150,12 @@ vdp_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len)
 int
 VDP_DeliverObj(struct req *req)
 {
-	int r;
+	int r, final;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	r = ObjIterate(req->wrk, req->objcore, req, vdp_objiterator,
-	    req->objcore->flags & OC_F_PRIVATE ? 1 : 0);
+	final = req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)
+	    ? 1 : 0;
+	r = ObjIterate(req->wrk, req->objcore, req, vdp_objiterator, final);
 	if (r < 0)
 		return (r);
 	return (0);
diff --git a/bin/varnishtest/tests/c00075.vtc b/bin/varnishtest/tests/c00075.vtc
index 6cafdde62..c671844eb 100644
--- a/bin/varnishtest/tests/c00075.vtc
+++ b/bin/varnishtest/tests/c00075.vtc
@@ -1,20 +1,148 @@
 varnishtest "Test large pass deleted during streaming"
 
+barrier ba1 cond 3
+barrier ba2 cond 2
+
+barrier bb1 cond 3
+barrier bb2 cond 2
+
+barrier bc1 cond 3
+barrier bc2 cond 2
+
+barrier bd1 cond 3
+barrier bd2 cond 2
+
 server s1 {
 	rxreq
-	txresp -nolen -hdr "Transfer-Encoding: chunked"
-	chunkedlen 65536
+	expect req.url == "/hfm"
+	txresp -nolen -hdr "Transfer-Encoding: chunked" \
+	    -hdr "Cache-Control: no-cache"
+	chunkedlen 32768
+	barrier ba1 sync
+	chunkedlen 32768
+	chunkedlen 0
+
+	rxreq
+	expect req.url == "/hfm"
+	txresp -nolen -hdr "Transfer-Encoding: chunked" \
+	    -hdr "Cache-Control: no-cache"
+	chunkedlen 32768
+	barrier bb1 sync
+	chunkedlen 32768
+	chunkedlen 0
+
+	rxreq
+	expect req.url == "/hfp"
+	txresp -nolen -hdr "Transfer-Encoding: chunked" \
+	    -hdr "Cache-Control: no-cache"
+	chunkedlen 32768
+	barrier bc1 sync
+	chunkedlen 32768
+	chunkedlen 0
+
+	rxreq
+	expect req.url == "/hfp"
+	txresp -nolen -hdr "Transfer-Encoding: chunked" \
+	    -hdr "Cache-Control: no-cache"
+	chunkedlen 32768
+	barrier bd1 sync
+	chunkedlen 32768
 	chunkedlen 0
 } -start
 
 varnish v1 \
-	-arg "-s default,1m" -vcl+backend { } -start
+	-arg "-s default,1m" -vcl+backend {
+	sub vcl_backend_response {
+		if (bereq.url == "/hfp") {
+			return (pass(10m));
+		}
+	}
+	sub vcl_deliver {
+		set resp.http.is-hitmiss = req.is_hitmiss;
+		set resp.http.is-hitpass = req.is_hitpass;
+	}
+} -start
 
-varnish v1 -cliok "debug.fragfetch 1024"
+varnish v1 -cliok "debug.fragfetch 8192"
 
 client c1 {
-	txreq -hdr "Cookie: bar"
-	rxresp
+	txreq -url "/hfm"
+	rxresphdrs
+	rxrespbody -max 8192
+	barrier ba1 sync
+	barrier ba2 sync
+	rxrespbody
 	expect resp.bodylen == 65536
+	expect resp.http.is-hitmiss == false
+	expect resp.http.is-hitpass == false
 	expect_pattern
-} -run
+} -start
+
+barrier ba1 sync
+varnish v1 -expect SM?.Transient.g_bytes < 24576
+barrier ba2 sync
+
+# HFM object
+varnish v1 -expect SM?.Transient.g_bytes < 500
+
+# pass on the HFM
+client c1 {
+	txreq -url "/hfm"
+	rxresphdrs
+	rxrespbody -max 8192
+	barrier bb1 sync
+	barrier bb2 sync
+	rxrespbody
+	expect resp.bodylen == 65536
+	expect resp.http.is-hitmiss == true
+	expect resp.http.is-hitpass == false
+	expect_pattern
+} -start
+
+barrier bb1 sync
+varnish v1 -expect SM?.Transient.g_bytes < 24576
+barrier bb2 sync
+
+client c1 -wait
+
+## hfp
+
+client c1 {
+	txreq -url "/hfp"
+	rxresphdrs
+	rxrespbody -max 8192
+	barrier bc1 sync
+	barrier bc2 sync
+	rxrespbody
+	expect resp.bodylen == 65536
+	expect resp.http.is-hitmiss == false
+	expect resp.http.is-hitpass == false
+	expect_pattern
+} -start
+
+barrier bc1 sync
+varnish v1 -expect SM?.Transient.g_bytes < 24576
+barrier bc2 sync
+
+# HFM object
+varnish v1 -expect SM?.Transient.g_bytes < 500
+
+# pass on the HFM
+client c1 {
+	txreq -url "/hfp"
+	rxresphdrs
+	rxrespbody -max 8192
+	barrier bd1 sync
+	barrier bd2 sync
+	rxrespbody
+	expect resp.bodylen == 65536
+	expect resp.http.is-hitmiss == false
+	expect resp.http.is-hitpass == true
+	expect_pattern
+} -start
+
+barrier bd1 sync
+varnish v1 -expect SM?.Transient.g_bytes < 24576
+barrier bd2 sync
+
+client c1 -wait