[master] 58ed5c9f6 vrb: Turn BS_CACHED into a request flag

[Dridi Boukelmoune]

commit 58ed5c9f617d0c11b4cd96f9278e6b5babd833ec
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Apr 12 07:01:28 2022 +0200

    vrb: Turn BS_CACHED into a request flag
    
    This way we can keep track of how we got the body (until it is taken by
    a busyobj) separately from the fact that we cached it.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b78566a69..05dc40ce9 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -289,15 +289,15 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
 	bo->ws_bo = WS_Snapshot(bo->ws);
 	HTTP_Clone(bo->bereq, bo->bereq0);
 
-	if (bo->req->req_body_status->avail == 0) {
-		bo->req = NULL;
-		ObjSetState(bo->wrk, oc, BOS_REQ_DONE);
-	} else if (bo->req->req_body_status == BS_CACHED) {
+	if (bo->req->req_body_cached) {
 		AN(bo->req->body_oc);
 		bo->bereq_body = bo->req->body_oc;
 		HSH_Ref(bo->bereq_body);
 		bo->req = NULL;
 		ObjSetState(bo->wrk, oc, BOS_REQ_DONE);
+	} else if (bo->req->req_body_status->avail == 0) {
+		bo->req = NULL;
+		ObjSetState(bo->wrk, oc, BOS_REQ_DONE);
 	}
 	return (F_STP_STARTFETCH);
 }
diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index f2c894cc0..fa4b4062b 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -177,7 +177,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
 		    (uintmax_t)req_bodybytes);
 	}
 
-	req->req_body_status = BS_CACHED;
+	req->req_body_cached = 1;
 	return (req_bodybytes);
 }
 
@@ -199,7 +199,7 @@ VRB_Iterate(struct worker *wrk, struct vsl_log *vsl,
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AN(func);
 
-	if (req->req_body_status == BS_CACHED) {
+	if (req->req_body_cached) {
 		AN(req->body_oc);
 		if (ObjIterate(wrk, req->body_oc, priv, func, 0))
 			return (-1);
@@ -277,10 +277,13 @@ VRB_Free(struct req *req)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	if (req->body_oc == NULL)
+	if (req->body_oc == NULL) {
+		AZ(req->req_body_cached);
 		return;
+	}
 
 	r = HSH_DerefObjCore(req->wrk, &req->body_oc, 0);
+	req->req_body_cached = 0;
 
 	// each busyobj may have gained a reference
 	assert (r >= 0);
@@ -312,13 +315,13 @@ VRB_Cache(struct req *req, ssize_t maxsize)
 	 * where we know we will have no competition or conflicts for the
 	 * updates to req.http.* etc.
 	 */
-	if (req->restarts > 0 && req->req_body_status != BS_CACHED) {
+	if (req->restarts > 0 && !req->req_body_cached) {
 		VSLb(req->vsl, SLT_VCL_Error,
 		    "req.body must be cached before restarts");
 		return (-1);
 	}
 
-	if (req->req_body_status == BS_CACHED) {
+	if (req->req_body_cached) {
 		AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u));
 		return (u);
 	}
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ed97a0110..31929e123 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -813,7 +813,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 			bo->req = req;
 			bo->wrk = wrk;
 			/* Unless cached, reqbody is not our job */
-			if (req->req_body_status != BS_CACHED)
+			if (!req->req_body_cached)
 				req->req_body_status = BS_NONE;
 			SES_Close(req->sp, VDI_Http1Pipe(req, bo));
 			nxt = REQ_FSM_DONE;
@@ -900,6 +900,7 @@ cnt_recv_prep(struct req *req, const char *ci)
 		req->client_identity = NULL;
 		req->storage = NULL;
 		req->trace = FEATURE(FEATURE_TRACE);
+		AZ(req->req_body_cached);
 	}
 
 	req->is_hit = 0;
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 4ea37e1d3..b1918253d 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -119,7 +119,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 			V1L_Chunked(wrk);
 		i = VRB_Iterate(wrk, bo->vsl, bo->req, vbf_iter_req_body, bo);
 
-		if (bo->req->req_body_status != BS_CACHED)
+		if (!bo->req->req_body_cached)
 			bo->no_retry = "req.body not cached";
 
 		if (bo->req->req_body_status == BS_ERROR) {
diff --git a/include/tbl/body_status.h b/include/tbl/body_status.h
index 108345c6c..39c45b4fb 100644
--- a/include/tbl/body_status.h
+++ b/include/tbl/body_status.h
@@ -39,7 +39,6 @@ BODYSTATUS(CHUNKED,	chunked,	2,	1,	0)
 BODYSTATUS(LENGTH,	length,		3,	1,	1)
 BODYSTATUS(EOF,		eof,		4,	1,	0)
 BODYSTATUS(TAKEN,	taken,		5,	0,	0)
-BODYSTATUS(CACHED,	cached,		6,	2,	1)
 #undef BODYSTATUS
 
 /*lint -restore */
diff --git a/include/tbl/req_flags.h b/include/tbl/req_flags.h
index 6a9e6acbb..282bd92ea 100644
--- a/include/tbl/req_flags.h
+++ b/include/tbl/req_flags.h
@@ -41,6 +41,7 @@ REQ_FLAG(waitinglist,		0, 0, "")
 REQ_FLAG(want100cont,		0, 0, "")
 REQ_FLAG(late100cont,		0, 0, "")
 REQ_FLAG(req_reset,		0, 0, "")
+REQ_FLAG(req_body_cached,	0, 0, "")
 #define REQ_BEREQ_FLAG(lower, vcl_r, vcl_w, doc) \
 	REQ_FLAG(lower, vcl_r, vcl_w, doc)
 #include "tbl/req_bereq_flags.h"