[master] 419001b6d hash: Trigger a rush when an objcore is withdrawn

[Dridi Boukelmoune]

commit 419001b6dec288f3e5e722465e05aa2508fb0f24
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Dec 19 15:18:02 2023 +0100

    hash: Trigger a rush when an objcore is withdrawn
    
    This is the counterpart of HSH_Unbusy() for the cases where the req task
    will not schedule a fetch task, at which point we know we can already
    wake up a request, if there is one on the waiting list.
    
    This encapsulates the "wake up only one request" semantic using a new
    OC_F_WITHDRAWN flag. Although this flag is not being used yet, it will
    when the state of the objcore determines the rush policy instead of the
    call site.
    
    When a bgfetch is withdrawn, the extra rush becomes redundant with the
    policy rush on the hit objcore. There will eventually be no more rush
    for the hit objcore, which will remove the redundancy just introduced.
    
    In order to withdraw objcores in a single critical section, a function
    HSH_DerefObjCoreUnlock() is added.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index ed8244f2a..69c7da8fd 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -908,6 +908,37 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc)
 	ObjSlim(wrk, oc);
 }
 
+/*---------------------------------------------------------------------
+ * Withdraw an objcore that will not proceed with a fetch.
+ */
+
+void
+HSH_Withdraw(struct worker *wrk, struct objcore **ocp)
+{
+	struct objhead *oh;
+	struct objcore *oc;
+	struct rush rush;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	TAKE_OBJ_NOTNULL(oc, ocp, OBJCORE_MAGIC);
+	INIT_OBJ(&rush, RUSH_MAGIC);
+
+	oh = oc->objhead;
+	CHECK_OBJ(oh, OBJHEAD_MAGIC);
+
+	Lck_Lock(&oh->mtx);
+	AZ(oc->stobj->stevedore);
+	AN(oc->flags & OC_F_BUSY);
+	assert(oc->refcnt == 1);
+	assert(oh->refcnt > 0);
+	oc->flags &= ~OC_F_BUSY;
+	oc->flags |= OC_F_WITHDRAWN;
+	hsh_rush1(wrk, oh, &rush, 1);
+	AZ(HSH_DerefObjCoreUnlock(wrk, &oc, 0));
+
+	hsh_rush2(wrk, &rush);
+}
+
 /*---------------------------------------------------------------------
  * Unbusy an objcore when the object is completely fetched.
  */
@@ -1094,6 +1125,23 @@ HSH_DerefBoc(struct worker *wrk, struct objcore *oc)
 
 int
 HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax)
+{
+	struct objcore *oc;
+	struct objhead *oh;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	TAKE_OBJ_NOTNULL(oc, ocp, OBJCORE_MAGIC);
+	assert(oc->refcnt > 0);
+
+	oh = oc->objhead;
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+
+	Lck_Lock(&oh->mtx);
+	return (HSH_DerefObjCoreUnlock(wrk, &oc, rushmax));
+}
+
+int
+HSH_DerefObjCoreUnlock(struct worker *wrk, struct objcore **ocp, int rushmax)
 {
 	struct objcore *oc;
 	struct objhead *oh;
@@ -1108,7 +1156,7 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax)
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-	Lck_Lock(&oh->mtx);
+	Lck_AssertHeld(&oh->mtx);
 	assert(oh->refcnt > 0);
 	r = --oc->refcnt;
 	if (!r)
diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h
index 866dbc4a2..8d598d7b1 100644
--- a/bin/varnishd/cache/cache_objhead.h
+++ b/bin/varnishd/cache/cache_objhead.h
@@ -69,6 +69,7 @@ void HSH_Kill(struct objcore *);
 void HSH_Replace(struct objcore *, const struct objcore *);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *,
     struct ban *);
+void HSH_Withdraw(struct worker *, struct objcore **);
 void HSH_Fail(struct worker *, struct objcore *);
 void HSH_Unbusy(struct worker *, struct objcore *);
 int HSH_Snipe(const struct worker *, struct objcore *);
@@ -79,6 +80,7 @@ void HSH_DeleteObjHead(const struct worker *, struct objhead *);
 int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax);
 #define HSH_RUSH_POLICY -1
 
+int HSH_DerefObjCoreUnlock(struct worker *, struct objcore **, int rushmax);
 enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **);
 void HSH_Ref(struct objcore *o);
 void HSH_AddString(struct req *, void *ctx, const char *str);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index a8fa68be9..fa00acfdb 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -688,7 +688,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	(void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY);
 
 	if (busy != NULL) {
-		(void)HSH_DerefObjCore(wrk, &busy, 0);
+		HSH_Withdraw(wrk, &busy);
 		VRY_Clear(req);
 	}
 
@@ -736,7 +736,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	VRY_Clear(req);
 	if (req->stale_oc != NULL)
 		(void)HSH_DerefObjCore(wrk, &req->stale_oc, 0);
-	AZ(HSH_DerefObjCore(wrk, &req->objcore, 1));
+	HSH_Withdraw(wrk, &req->objcore);
 	return (REQ_FSM_MORE);
 }
 
@@ -1111,7 +1111,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 
 	(void)HSH_Purge(wrk, boc->objhead, req->t_req, 0, 0, 0);
 
-	AZ(HSH_DerefObjCore(wrk, &boc, 1));
+	HSH_Withdraw(wrk, &boc);
 
 	VCL_purge_method(req->vcl, wrk, req, NULL, NULL);
 	switch (wrk->vpi->handling) {
diff --git a/include/tbl/oc_flags.h b/include/tbl/oc_flags.h
index 768962fda..0866cc38f 100644
--- a/include/tbl/oc_flags.h
+++ b/include/tbl/oc_flags.h
@@ -30,7 +30,8 @@
 
 /*lint -save -e525 -e539 */
 
-OC_FLAG(BUSY,		busy,		(1<<1))		//lint !e835
+OC_FLAG(WITHDRAWN,	withdrawn,	(1<<0))		//lint !e835
+OC_FLAG(BUSY,		busy,		(1<<1))
 OC_FLAG(HFM,		hfm,		(1<<2))
 OC_FLAG(HFP,		hfp,		(1<<3))
 OC_FLAG(CANCEL,		cancel,		(1<<4))