[master] d15c9bd Extract a protocol independent connection pool from the tcp-pool, and implement the tcp-pool with it.

Poul-Henning Kamp phk at FreeBSD.org
Thu Mar 8 00:40:07 UTC 2018


commit d15c9bdbba9003148d24968ab34ec57952dacac9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 8 00:37:28 2018 +0000

    Extract a protocol independent connection pool from
    the tcp-pool, and implement the tcp-pool with it.
    
    It should now be trivial to implement other pools,
    for instance UDS using this.
    
    Filenames subject to change still (s/tcp_pool/conn_pool/ ?)

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 5d49763..9c0231b 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -38,7 +38,6 @@
 
 #include "vtcp.h"
 #include "vtim.h"
-#include "waiter/waiter.h"
 
 #include "cache_director.h"
 #include "cache_backend.h"
@@ -78,6 +77,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
     unsigned force_fresh)
 {
 	struct pfd *pfd;
+	int *fdp;
 	double tmod;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
@@ -123,8 +123,9 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
 		return (NULL);
 	}
 
-	assert(pfd->fd >= 0);
-	AN(pfd->priv);
+	fdp = PFD_Fd(pfd);
+	AN(fdp);
+	assert(*fdp >= 0);
 
 	Lck_Lock(&bp->mtx);
 	bp->n_conn++;
@@ -133,16 +134,16 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
 	Lck_Unlock(&bp->mtx);
 
 	if (bp->proxy_header != 0)
-		VPX_Send_Proxy(pfd->fd, bp->proxy_header, bo->sp);
+		VPX_Send_Proxy(*fdp, bp->proxy_header, bo->sp);
 
-	VTCP_myname(pfd->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-	VTCP_hisname(pfd->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
+	VTCP_myname(*fdp, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
+	VTCP_hisname(*fdp, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
 	VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s",
-	    pfd->fd, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1);
+	    *fdp, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1);
 
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
 	bo->htc->priv = pfd;
-	bo->htc->rfd = &pfd->fd;
+	bo->htc->rfd = fdp;
 	FIND_TMO(first_byte_timeout,
 	    bo->htc->first_byte_timeout, bo, bp);
 	FIND_TMO(between_bytes_timeout,
@@ -175,20 +176,20 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
-	CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC);
+	pfd = bo->htc->priv;
 	bo->htc->priv = NULL;
-	if (pfd->state != PFD_STATE_USED)
+	if (PFD_State(pfd) != PFD_STATE_USED)
 		assert(bo->htc->doclose == SC_TX_PIPE ||
 		    bo->htc->doclose == SC_RX_TIMEOUT);
 	if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) {
-		VSLb(bo->vsl, SLT_BackendClose, "%d %s", pfd->fd,
+		VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd),
 		    bp->director->display_name);
 		VTP_Close(&pfd);
 		AZ(pfd);
 		Lck_Lock(&bp->mtx);
 	} else {
-		assert (pfd->state == PFD_STATE_USED);
-		VSLb(bo->vsl, SLT_BackendReuse, "%d %s", pfd->fd,
+		assert (PFD_State(pfd) == PFD_STATE_USED);
+		VSLb(bo->vsl, SLT_BackendReuse, "%d %s", *PFD_Fd(pfd),
 		    bp->director->display_name);
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
@@ -230,13 +231,13 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		if (pfd == NULL)
 			return (-1);
 		AN(bo->htc);
-		if (pfd->state != PFD_STATE_STOLEN)
+		if (PFD_State(pfd) != PFD_STATE_STOLEN)
 			extrachance = 0;
 
 		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes,
 				&bo->acct.bereq_bodybytes, 0);
 
-		if (pfd->state != PFD_STATE_USED) {
+		if (PFD_State(pfd) != PFD_STATE_USED) {
 			if (VTP_Wait(wrk, pfd, VTIM_real() +
 			    bo->htc->first_byte_timeout) != 0) {
 				bo->htc->doclose = SC_RX_TIMEOUT;
@@ -247,7 +248,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		}
 
 		if (bo->htc->doclose == SC_NULL) {
-			assert(pfd->state == PFD_STATE_USED);
+			assert(PFD_State(pfd) == PFD_STATE_USED);
 			if (i == 0)
 				i = V1F_FetchRespHdr(bo);
 			if (i == 0) {
@@ -284,9 +285,9 @@ vbe_dir_getip(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
-	CAST_OBJ_NOTNULL(pfd, bo->htc->priv, PFD_MAGIC);
+	pfd = bo->htc->priv;
 
-	return (pfd->priv);
+	return (VTP_getip(pfd));
 }
 
 /*--------------------------------------------------------------------*/
@@ -322,7 +323,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 		i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1);
 		VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk));
 		if (i == 0)
-			V1P_Process(req, pfd->fd, &v1a);
+			V1P_Process(req, *PFD_Fd(pfd), &v1a);
 		VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk));
 		bo->htc->doclose = SC_TX_PIPE;
 		vbe_dir_finish(d, req->wrk, bo);
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index a374f70..b5522dc 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -48,7 +48,6 @@
 #include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
-#include "waiter/waiter.h"
 
 #include "cache_director.h"
 #include "cache_backend.h"
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index fff58c0..73b7f4f 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -44,25 +44,61 @@
 #include "cache_tcp_pool.h"
 #include "cache_pool.h"
 
-typedef int cp_open_f(const struct tcp_pool *, double tmo, const void **privp);
+struct conn_pool;
+
+/*--------------------------------------------------------------------
+ */
+
+struct pfd {
+	unsigned		magic;
+#define PFD_MAGIC		0x0c5e6593
+	int			fd;
+	VTAILQ_ENTRY(pfd)	list;
+	const void		*priv;
+	uint8_t			state;
+	struct waited		waited[1];
+	struct conn_pool	*conn_pool;
+
+	pthread_cond_t		*cond;
+};
+
+unsigned
+PFD_State(const struct pfd *p)
+{
+	CHECK_OBJ_NOTNULL(p, PFD_MAGIC);
+	return (p->state);
+}
+
+int *
+PFD_Fd(struct pfd *p)
+{
+	CHECK_OBJ_NOTNULL(p, PFD_MAGIC);
+	return (&(p->fd));
+}
+
+/*--------------------------------------------------------------------
+ */
+
+typedef int cp_open_f(const struct conn_pool *, double tmo, const void **privp);
 typedef void cp_close_f(struct pfd *);
+typedef int cp_cmp_f(const struct conn_pool *, const void *priv);
 
 struct cp_methods {
 	cp_open_f				*open;
 	cp_close_f				*close;
+	cp_cmp_f				*cmp;
 };
 
-struct tcp_pool {
+struct conn_pool {
 	unsigned				magic;
-#define TCP_POOL_MAGIC				0x28b0e42a
+#define CONN_POOL_MAGIC				0x85099bc3
 
 	const struct cp_methods			*methods;
 
 	const void				*id;
-	struct suckaddr				*ip4;
-	struct suckaddr				*ip6;
+	void					*priv;
 
-	VTAILQ_ENTRY(tcp_pool)			list;
+	VTAILQ_ENTRY(conn_pool)			list;
 	int					refcnt;
 	struct lock				mtx;
 
@@ -75,273 +111,218 @@ struct tcp_pool {
 	int					n_used;
 };
 
-static struct lock		tcp_pools_mtx;
-static VTAILQ_HEAD(, tcp_pool)	tcp_pools = VTAILQ_HEAD_INITIALIZER(tcp_pools);
-
-/*--------------------------------------------------------------------
- */
-
-static int v_matchproto_(cp_open_f)
-vtp_open(const struct tcp_pool *tp, double tmo, const void **privp)
-{
-	int s;
-	int msec;
-
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
-
-	msec = (int)floor(tmo * 1000.0);
-	if (cache_param->prefer_ipv6) {
-		*privp = tp->ip6;
-		s = VTCP_connect(tp->ip6, msec);
-		if (s >= 0)
-			return (s);
-	}
-	*privp = tp->ip4;
-	s = VTCP_connect(tp->ip4, msec);
-	if (s >= 0)
-		return (s);
-	if (!cache_param->prefer_ipv6) {
-		*privp = tp->ip6;
-		s = VTCP_connect(tp->ip6, msec);
-	}
-	return (s);
-}
-
-static void v_matchproto_(cp_close_f)
-vtp_close(struct pfd *pfd)
-{
-
-	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
-	VTCP_close(&pfd->fd);
-}
+struct tcp_pool {
+	unsigned				magic;
+#define TCP_POOL_MAGIC				0x28b0e42a
 
-static const struct cp_methods vtp_methods = {
-	.open = vtp_open,
-	.close = vtp_close,
+	struct suckaddr				*ip4;
+	struct suckaddr				*ip6;
+	struct conn_pool			cp[1];
 };
 
+static struct lock		conn_pools_mtx;
+static VTAILQ_HEAD(, conn_pool)	conn_pools =
+    VTAILQ_HEAD_INITIALIZER(conn_pools);
+
 /*--------------------------------------------------------------------
  * Waiter-handler
  */
 
 static void  v_matchproto_(waiter_handle_f)
-tcp_handle(struct waited *w, enum wait_event ev, double now)
+vcp_handle(struct waited *w, enum wait_event ev, double now)
 {
 	struct pfd *pfd;
-	struct tcp_pool *tp;
+	struct conn_pool *cp;
 
 	CAST_OBJ_NOTNULL(pfd, w->priv1, PFD_MAGIC);
 	(void)ev;
 	(void)now;
-	CHECK_OBJ_NOTNULL(pfd->tcp_pool, TCP_POOL_MAGIC);
-	tp = pfd->tcp_pool;
+	CHECK_OBJ_NOTNULL(pfd->conn_pool, CONN_POOL_MAGIC);
+	cp = pfd->conn_pool;
 
-	Lck_Lock(&tp->mtx);
+	Lck_Lock(&cp->mtx);
 
 	switch (pfd->state) {
 	case PFD_STATE_STOLEN:
 		pfd->state = PFD_STATE_USED;
-		VTAILQ_REMOVE(&tp->connlist, pfd, list);
+		VTAILQ_REMOVE(&cp->connlist, pfd, list);
 		AN(pfd->cond);
 		AZ(pthread_cond_signal(pfd->cond));
 		break;
 	case PFD_STATE_AVAIL:
-		tp->methods->close(pfd);
-		VTAILQ_REMOVE(&tp->connlist, pfd, list);
-		tp->n_conn--;
+		cp->methods->close(pfd);
+		VTAILQ_REMOVE(&cp->connlist, pfd, list);
+		cp->n_conn--;
 		FREE_OBJ(pfd);
 		break;
 	case PFD_STATE_CLEANUP:
-		tp->methods->close(pfd);
-		tp->n_kill--;
-		VTAILQ_REMOVE(&tp->killlist, pfd, list);
+		cp->methods->close(pfd);
+		cp->n_kill--;
+		VTAILQ_REMOVE(&cp->killlist, pfd, list);
 		memset(pfd, 0x11, sizeof *pfd);
 		free(pfd);
 		break;
 	default:
 		WRONG("Wrong pfd state");
 	}
-	Lck_Unlock(&tp->mtx);
+	Lck_Unlock(&cp->mtx);
 }
 
 /*--------------------------------------------------------------------
- * Reference a TCP pool given by {ip4, ip6} pair.  Create if it
- * doesn't exist already.
  */
 
-struct tcp_pool *
-VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id)
+static struct conn_pool *
+VCP_Ref(const void *id, const void *priv)
 {
-	struct tcp_pool *tp;
+	struct conn_pool *cp;
 
-	assert(ip4 != NULL || ip6 != NULL);
-	Lck_Lock(&tcp_pools_mtx);
-	VTAILQ_FOREACH(tp, &tcp_pools, list) {
-		assert(tp->refcnt > 0);
-		if (tp->id != id)
+	Lck_Lock(&conn_pools_mtx);
+	VTAILQ_FOREACH(cp, &conn_pools, list) {
+		assert(cp->refcnt > 0);
+		if (cp->id != id)
 			continue;
-		if (ip4 == NULL) {
-			if (tp->ip4 != NULL)
-				continue;
-		} else {
-			if (tp->ip4 == NULL)
-				continue;
-			if (VSA_Compare(ip4, tp->ip4))
-				continue;
-		}
-		if (ip6 == NULL) {
-			if (tp->ip6 != NULL)
-				continue;
-		} else {
-			if (tp->ip6 == NULL)
-				continue;
-			if (VSA_Compare(ip6, tp->ip6))
-				continue;
-		}
-		tp->refcnt++;
-		Lck_Unlock(&tcp_pools_mtx);
-		return (tp);
+		if (cp->methods->cmp(cp, priv))
+			continue;
+		cp->refcnt++;
+		Lck_Unlock(&conn_pools_mtx);
+		return (cp);
 	}
-	Lck_Unlock(&tcp_pools_mtx);
+	Lck_Unlock(&conn_pools_mtx);
+	return (NULL);
+}
 
-	ALLOC_OBJ(tp, TCP_POOL_MAGIC);
-	AN(tp);
-	tp->methods = &vtp_methods;
-	if (ip4 != NULL)
-		tp->ip4 = VSA_Clone(ip4);
-	if (ip6 != NULL)
-		tp->ip6 = VSA_Clone(ip6);
-	tp->refcnt = 1;
-	tp->id = id;
-	Lck_New(&tp->mtx, lck_tcp_pool);
-	VTAILQ_INIT(&tp->connlist);
-	VTAILQ_INIT(&tp->killlist);
+/*--------------------------------------------------------------------
+ */
 
-	Lck_Lock(&tcp_pools_mtx);
-	VTAILQ_INSERT_HEAD(&tcp_pools, tp, list);
-	Lck_Unlock(&tcp_pools_mtx);
+static void *
+VCP_New(struct conn_pool *cp, const void *id, void *priv,
+    const struct cp_methods *cm)
+{
 
-	return (tp);
+	AN(cp);
+	AN(cm);
+	AN(cm->open);
+	AN(cm->close);
+	AN(cm->cmp);
+
+	INIT_OBJ(cp, CONN_POOL_MAGIC);
+	cp->id = id;
+	cp->priv = priv;
+	cp->methods = cm;
+	cp->refcnt = 1;
+	Lck_New(&cp->mtx, lck_tcp_pool);
+	VTAILQ_INIT(&cp->connlist);
+	VTAILQ_INIT(&cp->killlist);
+
+	Lck_Lock(&conn_pools_mtx);
+	VTAILQ_INSERT_HEAD(&conn_pools, cp, list);
+	Lck_Unlock(&conn_pools_mtx);
+
+	return (priv);
 }
 
+
 /*--------------------------------------------------------------------
- * Add a reference to a tcp_pool
  */
 
-void
-VTP_AddRef(struct tcp_pool *tp)
+static void
+VCP_AddRef(struct conn_pool *cp)
 {
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 
-	Lck_Lock(&tcp_pools_mtx);
-	assert(tp->refcnt > 0);
-	tp->refcnt++;
-	Lck_Unlock(&tcp_pools_mtx);
+	Lck_Lock(&conn_pools_mtx);
+	assert(cp->refcnt > 0);
+	cp->refcnt++;
+	Lck_Unlock(&conn_pools_mtx);
 }
 
 /*--------------------------------------------------------------------
- * Release TCP pool, destroy if last reference.
+ * Release Conn pool, destroy if last reference.
  */
 
-void
-VTP_Rel(struct tcp_pool **tpp)
+static int
+VCP_Rel(struct conn_pool *cp)
 {
-	struct tcp_pool *tp;
 	struct pfd *pfd, *pfd2;
 
-	TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 
-	Lck_Lock(&tcp_pools_mtx);
-	assert(tp->refcnt > 0);
-	if (--tp->refcnt > 0) {
-		Lck_Unlock(&tcp_pools_mtx);
-		return;
+	Lck_Lock(&conn_pools_mtx);
+	assert(cp->refcnt > 0);
+	if (--cp->refcnt > 0) {
+		Lck_Unlock(&conn_pools_mtx);
+		return (1);
 	}
-	AZ(tp->n_used);
-	VTAILQ_REMOVE(&tcp_pools, tp, list);
-	Lck_Unlock(&tcp_pools_mtx);
-
-	free(tp->ip4);
-	free(tp->ip6);
-	Lck_Lock(&tp->mtx);
-	VTAILQ_FOREACH_SAFE(pfd, &tp->connlist, list, pfd2) {
-		VTAILQ_REMOVE(&tp->connlist, pfd, list);
-		tp->n_conn--;
+	AZ(cp->n_used);
+	VTAILQ_REMOVE(&conn_pools, cp, list);
+	Lck_Unlock(&conn_pools_mtx);
+
+	Lck_Lock(&cp->mtx);
+	VTAILQ_FOREACH_SAFE(pfd, &cp->connlist, list, pfd2) {
+		VTAILQ_REMOVE(&cp->connlist, pfd, list);
+		cp->n_conn--;
 		assert(pfd->state == PFD_STATE_AVAIL);
 		pfd->state = PFD_STATE_CLEANUP;
 		(void)shutdown(pfd->fd, SHUT_WR);
-		VTAILQ_INSERT_TAIL(&tp->killlist, pfd, list);
-		tp->n_kill++;
+		VTAILQ_INSERT_TAIL(&cp->killlist, pfd, list);
+		cp->n_kill++;
 	}
-	while (tp->n_kill) {
-		Lck_Unlock(&tp->mtx);
+	while (cp->n_kill) {
+		Lck_Unlock(&cp->mtx);
 		(void)usleep(20000);
-		Lck_Lock(&tp->mtx);
+		Lck_Lock(&cp->mtx);
 	}
-	Lck_Unlock(&tp->mtx);
-	Lck_Delete(&tp->mtx);
-	AZ(tp->n_conn);
-	AZ(tp->n_kill);
-
-	FREE_OBJ(tp);
-}
-
-/*--------------------------------------------------------------------
- * Open a new connection from pool.  This is a distinct function since
- * probing cannot use a recycled connection.
- */
-
-int
-VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp)
-{
-
-	return (vtp_open(tp, tmo, privp));
+	Lck_Unlock(&cp->mtx);
+	Lck_Delete(&cp->mtx);
+	AZ(cp->n_conn);
+	AZ(cp->n_kill);
+	return (0);
 }
 
 /*--------------------------------------------------------------------
  * Recycle a connection.
  */
 
-void
-VTP_Recycle(const struct worker *wrk, struct pfd **pfdp)
+static void
+VCP_Recycle(const struct worker *wrk, struct pfd **pfdp)
 {
 	struct pfd *pfd;
-	struct tcp_pool *tp;
+	struct conn_pool *cp;
 	int i = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	pfd = *pfdp;
 	*pfdp = NULL;
 	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
-	tp = pfd->tcp_pool;
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	cp = pfd->conn_pool;
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 
 	assert(pfd->state == PFD_STATE_USED);
 	assert(pfd->fd > 0);
 
-	Lck_Lock(&tp->mtx);
-	tp->n_used--;
+	Lck_Lock(&cp->mtx);
+	cp->n_used--;
 
 	pfd->waited->priv1 = pfd;
 	pfd->waited->fd = pfd->fd;
 	pfd->waited->idle = VTIM_real();
 	pfd->state = PFD_STATE_AVAIL;
-	pfd->waited->func = tcp_handle;
+	pfd->waited->func = vcp_handle;
 	pfd->waited->tmo = &cache_param->backend_idle_timeout;
 	if (Wait_Enter(wrk->pool->waiter, pfd->waited)) {
-		tp->methods->close(pfd);
+		cp->methods->close(pfd);
 		memset(pfd, 0x33, sizeof *pfd);
 		free(pfd);
 		// XXX: stats
 		pfd = NULL;
 	} else {
-		VTAILQ_INSERT_HEAD(&tp->connlist, pfd, list);
+		VTAILQ_INSERT_HEAD(&cp->connlist, pfd, list);
 		i++;
 	}
 
 	if (pfd != NULL)
-		tp->n_conn++;
-	Lck_Unlock(&tp->mtx);
+		cp->n_conn++;
+	Lck_Unlock(&cp->mtx);
 
 	if (i && DO_DEBUG(DBG_VTC_MODE)) {
 		/*
@@ -361,72 +342,73 @@ VTP_Recycle(const struct worker *wrk, struct pfd **pfdp)
 	}
 }
 
+
 /*--------------------------------------------------------------------
  * Close a connection.
  */
 
-void
-VTP_Close(struct pfd **pfdp)
+static void
+VCP_Close(struct pfd **pfdp)
 {
 	struct pfd *pfd;
-	struct tcp_pool *tp;
+	struct conn_pool *cp;
 
 	pfd = *pfdp;
 	*pfdp = NULL;
 	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
-	tp = pfd->tcp_pool;
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	cp = pfd->conn_pool;
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 
 	assert(pfd->fd > 0);
 
-	Lck_Lock(&tp->mtx);
+	Lck_Lock(&cp->mtx);
 	assert(pfd->state == PFD_STATE_USED || pfd->state == PFD_STATE_STOLEN);
-	tp->n_used--;
+	cp->n_used--;
 	if (pfd->state == PFD_STATE_STOLEN) {
 		(void)shutdown(pfd->fd, SHUT_RDWR);
-		VTAILQ_REMOVE(&tp->connlist, pfd, list);
+		VTAILQ_REMOVE(&cp->connlist, pfd, list);
 		pfd->state = PFD_STATE_CLEANUP;
-		VTAILQ_INSERT_HEAD(&tp->killlist, pfd, list);
-		tp->n_kill++;
+		VTAILQ_INSERT_HEAD(&cp->killlist, pfd, list);
+		cp->n_kill++;
 	} else {
 		assert(pfd->state == PFD_STATE_USED);
-		tp->methods->close(pfd);
+		cp->methods->close(pfd);
 		memset(pfd, 0x44, sizeof *pfd);
 		free(pfd);
 	}
-	Lck_Unlock(&tp->mtx);
+	Lck_Unlock(&cp->mtx);
 }
 
 /*--------------------------------------------------------------------
  * Get a connection
  */
 
-struct pfd *
-VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
+static struct pfd *
+VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk,
     unsigned force_fresh)
 {
 	struct pfd *pfd;
 
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	Lck_Lock(&tp->mtx);
-	pfd = VTAILQ_FIRST(&tp->connlist);
+	Lck_Lock(&cp->mtx);
+	pfd = VTAILQ_FIRST(&cp->connlist);
 	CHECK_OBJ_ORNULL(pfd, PFD_MAGIC);
 	if (force_fresh || pfd == NULL || pfd->state == PFD_STATE_STOLEN)
 		pfd = NULL;
 	else {
-		assert(pfd->tcp_pool == tp);
+		assert(pfd->conn_pool == cp);
 		assert(pfd->state == PFD_STATE_AVAIL);
-		VTAILQ_REMOVE(&tp->connlist, pfd, list);
-		VTAILQ_INSERT_TAIL(&tp->connlist, pfd, list);
-		tp->n_conn--;
+		VTAILQ_REMOVE(&cp->connlist, pfd, list);
+		VTAILQ_INSERT_TAIL(&cp->connlist, pfd, list);
+		cp->n_conn--;
 		VSC_C_main->backend_reuse++;
 		pfd->state = PFD_STATE_STOLEN;
 		pfd->cond = &wrk->cond;
 	}
-	tp->n_used++;			// Opening mostly works
-	Lck_Unlock(&tp->mtx);
+	cp->n_used++;			// Opening mostly works
+	Lck_Unlock(&cp->mtx);
 
 	if (pfd != NULL)
 		return (pfd);
@@ -435,13 +417,13 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
 	AN(pfd);
 	INIT_OBJ(pfd->waited, WAITED_MAGIC);
 	pfd->state = PFD_STATE_USED;
-	pfd->tcp_pool = tp;
-	pfd->fd = tp->methods->open(tp, tmo, &pfd->priv);
+	pfd->conn_pool = cp;
+	pfd->fd = cp->methods->open(cp, tmo, &pfd->priv);
 	if (pfd->fd < 0) {
 		FREE_OBJ(pfd);
-		Lck_Lock(&tp->mtx);
-		tp->n_used--;		// Nope, didn't work after all.
-		Lck_Unlock(&tp->mtx);
+		Lck_Lock(&cp->mtx);
+		cp->n_used--;		// Nope, didn't work after all.
+		Lck_Unlock(&cp->mtx);
 	} else
 		VSC_C_main->backend_conn++;
 
@@ -451,39 +433,244 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
 /*--------------------------------------------------------------------
  */
 
-int
-VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+static int
+VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
 {
-	struct tcp_pool *tp;
+	struct conn_pool *cp;
 	int r;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
-	tp = pfd->tcp_pool;
-	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	cp = pfd->conn_pool;
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
 	assert(pfd->cond == &wrk->cond);
-	Lck_Lock(&tp->mtx);
+	Lck_Lock(&cp->mtx);
 	while (pfd->state == PFD_STATE_STOLEN) {
-		r = Lck_CondWait(&wrk->cond, &tp->mtx, tmo);
+		r = Lck_CondWait(&wrk->cond, &cp->mtx, tmo);
 		if (r != 0) {
 			if (r == EINTR)
 				continue;
 			assert(r == ETIMEDOUT);
-			Lck_Unlock(&tp->mtx);
+			Lck_Unlock(&cp->mtx);
 			return (1);
 		}
 	}
 	assert(pfd->state == PFD_STATE_USED);
 	pfd->cond = NULL;
-	Lck_Unlock(&tp->mtx);
+	Lck_Unlock(&cp->mtx);
 
 	return (0);
 }
 
+/*--------------------------------------------------------------------
+ */
+
+static int v_matchproto_(cp_open_f)
+vtp_open(const struct conn_pool *cp, double tmo, const void **privp)
+{
+	int s;
+	int msec;
+	struct tcp_pool *tp;
+
+	CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC);
+	CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC);
+
+	msec = (int)floor(tmo * 1000.0);
+	if (cache_param->prefer_ipv6) {
+		*privp = tp->ip6;
+		s = VTCP_connect(tp->ip6, msec);
+		if (s >= 0)
+			return (s);
+	}
+	*privp = tp->ip4;
+	s = VTCP_connect(tp->ip4, msec);
+	if (s >= 0)
+		return (s);
+	if (!cache_param->prefer_ipv6) {
+		*privp = tp->ip6;
+		s = VTCP_connect(tp->ip6, msec);
+	}
+	return (s);
+}
+
+static void v_matchproto_(cp_close_f)
+vtp_close(struct pfd *pfd)
+{
+
+	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
+	VTCP_close(&pfd->fd);
+}
+
+struct vtp_cs {
+	unsigned			magic;
+#define VTP_CS_MAGIC			0xc1e40447
+	const struct suckaddr		*ip4;
+	const struct suckaddr		*ip6;
+};
+
+static int v_matchproto_(cp_cmp_f)
+vtp_cmp(const struct conn_pool *cp, const void *priv)
+{
+	const struct vtp_cs *vcs;
+	const struct tcp_pool *tp;
+
+	CAST_OBJ_NOTNULL(vcs, priv, VTP_CS_MAGIC);
+	CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC);
+	if (tp->ip4 == NULL && vcs->ip4 != NULL)
+		return (1);
+	if (tp->ip4 != NULL && vcs->ip4 == NULL)
+		return (1);
+	if (tp->ip6 == NULL && vcs->ip6 != NULL)
+		return (1);
+	if (tp->ip6 != NULL && vcs->ip6 == NULL)
+		return (1);
+	if (tp->ip4 != NULL && vcs->ip4 != NULL &&
+	    VSA_Compare(tp->ip4, vcs->ip4))
+		return (1);
+	if (tp->ip6 != NULL && vcs->ip6 != NULL &&
+	    VSA_Compare(tp->ip6, vcs->ip6))
+		return (1);
+	return (0);
+}
+
+static const struct cp_methods vtp_methods = {
+	.open = vtp_open,
+	.close = vtp_close,
+	.cmp = vtp_cmp,
+};
+
+
+/*--------------------------------------------------------------------
+ * Reference a TCP pool given by {ip4, ip6} pair.  Create if it
+ * doesn't exist already.
+ */
+
+struct tcp_pool *
+VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id)
+{
+	struct tcp_pool *tp;
+	struct conn_pool *cp;
+	struct vtp_cs vcs;
+
+	assert(ip4 != NULL || ip6 != NULL);
+	INIT_OBJ(&vcs, VTP_CS_MAGIC);
+	vcs.ip4 = ip4;
+	vcs.ip6 = ip6;
+
+	cp = VCP_Ref(id, &vcs);
+	if (cp != NULL)
+		return (cp->priv);
+
+	ALLOC_OBJ(tp, TCP_POOL_MAGIC);
+	AN(tp);
+	if (ip4 != NULL)
+		tp->ip4 = VSA_Clone(ip4);
+	if (ip6 != NULL)
+		tp->ip6 = VSA_Clone(ip6);
+	return(VCP_New(tp->cp, id, tp, &vtp_methods));
+}
+
+/*--------------------------------------------------------------------
+ * Add a reference to a tcp_pool
+ */
+
+void
+VTP_AddRef(struct tcp_pool *tp)
+{
+	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	VCP_AddRef(tp->cp);
+}
+
+/*--------------------------------------------------------------------
+ * Release TCP pool, destroy if last reference.
+ */
+
+void
+VTP_Rel(struct tcp_pool **tpp)
+{
+	struct tcp_pool *tp;
+
+	TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC);
+	if (VCP_Rel(tp->cp))
+		return;
+
+	free(tp->ip4);
+	free(tp->ip6);
+	FREE_OBJ(tp);
+}
+
+/*--------------------------------------------------------------------
+ * Open a new connection from pool.  This is a distinct function since
+ * probing cannot use a recycled connection.
+ */
+
+int
+VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp)
+{
+
+	return (vtp_open(tp->cp, tmo, privp));
+}
+
+/*--------------------------------------------------------------------
+ * Recycle a connection.
+ */
+
+void
+VTP_Recycle(const struct worker *wrk, struct pfd **pfdp)
+{
+
+	VCP_Recycle(wrk, pfdp);
+}
+
+/*--------------------------------------------------------------------
+ * Close a connection.
+ */
+
+void
+VTP_Close(struct pfd **pfdp)
+{
+
+	VCP_Close(pfdp);
+}
+
+/*--------------------------------------------------------------------
+ * Get a connection
+ */
+
+struct pfd *
+VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
+    unsigned force_fresh)
+{
+
+	return VCP_Get(tp->cp, tmo, wrk, force_fresh);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+int
+VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+{
+	return (VCP_Wait(wrk, pfd, tmo));
+}
+
+/*--------------------------------------------------------------------
+ */
+
+const struct suckaddr *
+VTP_getip(struct pfd *pfd)
+{
+	struct tcp_pool *tp;
+
+	CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC);
+	CAST_OBJ_NOTNULL(tp, pfd->conn_pool->priv, TCP_POOL_MAGIC);
+	return (pfd->priv);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
 VTP_Init(void)
 {
-	Lck_New(&tcp_pools_mtx, lck_tcp_pool);
+	Lck_New(&conn_pools_mtx, lck_tcp_pool);
 }
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index ceae8a9..f5e25c5 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -31,25 +31,20 @@
  */
 
 struct tcp_pool;
-
-struct pfd {
-	unsigned		magic;
-#define PFD_MAGIC		0x0c5e6593
-	int			fd;
-	VTAILQ_ENTRY(pfd)	list;
-	const void		*priv;
-	uint8_t			state;
+struct pfd;
 #define PFD_STATE_AVAIL		(1<<0)
 #define PFD_STATE_USED		(1<<1)
 #define PFD_STATE_STOLEN	(1<<2)
 #define PFD_STATE_CLEANUP	(1<<3)
-	struct waited		waited[1];
-	struct tcp_pool		*tcp_pool;
 
-	pthread_cond_t		*cond;
-};
+/*---------------------------------------------------------------------
+ */
+
+unsigned PFD_State(const struct pfd *);
+int *PFD_Fd(struct pfd *);
 
 /*---------------------------------------------------------------------
+
  * Prototypes
  */
 
@@ -98,3 +93,6 @@ int VTP_Wait(struct worker *, struct pfd *, double tmo);
 	 * If the connection was recycled (state != VTP_STATE_USED) call this
 	 * function before attempting to receive on the connection.
 	 */
+
+const struct suckaddr *VTP_getip(struct pfd *);
+
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 0925a2b..8ec868d 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -94,6 +94,7 @@
 
 +libh mgt_event.h
 
+-sem(VCP_New, custodial(3))
 -sem(vsmw_addseg, custodial(2))
 -sem(BAN_Free, custodial(1))
 -sem(EXP_Inject, custodial(1))


More information about the varnish-commit mailing list