[master] ac94f16 Now really make chunked req.body work with pass.

Poul-Henning Kamp phk at FreeBSD.org
Mon Jun 23 14:30:24 CEST 2014


commit ac94f1662761f9d9b7e3f527a08476a5cddb7dfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 12:29:33 2014 +0000

    Now really make chunked req.body work with pass.
    
    Please test this ASAP (before 4.0.1)

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 30fcaea..cfb2238 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -60,12 +60,6 @@ sub vcl_recv {
         return (pipe);
     }
 
-    /* We don't support chunked uploads, except when piping. */
-    if ((req.method == "POST" || req.method == "PUT") &&
-      req.http.transfer-encoding ~ "chunked") {
-        return(pipe);
-    }
-
     if (req.method != "GET" && req.method != "HEAD") {
         /* We only deal with GET and HEAD by default */
         return (pass);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index aa009f2..b6d72dd 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -192,6 +192,31 @@ V1F_Setup_Fetch(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Pass the request body to the backend with chunks
+ */
+
+static int __match_proto__(req_body_iter_f)
+vbf_iter_req_body_chunked(struct req *req, void *priv, void *ptr, size_t l)
+{
+	struct worker *wrk;
+	char buf[20];
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC);
+
+	if (l > 0) {
+		bprintf(buf, "%jx\r\n", (uintmax_t)l);
+		VSLb(req->vsl, SLT_Debug, "WWWW: %s", buf);
+		(void)WRW_Write(wrk, buf, strlen(buf));
+		(void)WRW_Write(wrk, ptr, l);
+		(void)WRW_Write(wrk, "\r\n", 2);
+		if (WRW_Flush(wrk))
+			return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Pass the request body to the backend
  */
 
@@ -231,6 +256,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	int i, j, first;
 	struct http_conn *htc;
 	ssize_t hdrbytes;
+	int do_chunked = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -262,6 +288,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	if (!http_GetHdr(bo->bereq, H_Host, NULL))
 		VDI_AddHostHeader(bo->bereq, vc);
 
+	if (req != NULL && req->req_body_status == REQ_BODY_CHUNKED) {
+		http_PrintfHeader(hp, "Transfer-Encoding: chunked");
+		do_chunked = 1;
+	}
+
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
 	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
@@ -270,10 +301,16 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	i = 0;
 
 	if (req != NULL) {
-		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
-		if (req->req_body_status == REQ_BODY_DONE)
+		if (do_chunked) {
+			i = HTTP1_IterateReqBody(req,
+			    vbf_iter_req_body_chunked, wrk);
+			(void)WRW_Write(wrk, "0\r\n\r\n", 5);
+		} else {
+			i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
+		}
+		if (req->req_body_status == REQ_BODY_DONE) {
 			retry = -1;
-		if (req->req_body_status == REQ_BODY_FAIL) {
+		} else if (req->req_body_status == REQ_BODY_FAIL) {
 			VSLb(bo->vsl, SLT_FetchError,
 			    "req.body read error: %d (%s)",
 			    errno, strerror(errno));
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index dac34e0..454ec28 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,8 +285,10 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
-	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked")) {
+		req->chunk_ctr = -1;
 		return (REQ_BODY_CHUNKED);
+	}
 	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
@@ -512,6 +514,7 @@ http1_iter_req_body(struct req *req, enum req_body_state_e bs,
 		switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
 		    &req->acct.req_bodybytes, buf, &len)) {
 		case H1CR_ERROR:
+			VSLb(req->vsl, SLT_Debug, "CHUNKERR: %s", err);
 			return (-1);
 		case H1CR_MORE:
 			return (len);
@@ -639,7 +642,7 @@ int
 HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 {
 	struct storage *st;
-	ssize_t l;
+	ssize_t l, l2;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
@@ -661,6 +664,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		req->req_body_status = REQ_BODY_FAIL;
 		return (-1);
 	}
+	l2 = 0;
 
 	st = NULL;
 	do {
@@ -690,13 +694,28 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 			break;
 		}
 		if (l > 0) {
+			l2 += l;
 			st->len += l;
 			if (st->space == st->len)
 				st = NULL;
 		}
 	} while (l > 0);
-	if (l == 0)
+	if (l == 0) {
+		req->req_bodybytes = l2;
+		/* We must update also the "pristine" req.* copy */
+
+		http_Unset(req->http0, H_Content_Length);
+		http_Unset(req->http0, H_Transfer_Encoding);
+		http_PrintfHeader(req->http0, "Content-Length: %ju",
+		    req->req_bodybytes);
+
+		http_Unset(req->http, H_Content_Length);
+		http_Unset(req->http, H_Transfer_Encoding);
+		http_PrintfHeader(req->http, "Content-Length: %ju",
+		    req->req_bodybytes);
+
 		req->req_body_status = REQ_BODY_CACHED;
+	}
 	VSLb_ts_req(req, "ReqBody", VTIM_real());
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a3e2005..f0a1abc 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -196,7 +196,7 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 {
 	size_t l;
 	unsigned char *p;
-	ssize_t i;
+	ssize_t i = 0;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	l = 0;
@@ -212,12 +212,12 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 		if (htc->pipeline.b == htc->pipeline.e)
 			htc->pipeline.b = htc->pipeline.e = NULL;
 	}
-	if (len == 0)
-		return (l);
-	i = read(htc->fd, p, len);
-	if (i < 0) {
-		VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
-		return (i);
+	if (len > 0) {
+		i = read(htc->fd, p, len);
+		if (i < 0) {
+			VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
+			return (i);
+		}
 	}
 	return (i + l);
 }
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index e02bc47..0344d02 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -25,7 +25,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
 varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
-	txreq -body "FOO"
+	txreq -req "POST" -body "FOO"
 	rxresp
 	expect resp.http.Foo == "Foo"
 	expect resp.bodylen == 2
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
index b57f904..73bcb34 100644
--- a/bin/varnishtest/tests/c00067.vtc
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -4,11 +4,16 @@ server s1 {
 	rxreq
 	expect req.bodylen == 106
 	txresp -body "ABCD"
+	rxreq
+	expect req.bodylen == 108
+	txresp -body "ABCDE"
 } -start
 
 varnish v1 -vcl+backend {
 } -start
 
+varnish v1 -cliok "param.set debug +syncvsl"
+
 client c1 {
 	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
 	chunked {BLA}
@@ -22,3 +27,25 @@ client c1 {
 	expect resp.status == 200
 	expect resp.bodylen == 4
 } -run
+
+delay .2
+
+varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
+	sub vcl_recv {
+		C{ VRT_CacheReqBody(ctx, 1000); }C
+	}
+} 
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLAS}
+	delay .2
+	chunkedlen 100
+	delay .2
+	chunked {TFOO}
+	delay .2
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 966f043..28b7340 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -458,7 +458,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 		return;
 	}
 	p = http_find_header(hh, "transfer-encoding");
-	if (p != NULL && !strcmp(p, "chunked")) {
+	if (p != NULL && !strcasecmp(p, "chunked")) {
 		while (http_rxchunk(hp) != 0)
 			continue;
 		vtc_dump(hp->vl, 4, "body", hp->body, ll);



More information about the varnish-commit mailing list