varnish-cache/bin/varnishd/http1/cache_http1_fetch.c
1
/*-
2
 * Copyright (c) 2006 Verdens Gang AS
3
 * Copyright (c) 2006-2015 Varnish Software AS
4
 * All rights reserved.
5
 *
6
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
7
 *
8
 * SPDX-License-Identifier: BSD-2-Clause
9
 *
10
 * Redistribution and use in source and binary forms, with or without
11
 * modification, are permitted provided that the following conditions
12
 * are met:
13
 * 1. Redistributions of source code must retain the above copyright
14
 *    notice, this list of conditions and the following disclaimer.
15
 * 2. Redistributions in binary form must reproduce the above copyright
16
 *    notice, this list of conditions and the following disclaimer in the
17
 *    documentation and/or other materials provided with the distribution.
18
 *
19
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
23
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29
 * SUCH DAMAGE.
30
 */
31
32
#include "config.h"
33
34
#include "cache/cache_varnishd.h"
35
#include "cache/cache_filter.h"
36
37
#include <stdio.h>
38
#include <stdlib.h>
39
40
#include "vtcp.h"
41
#include "vtim.h"
42
43
#include "cache_http1.h"
44
45
/*--------------------------------------------------------------------
46
 * Pass the request body to the backend
47
 */
48
49
static int v_matchproto_(objiterate_f)
50 34
vbf_iter_req_body(void *priv, unsigned flush, const void *ptr, ssize_t l)
51
{
52
        struct busyobj *bo;
53
54 34
        CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
55
56 34
        if (l > 0) {
57 34
                (void)V1L_Write(bo->wrk, ptr, l);
58 34
                if (flush && V1L_Flush(bo->wrk))
59 0
                        return (-1);
60 34
        }
61 34
        return (0);
62 34
}
63
64
/*--------------------------------------------------------------------
65
 * Send request to backend, including any (cached) req.body
66
 *
67
 * Return value:
68
 *       0 success
69
 *       1 failure
70
 */
71
72
int
73 1869
V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
74
    uint64_t *ctr_bodybytes)
75
{
76
        struct http *hp;
77
        enum sess_close sc;
78
        ssize_t i;
79
        uint64_t bytes, hdrbytes;
80
        struct http_conn *htc;
81 1869
        int do_chunked = 0;
82
83 1869
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
84 1871
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
85 1871
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
86 1871
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
87 1871
        AN(ctr_hdrbytes);
88 1871
        AN(ctr_bodybytes);
89
90 1871
        htc = bo->htc;
91 1871
        assert(*htc->rfd > 0);
92 1871
        hp = bo->bereq;
93
94 1871
        if (bo->req != NULL && !bo->req->req_body_status->length_known) {
95 7
                http_PrintfHeader(hp, "Transfer-Encoding: chunked");
96 7
                do_chunked = 1;
97 7
        }
98
99 1871
        VTCP_blocking(*htc->rfd);       /* XXX: we should timeout instead */
100
        /* XXX: need a send_timeout for the backend side */
101 1871
        V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, nan(""), 0);
102 1871
        hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
103
104
        /* Deal with any message-body the request might (still) have */
105 1871
        i = 0;
106
107 1871
        if (bo->bereq_body != NULL) {
108 6
                AZ(bo->req);
109 6
                AZ(do_chunked);
110 12
                (void)ObjIterate(bo->wrk, bo->bereq_body,
111 6
                    bo, vbf_iter_req_body, 0);
112 1871
        } else if (bo->req != NULL &&
113 40
            bo->req->req_body_status != BS_NONE) {
114 27
                if (do_chunked)
115 7
                        V1L_Chunked(wrk);
116 27
                i = VRB_Iterate(wrk, bo->vsl, bo->req, vbf_iter_req_body, bo);
117
118 27
                if (bo->req->req_body_status != BS_CACHED)
119 26
                        bo->no_retry = "req.body not cached";
120
121 27
                if (bo->req->req_body_status == BS_ERROR) {
122
                        /*
123
                         * XXX: (#2332) We should test to see if the backend
124
                         * XXX: sent us some headers explaining why.
125
                         * XXX: This is hard because of the mistaken API split
126
                         * XXX: between cache_backend.c and V1F, and therefore
127
                         * XXX: Parked in this comment, pending renovation of
128
                         * XXX: the VDI/backend-protocol API to allow non-H1
129
                         * XXX: backends.
130
                         */
131 5
                        assert(i < 0);
132 10
                        VSLb(bo->vsl, SLT_FetchError,
133
                            "req.body read error: %d (%s)",
134 5
                            errno, vstrerror(errno));
135 5
                        bo->req->doclose = SC_RX_BODY;
136 5
                }
137 27
                if (do_chunked)
138 7
                        V1L_EndChunk(wrk);
139 27
        }
140
141 1871
        sc = V1L_Close(wrk, &bytes);
142
143
        /* Bytes accounting */
144 1871
        if (bytes < hdrbytes)
145 0
                *ctr_hdrbytes += bytes;
146
        else {
147 1871
                *ctr_hdrbytes += hdrbytes;
148 1871
                *ctr_bodybytes += bytes - hdrbytes;
149
        }
150
151 1871
        if (sc == SC_NULL && i < 0)
152 5
                sc = SC_TX_ERROR;
153
154 1871
        if (sc != SC_NULL) {
155 10
                VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
156 5
                    errno, vstrerror(errno));
157 5
                VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
158 5
                htc->doclose = sc;
159 5
                return (-1);
160
        }
161 1866
        VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
162 1866
        return (0);
163 1871
}
164
165
int
166 1851
V1F_FetchRespHdr(struct busyobj *bo)
167
{
168
169
        struct http *hp;
170
        int i;
171
        double t;
172
        struct http_conn *htc;
173
        enum htc_status_e hs;
174
175 1851
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
176 1851
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
177 1851
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
178
179 1851
        htc = bo->htc;
180 1851
        assert(*htc->rfd > 0);
181
182 1851
        VSC_C_main->backend_req++;
183
184
        /* Receive response */
185
186 1851
        HTC_RxInit(htc, bo->ws);
187 1851
        CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
188 1851
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
189
190 1851
        t = VTIM_real() + htc->first_byte_timeout;
191 3702
        hs = HTC_RxStuff(htc, HTTP1_Complete, NULL, NULL,
192 1851
            t, NAN, htc->between_bytes_timeout, cache_param->http_resp_size);
193 1851
        if (hs != HTC_S_COMPLETE) {
194 31
                bo->acct.beresp_hdrbytes +=
195 31
                    htc->rxbuf_e - htc->rxbuf_b;
196 31
                switch (hs) {
197
                case HTC_S_JUNK:
198 0
                        VSLb(bo->vsl, SLT_FetchError, "Received junk");
199 0
                        htc->doclose = SC_RX_JUNK;
200 0
                        break;
201
                case HTC_S_CLOSE:
202 0
                        VSLb(bo->vsl, SLT_FetchError, "backend closed");
203 0
                        htc->doclose = SC_RESP_CLOSE;
204 0
                        break;
205
                case HTC_S_TIMEOUT:
206 1
                        VSLb(bo->vsl, SLT_FetchError, "timeout");
207 1
                        htc->doclose = SC_RX_TIMEOUT;
208 1
                        break;
209
                case HTC_S_OVERFLOW:
210 14
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
211 14
                        htc->doclose = SC_RX_OVERFLOW;
212 14
                        break;
213
                case HTC_S_IDLE:
214 4
                        VSLb(bo->vsl, SLT_FetchError, "first byte timeout");
215 4
                        htc->doclose = SC_RX_TIMEOUT;
216 4
                        break;
217
                default:
218 24
                        VSLb(bo->vsl, SLT_FetchError, "HTC %s (%d)",
219 12
                             HTC_Status(hs), hs);
220 12
                        htc->doclose = SC_RX_BAD;
221 12
                        break;
222
                }
223 31
                return (htc->rxbuf_e == htc->rxbuf_b ? 1 : -1);
224
        }
225 1820
        VTCP_set_read_timeout(*htc->rfd, htc->between_bytes_timeout);
226
227 1820
        hp = bo->beresp;
228
229 1820
        i = HTTP1_DissectResponse(htc, hp, bo->bereq);
230 1820
        bo->acct.beresp_hdrbytes += htc->rxbuf_e - htc->rxbuf_b;
231 1820
        if (i) {
232 17
                VSLb(bo->vsl, SLT_FetchError, "http format error");
233 17
                htc->doclose = SC_RX_JUNK;
234 17
                return (-1);
235
        }
236
237 1803
        htc->doclose = http_DoConnection(hp);
238
239
        /*
240
         * Figure out how the fetch is supposed to happen, before the
241
         * headers are adultered by VCL
242
         */
243 1803
        if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
244
                /*
245
                 * A HEAD request can never have a body in the reply,
246
                 * no matter what the headers might say.
247
                 * [RFC7231 4.3.2 p25]
248
                 */
249 2
                bo->wrk->stats->fetch_head++;
250 2
                bo->htc->body_status = BS_NONE;
251 1803
        } else if (http_GetStatus(bo->beresp) <= 199) {
252
                /*
253
                 * 1xx responses never have a body.
254
                 * [RFC7230 3.3.2 p31]
255
                 * ... but we should never see them.
256
                 */
257 0
                bo->wrk->stats->fetch_1xx++;
258 0
                bo->htc->body_status = BS_ERROR;
259 1801
        } else if (http_IsStatus(bo->beresp, 204)) {
260
                /*
261
                 * 204 is "No Content", obviously don't expect a body.
262
                 * [RFC7230 3.3.1 p29 and 3.3.2 p31]
263
                 */
264 5
                bo->wrk->stats->fetch_204++;
265 7
                if ((http_GetHdr(bo->beresp, H_Content_Length, NULL) &&
266 4
                    bo->htc->content_length != 0) ||
267 3
                    http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
268 2
                        bo->htc->body_status = BS_ERROR;
269
                else
270 3
                        bo->htc->body_status = BS_NONE;
271 1801
        } else if (http_IsStatus(bo->beresp, 304)) {
272
                /*
273
                 * 304 is "Not Modified" it has no body.
274
                 * [RFC7230 3.3 p28]
275
                 */
276 27
                bo->wrk->stats->fetch_304++;
277 27
                bo->htc->body_status = BS_NONE;
278 1796
        } else if (bo->htc->body_status == BS_CHUNKED) {
279 138
                bo->wrk->stats->fetch_chunked++;
280 1769
        } else if (bo->htc->body_status == BS_LENGTH) {
281 964
                assert(bo->htc->content_length > 0);
282 964
                bo->wrk->stats->fetch_length++;
283 1631
        } else if (bo->htc->body_status == BS_EOF) {
284 22
                bo->wrk->stats->fetch_eof++;
285 667
        } else if (bo->htc->body_status == BS_ERROR) {
286 4
                bo->wrk->stats->fetch_bad++;
287 645
        } else if (bo->htc->body_status == BS_NONE) {
288 641
                bo->wrk->stats->fetch_none++;
289 641
        } else {
290 0
                WRONG("wrong bodystatus");
291
        }
292
293 1803
        assert(bo->vfc->resp == bo->beresp);
294 1803
        if (bo->htc->body_status != BS_NONE &&
295 1130
            bo->htc->body_status != BS_ERROR)
296 1124
                if (V1F_Setup_Fetch(bo->vfc, bo->htc)) {
297 16
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
298 16
                        htc->doclose = SC_RX_OVERFLOW;
299 16
                        return (-1);
300
                }
301
302 1787
        return (0);
303 1851
}