varnish-cache/bin/varnishd/http1/cache_http1_fetch.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2015 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 */
30
31
#include "config.h"
32
33
#include "cache/cache_varnishd.h"
34
#include "cache/cache_filter.h"
35
36
#include <stdio.h>
37
#include <stdlib.h>
38
39
#include "vtcp.h"
40
#include "vtim.h"
41
42
#include "cache_http1.h"
43
44
/*--------------------------------------------------------------------
45
 * Pass the request body to the backend
46
 */
47
48
static int v_matchproto_(objiterate_f)
49 2401
vbf_iter_req_body(void *priv, unsigned flush, const void *ptr, ssize_t l)
50
{
51
        struct busyobj *bo;
52
53 2401
        CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
54
55 2401
        if (l > 0) {
56 2277
                (void)V1L_Write(bo->wrk, ptr, l);
57 2277
                if (flush && V1L_Flush(bo->wrk) != SC_NULL)
58 0
                        return (-1);
59 2277
        }
60 2401
        return (0);
61 2401
}
62
63
/*--------------------------------------------------------------------
64
 * Send request to backend, including any (cached) req.body
65
 *
66
 * Return value:
67
 *       0 success
68
 *       1 failure
69
 */
70
71
int
72 51150
V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
73
    uint64_t *ctr_bodybytes)
74
{
75
        struct http *hp;
76
        stream_close_t sc;
77
        ssize_t i;
78
        uint64_t bytes, hdrbytes;
79
        struct http_conn *htc;
80 51150
        int do_chunked = 0;
81
82 51150
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
83 51150
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
84 51150
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
85 51150
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
86 51150
        AN(ctr_hdrbytes);
87 51150
        AN(ctr_bodybytes);
88
89 51150
        htc = bo->htc;
90 51150
        assert(*htc->rfd > 0);
91 51150
        hp = bo->bereq;
92
93 51150
        if (bo->req != NULL && !bo->req->req_body_status->length_known) {
94 250
                http_PrintfHeader(hp, "Transfer-Encoding: chunked");
95 250
                do_chunked = 1;
96 250
        }
97
98 51150
        VTCP_blocking(*htc->rfd);       /* XXX: we should timeout instead */
99
        /* XXX: need a send_timeout for the backend side */
100 51150
        V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, nan(""), 0);
101 51150
        hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
102
103
        /* Deal with any message-body the request might (still) have */
104 51150
        i = 0;
105
106 51150
        if (bo->bereq_body != NULL) {
107 303
                AZ(bo->req);
108 303
                AZ(do_chunked);
109 606
                (void)ObjIterate(bo->wrk, bo->bereq_body,
110 303
                    bo, vbf_iter_req_body, 0);
111 51150
        } else if (bo->req != NULL &&
112 1775
            bo->req->req_body_status != BS_NONE) {
113 1200
                if (do_chunked)
114 250
                        V1L_Chunked(wrk);
115 1200
                i = VRB_Iterate(wrk, bo->vsl, bo->req, vbf_iter_req_body, bo);
116
117 1200
                if (bo->req->req_body_status != BS_CACHED)
118 1175
                        bo->no_retry = "req.body not cached";
119
120 1200
                if (bo->req->req_body_status == BS_ERROR) {
121
                        /*
122
                         * XXX: (#2332) We should test to see if the backend
123
                         * XXX: sent us some headers explaining why.
124
                         * XXX: This is hard because of the mistaken API split
125
                         * XXX: between cache_backend.c and V1F, and therefore
126
                         * XXX: Parked in this comment, pending renovation of
127
                         * XXX: the VDI/backend-protocol API to allow non-H1
128
                         * XXX: backends.
129
                         */
130 275
                        assert(i < 0);
131 550
                        VSLb(bo->vsl, SLT_FetchError,
132
                            "req.body read error: %d (%s)",
133 275
                            errno, VAS_errtxt(errno));
134 275
                        bo->req->doclose = SC_RX_BODY;
135 275
                }
136 1200
                if (do_chunked)
137 250
                        V1L_EndChunk(wrk);
138 1200
        }
139
140 51150
        sc = V1L_Close(wrk, &bytes);
141 51150
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
142
143
        /* Bytes accounting */
144 51150
        if (bytes < hdrbytes)
145 0
                *ctr_hdrbytes += bytes;
146
        else {
147 51150
                *ctr_hdrbytes += hdrbytes;
148 51150
                *ctr_bodybytes += bytes - hdrbytes;
149
        }
150
151 51150
        if (sc == SC_NULL && i < 0)
152 275
                sc = SC_TX_ERROR;
153
154 51150
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
155 51150
        if (sc != SC_NULL) {
156 550
                VSLb(bo->vsl, SLT_FetchError,
157
                    "backend write error: %d (%s) (%s)",
158 275
                    errno, VAS_errtxt(errno), sc->desc);
159 275
                VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
160 275
                htc->doclose = sc;
161 275
                return (-1);
162
        }
163 50875
        CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
164 50875
        VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
165 50875
        return (0);
166 51150
}
167
168
int
169 50347
V1F_FetchRespHdr(struct busyobj *bo)
170
{
171
172
        struct http *hp;
173
        int i;
174
        double t;
175
        struct http_conn *htc;
176
        enum htc_status_e hs;
177
        const char *name, *desc;
178
179 50347
        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
180 50347
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
181 50347
        CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
182
183 50347
        htc = bo->htc;
184 50347
        assert(*htc->rfd > 0);
185
186 50347
        VSC_C_main->backend_req++;
187
188
        /* Receive response */
189
190 50347
        HTC_RxInit(htc, bo->ws);
191 50347
        CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
192 50347
        CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
193
194 50347
        t = VTIM_real() + htc->first_byte_timeout;
195 100694
        hs = HTC_RxStuff(htc, HTTP1_Complete, NULL, NULL,
196 50347
            t, NAN, htc->between_bytes_timeout, cache_param->http_resp_size);
197 50347
        if (hs != HTC_S_COMPLETE) {
198 1001
                bo->acct.beresp_hdrbytes +=
199 1001
                    htc->rxbuf_e - htc->rxbuf_b;
200 1001
                switch (hs) {
201
                case HTC_S_JUNK:
202 0
                        VSLb(bo->vsl, SLT_FetchError, "Received junk");
203 0
                        htc->doclose = SC_RX_JUNK;
204 0
                        break;
205
                case HTC_S_CLOSE:
206 0
                        VSLb(bo->vsl, SLT_FetchError, "backend closed");
207 0
                        htc->doclose = SC_RESP_CLOSE;
208 0
                        break;
209
                case HTC_S_TIMEOUT:
210 25
                        VSLb(bo->vsl, SLT_FetchError, "timeout");
211 25
                        htc->doclose = SC_RX_TIMEOUT;
212 25
                        break;
213
                case HTC_S_OVERFLOW:
214 500
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
215 500
                        htc->doclose = SC_RX_OVERFLOW;
216 500
                        break;
217
                case HTC_S_IDLE:
218 125
                        VSLb(bo->vsl, SLT_FetchError, "first byte timeout");
219 125
                        htc->doclose = SC_RX_TIMEOUT;
220 125
                        break;
221
                default:
222 351
                        HTC_Status(hs, &name, &desc);
223 702
                        VSLb(bo->vsl, SLT_FetchError, "HTC %s (%s)",
224 351
                             name, desc);
225 351
                        htc->doclose = SC_RX_BAD;
226 351
                        break;
227
                }
228 1001
                return (htc->rxbuf_e == htc->rxbuf_b ? 1 : -1);
229
        }
230 49346
        VTCP_set_read_timeout(*htc->rfd, htc->between_bytes_timeout);
231
232 49346
        hp = bo->beresp;
233
234 49346
        i = HTTP1_DissectResponse(htc, hp, bo->bereq);
235 49346
        bo->acct.beresp_hdrbytes += htc->rxbuf_e - htc->rxbuf_b;
236 49346
        if (i) {
237 425
                VSLb(bo->vsl, SLT_FetchError, "http format error");
238 425
                htc->doclose = SC_RX_JUNK;
239 425
                return (-1);
240
        }
241
242 48921
        htc->doclose = http_DoConnection(hp, SC_RESP_CLOSE);
243
244
        /*
245
         * Figure out how the fetch is supposed to happen, before the
246
         * headers are adultered by VCL
247
         */
248 48921
        if (http_method_eq(http_GetMethod(bo->bereq), HEAD)) {
249
                /*
250
                 * A HEAD request can never have a body in the reply,
251
                 * no matter what the headers might say.
252
                 * [RFC7231 4.3.2 p25]
253
                 */
254 50
                bo->wrk->stats->fetch_head++;
255 50
                bo->htc->body_status = BS_NONE;
256 48921
        } else if (http_GetStatus(bo->beresp) <= 199) {
257
                /*
258
                 * 1xx responses never have a body.
259
                 * [RFC7230 3.3.2 p31]
260
                 * ... but we should never see them.
261
                 */
262 0
                bo->wrk->stats->fetch_1xx++;
263 0
                bo->htc->body_status = BS_ERROR;
264 48871
        } else if (http_IsStatus(bo->beresp, 204)) {
265
                /*
266
                 * 204 is "No Content", obviously don't expect a body.
267
                 * [RFC7230 3.3.1 p29 and 3.3.2 p31]
268
                 */
269 225
                bo->wrk->stats->fetch_204++;
270 225
                if ((http_GetHdr(bo->beresp, H_Content_Length, NULL) &&
271 100
                    bo->htc->content_length != 0) ||
272 125
                    http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
273 150
                        bo->htc->body_status = BS_ERROR;
274
                else
275 75
                        bo->htc->body_status = BS_NONE;
276 48771
        } else if (http_IsStatus(bo->beresp, 304)) {
277
                /*
278
                 * 304 is "Not Modified" it has no body.
279
                 * [RFC7230 3.3 p28]
280
                 */
281 799
                bo->wrk->stats->fetch_304++;
282 799
                bo->htc->body_status = BS_NONE;
283 48646
        } else if (bo->htc->body_status == BS_CHUNKED) {
284 3674
                bo->wrk->stats->fetch_chunked++;
285 47847
        } else if (bo->htc->body_status == BS_LENGTH) {
286 25398
                assert(bo->htc->content_length > 0);
287 25398
                bo->wrk->stats->fetch_length++;
288 44173
        } else if (bo->htc->body_status == BS_EOF) {
289 575
                bo->wrk->stats->fetch_eof++;
290 18775
        } else if (bo->htc->body_status == BS_ERROR) {
291 100
                bo->wrk->stats->fetch_bad++;
292 18200
        } else if (bo->htc->body_status == BS_NONE) {
293 18100
                bo->wrk->stats->fetch_none++;
294 18100
        } else {
295 0
                WRONG("wrong bodystatus");
296
        }
297
298 48821
        assert(bo->vfc->resp == bo->beresp);
299 48821
        if (bo->htc->body_status != BS_NONE &&
300 29797
            bo->htc->body_status != BS_ERROR)
301 29648
                if (V1F_Setup_Fetch(bo->vfc, bo->htc)) {
302 400
                        VSLb(bo->vsl, SLT_FetchError, "overflow");
303 400
                        htc->doclose = SC_RX_OVERFLOW;
304 400
                        return (-1);
305
                }
306
307 48421
        return (0);
308 50247
}