| | varnish-cache/bin/varnishd/cache/cache_fetch.c |
0 |
|
/*- |
1 |
|
* Copyright (c) 2006 Verdens Gang AS |
2 |
|
* Copyright (c) 2006-2015 Varnish Software AS |
3 |
|
* All rights reserved. |
4 |
|
* |
5 |
|
* Author: Poul-Henning Kamp <phk@phk.freebsd.dk> |
6 |
|
* |
7 |
|
* SPDX-License-Identifier: BSD-2-Clause |
8 |
|
* |
9 |
|
* Redistribution and use in source and binary forms, with or without |
10 |
|
* modification, are permitted provided that the following conditions |
11 |
|
* are met: |
12 |
|
* 1. Redistributions of source code must retain the above copyright |
13 |
|
* notice, this list of conditions and the following disclaimer. |
14 |
|
* 2. Redistributions in binary form must reproduce the above copyright |
15 |
|
* notice, this list of conditions and the following disclaimer in the |
16 |
|
* documentation and/or other materials provided with the distribution. |
17 |
|
* |
18 |
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
19 |
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
20 |
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
21 |
|
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE |
22 |
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
23 |
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
24 |
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
25 |
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
26 |
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
27 |
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
28 |
|
* SUCH DAMAGE. |
29 |
|
*/ |
30 |
|
|
31 |
|
#include "config.h" |
32 |
|
|
33 |
|
#include "cache_varnishd.h" |
34 |
|
#include "cache_filter.h" |
35 |
|
#include "cache_objhead.h" |
36 |
|
#include "storage/storage.h" |
37 |
|
#include "vcl.h" |
38 |
|
#include "vtim.h" |
39 |
|
#include "vcc_interface.h" |
40 |
|
|
41 |
|
#define FETCH_STEPS \ |
42 |
|
FETCH_STEP(mkbereq, MKBEREQ) \ |
43 |
|
FETCH_STEP(retry, RETRY) \ |
44 |
|
FETCH_STEP(startfetch, STARTFETCH) \ |
45 |
|
FETCH_STEP(condfetch, CONDFETCH) \ |
46 |
|
FETCH_STEP(fetch, FETCH) \ |
47 |
|
FETCH_STEP(fetchbody, FETCHBODY) \ |
48 |
|
FETCH_STEP(fetchend, FETCHEND) \ |
49 |
|
FETCH_STEP(error, ERROR) \ |
50 |
|
FETCH_STEP(fail, FAIL) \ |
51 |
|
FETCH_STEP(done, DONE) |
52 |
|
|
53 |
|
typedef const struct fetch_step *vbf_state_f(struct worker *, struct busyobj *); |
54 |
|
|
55 |
|
struct fetch_step { |
56 |
|
const char *name; |
57 |
|
vbf_state_f *func; |
58 |
1853 |
}; |
59 |
1853 |
|
60 |
1853 |
#define FETCH_STEP(l, U) \ |
61 |
1853 |
static vbf_state_f vbf_stp_##l; \ |
62 |
|
static const struct fetch_step F_STP_##U[1] = {{ .name = "Fetch Step " #l, .func = vbf_stp_##l, }}; |
63 |
|
FETCH_STEPS |
64 |
|
#undef FETCH_STEP |
65 |
|
|
66 |
|
static hdr_t const H_X_Varnish = HDR("X-Varnish"); |
67 |
|
|
68 |
|
/*-------------------------------------------------------------------- |
69 |
|
* Allocate an object, with fall-back to Transient. |
70 |
|
* XXX: This somewhat overlaps the stuff in stevedore.c |
71 |
|
* XXX: Should this be merged over there ? |
72 |
|
*/ |
73 |
|
|
74 |
|
static int |
75 |
2250 |
vbf_allocobj(struct busyobj *bo, unsigned l) |
76 |
|
{ |
77 |
|
struct objcore *oc; |
78 |
|
const struct stevedore *stv; |
79 |
|
vtim_dur lifetime; |
80 |
|
|
81 |
2250 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
82 |
2250 |
oc = bo->fetch_objcore; |
83 |
2250 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
84 |
|
|
85 |
2250 |
lifetime = oc->ttl + oc->grace + oc->keep; |
86 |
|
|
87 |
2250 |
if (bo->uncacheable) { |
88 |
887 |
stv = stv_transient; |
89 |
887 |
bo->wrk->stats->beresp_uncacheable++; |
90 |
887 |
} |
91 |
1363 |
else if (lifetime < cache_param->shortlived) { |
92 |
127 |
stv = stv_transient; |
93 |
127 |
bo->wrk->stats->beresp_shortlived++; |
94 |
127 |
} |
95 |
|
else |
96 |
1236 |
stv = bo->storage; |
97 |
|
|
98 |
2250 |
bo->storage = NULL; |
99 |
|
|
100 |
2250 |
if (stv == NULL) |
101 |
1 |
return (0); |
102 |
|
|
103 |
2249 |
if (STV_NewObject(bo->wrk, oc, stv, l)) |
104 |
2242 |
return (1); |
105 |
|
|
106 |
7 |
if (stv == stv_transient) |
107 |
4 |
return (0); |
108 |
|
|
109 |
|
/* |
110 |
|
* Try to salvage the transaction by allocating a shortlived object |
111 |
|
* on Transient storage. |
112 |
|
*/ |
113 |
|
|
114 |
3 |
oc->ttl = vmin_t(float, oc->ttl, cache_param->shortlived); |
115 |
3 |
oc->grace = 0.0; |
116 |
3 |
oc->keep = 0.0; |
117 |
3 |
return (STV_NewObject(bo->wrk, oc, stv_transient, l)); |
118 |
2250 |
} |
119 |
|
|
120 |
|
static void |
121 |
2025 |
vbf_cleanup(struct busyobj *bo) |
122 |
|
{ |
123 |
|
struct vfp_ctx *vfc; |
124 |
|
|
125 |
2025 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
126 |
2025 |
vfc = bo->vfc; |
127 |
2025 |
CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); |
128 |
|
|
129 |
2025 |
bo->acct.beresp_bodybytes += VFP_Close(vfc); |
130 |
2025 |
bo->vfp_filter_list = NULL; |
131 |
|
|
132 |
2025 |
if (bo->director_state != DIR_S_NULL) |
133 |
2017 |
VDI_Finish(bo); |
134 |
2025 |
} |
135 |
|
|
136 |
|
void |
137 |
9 |
Bereq_Rollback(VRT_CTX) |
138 |
|
{ |
139 |
|
struct busyobj *bo; |
140 |
|
|
141 |
9 |
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); |
142 |
9 |
bo = ctx->bo; |
143 |
9 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
144 |
|
|
145 |
9 |
if (bo->htc != NULL) { |
146 |
8 |
assert(bo->htc->body_status != BS_TAKEN); |
147 |
8 |
if (bo->htc->body_status != BS_NONE) |
148 |
1 |
bo->htc->doclose = SC_RESP_CLOSE; |
149 |
8 |
} |
150 |
|
|
151 |
9 |
vbf_cleanup(bo); |
152 |
9 |
VCL_TaskLeave(ctx, bo->privs); |
153 |
9 |
VCL_TaskEnter(bo->privs); |
154 |
9 |
HTTP_Clone(bo->bereq, bo->bereq0); |
155 |
9 |
bo->vfp_filter_list = NULL; |
156 |
9 |
bo->err_reason = NULL; |
157 |
9 |
AN(bo->ws_bo); |
158 |
9 |
WS_Rollback(bo->ws, bo->ws_bo); |
159 |
9 |
} |
160 |
|
|
161 |
|
/*-------------------------------------------------------------------- |
162 |
|
* Turn the beresp into a obj |
163 |
|
*/ |
164 |
|
|
165 |
|
static int |
166 |
2250 |
vbf_beresp2obj(struct busyobj *bo) |
167 |
|
{ |
168 |
|
unsigned l, l2; |
169 |
|
const char *b; |
170 |
|
uint8_t *bp; |
171 |
2250 |
struct vsb *vary = NULL; |
172 |
2250 |
int varyl = 0; |
173 |
|
struct objcore *oc; |
174 |
|
|
175 |
2250 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
176 |
2250 |
oc = bo->fetch_objcore; |
177 |
2250 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
178 |
|
|
179 |
2250 |
l = 0; |
180 |
|
|
181 |
|
/* Create Vary instructions */ |
182 |
2250 |
if (!(oc->flags & OC_F_PRIVATE)) { |
183 |
1465 |
varyl = VRY_Create(bo, &vary); |
184 |
1465 |
if (varyl > 0) { |
185 |
215 |
AN(vary); |
186 |
215 |
assert(varyl == VSB_len(vary)); |
187 |
215 |
l += PRNDUP((intptr_t)varyl); |
188 |
1465 |
} else if (varyl < 0) { |
189 |
|
/* |
190 |
|
* Vary parse error |
191 |
|
* Complain about it, and make this a pass. |
192 |
|
*/ |
193 |
5 |
VSLb(bo->vsl, SLT_Error, |
194 |
|
"Illegal 'Vary' header from backend, " |
195 |
|
"making this a pass."); |
196 |
5 |
bo->uncacheable = 1; |
197 |
5 |
AZ(vary); |
198 |
5 |
} else |
199 |
|
/* No vary */ |
200 |
1245 |
AZ(vary); |
201 |
1465 |
} |
202 |
|
|
203 |
4500 |
l2 = http_EstimateWS(bo->beresp, |
204 |
2250 |
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS); |
205 |
2250 |
l += l2; |
206 |
|
|
207 |
2250 |
if (bo->uncacheable) |
208 |
887 |
oc->flags |= OC_F_HFM; |
209 |
|
|
210 |
2250 |
if (!vbf_allocobj(bo, l)) { |
211 |
6 |
if (vary != NULL) |
212 |
0 |
VSB_destroy(&vary); |
213 |
6 |
AZ(vary); |
214 |
6 |
return (VFP_Error(bo->vfc, "Could not get storage")); |
215 |
|
} |
216 |
|
|
217 |
2244 |
if (vary != NULL) { |
218 |
215 |
AN(ObjSetAttr(bo->wrk, oc, OA_VARY, varyl, VSB_data(vary))); |
219 |
215 |
VSB_destroy(&vary); |
220 |
215 |
} |
221 |
|
|
222 |
2244 |
AZ(ObjSetXID(bo->wrk, oc, bo->vsl->wid)); |
223 |
|
|
224 |
|
/* for HTTP_Encode() VSLH call */ |
225 |
2244 |
bo->beresp->logtag = SLT_ObjMethod; |
226 |
|
|
227 |
|
/* Filter into object */ |
228 |
2244 |
bp = ObjSetAttr(bo->wrk, oc, OA_HEADERS, l2, NULL); |
229 |
2244 |
AN(bp); |
230 |
4488 |
HTTP_Encode(bo->beresp, bp, l2, |
231 |
2244 |
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS); |
232 |
|
|
233 |
2244 |
if (http_GetHdr(bo->beresp, H_Last_Modified, &b)) |
234 |
43 |
AZ(ObjSetDouble(bo->wrk, oc, OA_LASTMODIFIED, VTIM_parse(b))); |
235 |
|
else |
236 |
2201 |
AZ(ObjSetDouble(bo->wrk, oc, OA_LASTMODIFIED, |
237 |
|
floor(oc->t_origin))); |
238 |
|
|
239 |
2244 |
return (0); |
240 |
2250 |
} |
241 |
|
|
242 |
|
/*-------------------------------------------------------------------- |
243 |
|
* Copy req->bereq and release req if no body |
244 |
|
*/ |
245 |
|
|
246 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
247 |
2273 |
vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) |
248 |
|
{ |
249 |
|
const char *q; |
250 |
|
struct objcore *oc; |
251 |
|
|
252 |
2273 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
253 |
2273 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
254 |
2273 |
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); |
255 |
2273 |
oc = bo->fetch_objcore; |
256 |
2273 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
257 |
|
|
258 |
2273 |
assert(oc->boc->state == BOS_INVALID); |
259 |
2273 |
AZ(bo->storage); |
260 |
|
|
261 |
2273 |
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod); |
262 |
4546 |
http_FilterReq(bo->bereq0, bo->req->http, |
263 |
2273 |
bo->uncacheable ? HTTPH_R_PASS : HTTPH_R_FETCH); |
264 |
|
|
265 |
2273 |
if (bo->uncacheable) |
266 |
784 |
AZ(bo->stale_oc); |
267 |
|
else { |
268 |
1489 |
http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET"); |
269 |
1489 |
if (cache_param->http_gzip_support) |
270 |
1484 |
http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip"); |
271 |
|
} |
272 |
2273 |
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1"); |
273 |
|
|
274 |
2309 |
if (bo->stale_oc != NULL && |
275 |
154 |
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) && |
276 |
40 |
(bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) { |
277 |
38 |
AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); |
278 |
38 |
q = RFC2616_Strong_LM(NULL, wrk, bo->stale_oc); |
279 |
38 |
if (q != NULL) |
280 |
26 |
http_PrintfHeader(bo->bereq0, |
281 |
13 |
"If-Modified-Since: %s", q); |
282 |
38 |
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_ETag); |
283 |
38 |
if (q != NULL) |
284 |
52 |
http_PrintfHeader(bo->bereq0, |
285 |
26 |
"If-None-Match: %s", q); |
286 |
38 |
} |
287 |
|
|
288 |
2273 |
http_CopyHome(bo->bereq0); |
289 |
2273 |
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); |
290 |
2273 |
bo->ws_bo = WS_Snapshot(bo->ws); |
291 |
2273 |
HTTP_Clone(bo->bereq, bo->bereq0); |
292 |
|
|
293 |
2273 |
if (bo->req->req_body_status->avail == 0) { |
294 |
2182 |
VBO_SetState(bo->wrk, bo, BOS_REQ_DONE); |
295 |
2273 |
} else if (bo->req->req_body_status == BS_CACHED) { |
296 |
23 |
AN(bo->req->body_oc); |
297 |
23 |
bo->bereq_body = bo->req->body_oc; |
298 |
23 |
HSH_Ref(bo->bereq_body); |
299 |
23 |
VBO_SetState(bo->wrk, bo, BOS_REQ_DONE); |
300 |
23 |
} |
301 |
2273 |
return (F_STP_STARTFETCH); |
302 |
|
} |
303 |
|
|
304 |
|
/*-------------------------------------------------------------------- |
305 |
|
* Start a new VSL transaction and try again |
306 |
|
* Prepare the busyobj and fetch processors |
307 |
|
*/ |
308 |
|
|
309 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
310 |
36 |
vbf_stp_retry(struct worker *wrk, struct busyobj *bo) |
311 |
|
{ |
312 |
36 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
313 |
36 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
314 |
|
|
315 |
36 |
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); |
316 |
|
|
317 |
36 |
if (bo->no_retry != NULL) { |
318 |
4 |
VSLb(bo->vsl, SLT_Error, |
319 |
2 |
"Retry not possible, %s", bo->no_retry); |
320 |
2 |
return (F_STP_FAIL); |
321 |
|
} |
322 |
|
|
323 |
34 |
VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk)); |
324 |
|
|
325 |
|
/* VDI_Finish (via vbf_cleanup) must have been called before */ |
326 |
34 |
assert(bo->director_state == DIR_S_NULL); |
327 |
|
|
328 |
|
/* reset other bo attributes - See VBO_GetBusyObj */ |
329 |
34 |
bo->storage = NULL; |
330 |
34 |
bo->do_esi = 0; |
331 |
34 |
bo->do_stream = 1; |
332 |
34 |
bo->was_304 = 0; |
333 |
34 |
bo->err_code = 0; |
334 |
34 |
bo->err_reason = NULL; |
335 |
34 |
bo->connect_timeout = NAN; |
336 |
34 |
bo->first_byte_timeout = NAN; |
337 |
34 |
bo->between_bytes_timeout = NAN; |
338 |
34 |
if (bo->htc != NULL) |
339 |
0 |
bo->htc->doclose = SC_NULL; |
340 |
|
|
341 |
|
// XXX: BereqEnd + BereqAcct ? |
342 |
34 |
VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); |
343 |
34 |
VSLb_ts_busyobj(bo, "Start", bo->t_prev); |
344 |
34 |
http_VSL_log(bo->bereq); |
345 |
|
|
346 |
34 |
return (F_STP_STARTFETCH); |
347 |
36 |
} |
348 |
|
|
349 |
|
/*-------------------------------------------------------------------- |
350 |
|
* 304 setup logic |
351 |
|
*/ |
352 |
|
|
353 |
|
static void |
354 |
30 |
vbf_304_logic(struct busyobj *bo) |
355 |
|
{ |
356 |
|
|
357 |
30 |
AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); |
358 |
30 |
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) { |
359 |
|
/* |
360 |
|
* If a VFP changed C-E in the stored |
361 |
|
* object, then don't overwrite C-E from |
362 |
|
* the IMS fetch, and we must weaken any |
363 |
|
* new ETag we get. |
364 |
|
*/ |
365 |
2 |
RFC2616_Weaken_Etag(bo->beresp); |
366 |
2 |
} |
367 |
30 |
http_Unset(bo->beresp, H_Content_Encoding); |
368 |
30 |
http_Unset(bo->beresp, H_Content_Length); |
369 |
30 |
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); |
370 |
30 |
} |
371 |
|
|
372 |
|
/*-------------------------------------------------------------------- |
373 |
|
* Setup bereq from bereq0, run vcl_backend_fetch |
374 |
|
*/ |
375 |
|
|
376 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
377 |
2307 |
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) |
378 |
|
{ |
379 |
|
int i; |
380 |
|
const char *q; |
381 |
|
vtim_real now; |
382 |
2307 |
unsigned handling, skip_vbr = 0; |
383 |
|
struct objcore *oc; |
384 |
|
|
385 |
2307 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
386 |
2307 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
387 |
2307 |
oc = bo->fetch_objcore; |
388 |
2307 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
389 |
|
|
390 |
2307 |
AZ(bo->storage); |
391 |
2307 |
bo->storage = bo->uncacheable ? stv_transient : STV_next(); |
392 |
|
|
393 |
2307 |
if (bo->retries > 0) |
394 |
34 |
http_Unset(bo->bereq, H_X_Varnish); |
395 |
|
|
396 |
2307 |
http_PrintfHeader(bo->bereq, "X-Varnish: %ju", VXID(bo->vsl->wid)); |
397 |
|
|
398 |
2307 |
if (bo->bereq_body == NULL && bo->req == NULL) |
399 |
2216 |
http_Unset(bo->bereq, H_Content_Length); |
400 |
|
|
401 |
2307 |
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); |
402 |
|
|
403 |
2307 |
if (wrk->vpi->handling == VCL_RET_ABANDON || |
404 |
2304 |
wrk->vpi->handling == VCL_RET_FAIL) |
405 |
7 |
return (F_STP_FAIL); |
406 |
|
|
407 |
2300 |
assert (wrk->vpi->handling == VCL_RET_FETCH || |
408 |
|
wrk->vpi->handling == VCL_RET_ERROR); |
409 |
|
|
410 |
2300 |
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); |
411 |
|
|
412 |
2300 |
assert(oc->boc->state <= BOS_REQ_DONE); |
413 |
|
|
414 |
2300 |
AZ(bo->htc); |
415 |
|
|
416 |
2300 |
VFP_Setup(bo->vfc, wrk); |
417 |
2300 |
bo->vfc->oc = oc; |
418 |
2300 |
bo->vfc->resp = bo->beresp; |
419 |
2300 |
bo->vfc->req = bo->bereq; |
420 |
|
|
421 |
2300 |
if (wrk->vpi->handling == VCL_RET_ERROR) |
422 |
10 |
return (F_STP_ERROR); |
423 |
|
|
424 |
2290 |
VSLb_ts_busyobj(bo, "Fetch", W_TIM_real(wrk)); |
425 |
2290 |
i = VDI_GetHdr(bo); |
426 |
2290 |
if (bo->htc != NULL) |
427 |
2023 |
CHECK_OBJ_NOTNULL(bo->htc->doclose, STREAM_CLOSE_MAGIC); |
428 |
|
|
429 |
2290 |
bo->t_resp = now = W_TIM_real(wrk); |
430 |
2290 |
VSLb_ts_busyobj(bo, "Beresp", now); |
431 |
|
|
432 |
2290 |
if (i) { |
433 |
265 |
assert(bo->director_state == DIR_S_NULL); |
434 |
265 |
return (F_STP_ERROR); |
435 |
|
} |
436 |
|
|
437 |
2025 |
if (bo->htc != NULL && bo->htc->body_status == BS_ERROR) { |
438 |
6 |
bo->htc->doclose = SC_RX_BODY; |
439 |
6 |
vbf_cleanup(bo); |
440 |
6 |
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); |
441 |
6 |
assert(bo->director_state == DIR_S_NULL); |
442 |
6 |
return (F_STP_ERROR); |
443 |
|
} |
444 |
|
|
445 |
2019 |
if (!http_GetHdr(bo->beresp, H_Date, NULL)) { |
446 |
|
/* |
447 |
|
* RFC 2616 14.18 Date: The Date general-header field |
448 |
|
* represents the date and time at which the message was |
449 |
|
* originated, having the same semantics as orig-date in |
450 |
|
* RFC 822. ... A received message that does not have a |
451 |
|
* Date header field MUST be assigned one by the recipient |
452 |
|
* if the message will be cached by that recipient or |
453 |
|
* gatewayed via a protocol which requires a Date. |
454 |
|
* |
455 |
|
* If we didn't get a Date header, we assign one here. |
456 |
|
*/ |
457 |
83 |
http_TimeHeader(bo->beresp, "Date: ", now); |
458 |
83 |
} |
459 |
|
|
460 |
|
/* |
461 |
|
* These two headers can be spread over multiple actual headers |
462 |
|
* and we rely on their content outside of VCL, so collect them |
463 |
|
* into one line here. |
464 |
|
*/ |
465 |
2019 |
http_CollectHdr(bo->beresp, H_Cache_Control); |
466 |
2019 |
http_CollectHdr(bo->beresp, H_Vary); |
467 |
|
|
468 |
|
/* What does RFC2616 think about TTL ? */ |
469 |
4038 |
RFC2616_Ttl(bo, now, |
470 |
2019 |
&oc->t_origin, |
471 |
2019 |
&oc->ttl, |
472 |
2019 |
&oc->grace, |
473 |
2019 |
&oc->keep); |
474 |
|
|
475 |
2019 |
AZ(bo->do_esi); |
476 |
2019 |
AZ(bo->was_304); |
477 |
|
|
478 |
2019 |
if (http_IsStatus(bo->beresp, 304) && !bo->uncacheable) { |
479 |
38 |
if (bo->stale_oc == NULL){ |
480 |
0 |
VSLb(bo->vsl, SLT_Error, |
481 |
|
"304 response but not conditional fetch"); |
482 |
0 |
bo->htc->doclose = SC_RX_BAD; |
483 |
0 |
vbf_cleanup(bo); |
484 |
0 |
return (F_STP_ERROR); |
485 |
|
} |
486 |
38 |
bo->was_304 = 1; |
487 |
38 |
VCL_backend_refresh_method(bo->vcl, wrk, NULL, bo, NULL); |
488 |
38 |
switch (wrk->vpi->handling) { |
489 |
|
case VCL_RET_MERGE: |
490 |
30 |
vbf_304_logic(bo); |
491 |
30 |
break; |
492 |
|
case VCL_RET_BERESP: |
493 |
2 |
http_SetStatus(bo->beresp, 200, NULL); |
494 |
2 |
http_Unset(bo->beresp, H_Content_Length); |
495 |
2 |
http_Unset(bo->beresp, H_Content_Encoding); |
496 |
4 |
q = HTTP_GetHdrPack(wrk, bo->stale_oc, |
497 |
2 |
H_Content_Length); |
498 |
2 |
if (q != NULL) { |
499 |
2 |
http_ForceHeader(bo->beresp, |
500 |
1 |
H_Content_Length, q); |
501 |
1 |
} |
502 |
4 |
q = HTTP_GetHdrPack(wrk, bo->stale_oc, |
503 |
2 |
H_Content_Encoding); |
504 |
2 |
if (q != NULL) { |
505 |
0 |
http_ForceHeader(bo->beresp, |
506 |
0 |
H_Content_Encoding, q); |
507 |
0 |
} |
508 |
4 |
q = HTTP_GetHdrPack(wrk, bo->stale_oc, |
509 |
2 |
H_Last_Modified); |
510 |
2 |
if (q != NULL) { |
511 |
0 |
http_ForceHeader(bo->beresp, |
512 |
0 |
H_Last_Modified, q); |
513 |
0 |
} |
514 |
2 |
q = HTTP_GetHdrPack(wrk, bo->stale_oc, H_ETag); |
515 |
2 |
if (q != NULL) |
516 |
2 |
http_ForceHeader(bo->beresp, H_ETag, q); |
517 |
2 |
break; |
518 |
|
case VCL_RET_OBJ_STALE: |
519 |
2 |
if (HTTP_Decode(bo->beresp, ObjGetAttr(bo->wrk, |
520 |
1 |
bo->stale_oc, OA_HEADERS, NULL))) { |
521 |
0 |
bo->htc->doclose = SC_RX_OVERFLOW; |
522 |
0 |
vbf_cleanup(bo); |
523 |
0 |
return (F_STP_ERROR); |
524 |
|
} |
525 |
1 |
break; |
526 |
|
case VCL_RET_RETRY: |
527 |
|
case VCL_RET_ERROR: |
528 |
|
case VCL_RET_ABANDON: |
529 |
|
case VCL_RET_FAIL: |
530 |
5 |
skip_vbr = 1; |
531 |
5 |
break; |
532 |
|
default: |
533 |
0 |
WRONG("Illegal return from vcl_backend_refresh{}"); |
534 |
0 |
} |
535 |
38 |
} |
536 |
|
|
537 |
2019 |
if (bo->htc != NULL && bo->htc->doclose == SC_NULL && |
538 |
1927 |
http_GetHdrField(bo->bereq, H_Connection, "close", NULL)) |
539 |
13 |
bo->htc->doclose = SC_REQ_CLOSE; |
540 |
2019 |
if (!skip_vbr) |
541 |
2013 |
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); |
542 |
|
|
543 |
2017 |
if (bo->htc != NULL && bo->htc->doclose == SC_NULL && |
544 |
1905 |
http_GetHdrField(bo->beresp, H_Connection, "close", NULL)) |
545 |
1 |
bo->htc->doclose = SC_RESP_CLOSE; |
546 |
|
|
547 |
2017 |
if (VRG_CheckBo(bo) < 0) { |
548 |
7 |
if (bo->director_state != DIR_S_NULL) |
549 |
6 |
VDI_Finish(bo); |
550 |
7 |
return (F_STP_ERROR); |
551 |
|
} |
552 |
|
|
553 |
4014 |
if (wrk->vpi->handling == VCL_RET_ABANDON || |
554 |
2006 |
wrk->vpi->handling == VCL_RET_FAIL || |
555 |
2004 |
wrk->vpi->handling == VCL_RET_ERROR) { |
556 |
|
/* do not count deliberately ending the backend connection as |
557 |
|
* fetch failure |
558 |
|
*/ |
559 |
14 |
handling = wrk->vpi->handling; |
560 |
14 |
if (bo->htc) |
561 |
14 |
bo->htc->doclose = SC_RESP_CLOSE; |
562 |
14 |
vbf_cleanup(bo); |
563 |
14 |
wrk->vpi->handling = handling; |
564 |
|
|
565 |
14 |
if (wrk->vpi->handling == VCL_RET_ERROR) |
566 |
9 |
return (F_STP_ERROR); |
567 |
|
else |
568 |
5 |
return (F_STP_FAIL); |
569 |
|
} |
570 |
|
|
571 |
1996 |
if (wrk->vpi->handling == VCL_RET_RETRY) { |
572 |
29 |
if (bo->htc && bo->htc->body_status != BS_NONE) |
573 |
7 |
bo->htc->doclose = SC_RESP_CLOSE; |
574 |
29 |
vbf_cleanup(bo); |
575 |
|
|
576 |
29 |
if (bo->retries++ < bo->max_retries) |
577 |
27 |
return (F_STP_RETRY); |
578 |
|
|
579 |
2 |
VSLb(bo->vsl, SLT_VCL_Error, |
580 |
|
"Too many retries, delivering 503"); |
581 |
2 |
assert(bo->director_state == DIR_S_NULL); |
582 |
2 |
return (F_STP_ERROR); |
583 |
|
} |
584 |
|
|
585 |
1967 |
VSLb_ts_busyobj(bo, "Process", W_TIM_real(wrk)); |
586 |
1967 |
assert(oc->boc->state <= BOS_REQ_DONE); |
587 |
1967 |
if (oc->boc->state != BOS_REQ_DONE) |
588 |
40 |
VBO_SetState(wrk, bo, BOS_REQ_DONE); |
589 |
|
|
590 |
1967 |
if (bo->do_esi) |
591 |
310 |
bo->do_stream = 0; |
592 |
1967 |
if (wrk->vpi->handling == VCL_RET_PASS) { |
593 |
22 |
oc->flags |= OC_F_HFP; |
594 |
22 |
bo->uncacheable = 1; |
595 |
22 |
wrk->vpi->handling = VCL_RET_DELIVER; |
596 |
22 |
} |
597 |
1967 |
if (!bo->uncacheable || !bo->do_stream) |
598 |
1361 |
oc->boc->transit_buffer = 0; |
599 |
1967 |
if (bo->uncacheable) |
600 |
744 |
oc->flags |= OC_F_HFM; |
601 |
|
|
602 |
1967 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
603 |
|
|
604 |
1967 |
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH); |
605 |
2305 |
} |
606 |
|
|
607 |
|
/*-------------------------------------------------------------------- |
608 |
|
*/ |
609 |
|
|
610 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
611 |
1101 |
vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) |
612 |
|
{ |
613 |
|
ssize_t l; |
614 |
|
uint8_t *ptr; |
615 |
1101 |
enum vfp_status vfps = VFP_ERROR; |
616 |
|
ssize_t est; |
617 |
|
struct vfp_ctx *vfc; |
618 |
|
struct objcore *oc; |
619 |
|
|
620 |
1101 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
621 |
1101 |
vfc = bo->vfc; |
622 |
1101 |
CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); |
623 |
1101 |
oc = bo->fetch_objcore; |
624 |
1101 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
625 |
|
|
626 |
1101 |
AN(vfc->vfp_nxt); |
627 |
|
|
628 |
1101 |
est = bo->htc->content_length; |
629 |
1101 |
if (est < 0) |
630 |
149 |
est = 0; |
631 |
|
|
632 |
1101 |
do { |
633 |
55981 |
if (oc->flags & OC_F_CANCEL) { |
634 |
|
/* |
635 |
|
* A pass object and delivery was terminated |
636 |
|
* We don't fail the fetch, in order for HitMiss |
637 |
|
* objects to be created. |
638 |
|
*/ |
639 |
3 |
AN(oc->flags & OC_F_HFM); |
640 |
3 |
VSLb(wrk->vsl, SLT_Debug, |
641 |
|
"Fetch: Pass delivery abandoned"); |
642 |
3 |
bo->htc->doclose = SC_RX_BODY; |
643 |
3 |
break; |
644 |
|
} |
645 |
55978 |
AZ(vfc->failed); |
646 |
55978 |
l = est; |
647 |
55978 |
assert(l >= 0); |
648 |
55978 |
if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { |
649 |
5 |
bo->htc->doclose = SC_RX_BODY; |
650 |
5 |
break; |
651 |
|
} |
652 |
|
|
653 |
55973 |
AZ(vfc->failed); |
654 |
55973 |
vfps = VFP_Suck(vfc, ptr, &l); |
655 |
55973 |
if (l >= 0 && vfps != VFP_ERROR) { |
656 |
55941 |
VFP_Extend(vfc, l, vfps); |
657 |
55941 |
if (est >= l) |
658 |
2853 |
est -= l; |
659 |
|
else |
660 |
53088 |
est = 0; |
661 |
55941 |
} |
662 |
55973 |
} while (vfps == VFP_OK); |
663 |
|
|
664 |
1101 |
if (vfc->failed) { |
665 |
37 |
(void)VFP_Error(vfc, "Fetch pipeline failed to process"); |
666 |
37 |
bo->htc->doclose = SC_RX_BODY; |
667 |
37 |
vbf_cleanup(bo); |
668 |
37 |
if (!bo->do_stream) { |
669 |
21 |
assert(oc->boc->state < BOS_STREAM); |
670 |
|
// XXX: doclose = ? |
671 |
21 |
return (F_STP_ERROR); |
672 |
|
} else { |
673 |
16 |
wrk->stats->fetch_failed++; |
674 |
16 |
return (F_STP_FAIL); |
675 |
|
} |
676 |
|
} |
677 |
|
|
678 |
1064 |
return (F_STP_FETCHEND); |
679 |
1101 |
} |
680 |
|
|
681 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
682 |
1935 |
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) |
683 |
|
{ |
684 |
|
struct vrt_ctx ctx[1]; |
685 |
|
struct objcore *oc; |
686 |
|
|
687 |
1935 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
688 |
1935 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
689 |
1935 |
oc = bo->fetch_objcore; |
690 |
1935 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
691 |
|
|
692 |
1935 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
693 |
|
|
694 |
1935 |
if (bo->htc == NULL) { |
695 |
2 |
(void)VFP_Error(bo->vfc, "No backend connection (rollback?)"); |
696 |
2 |
vbf_cleanup(bo); |
697 |
2 |
return (F_STP_ERROR); |
698 |
|
} |
699 |
|
|
700 |
|
/* No body -> done */ |
701 |
1933 |
if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { |
702 |
754 |
http_Unset(bo->beresp, H_Content_Encoding); |
703 |
754 |
bo->do_gzip = bo->do_gunzip = 0; |
704 |
754 |
bo->do_stream = 0; |
705 |
754 |
bo->vfp_filter_list = ""; |
706 |
1933 |
} else if (bo->vfp_filter_list == NULL) { |
707 |
1173 |
bo->vfp_filter_list = VBF_Get_Filter_List(bo); |
708 |
1173 |
} |
709 |
|
|
710 |
1933 |
if (bo->vfp_filter_list == NULL || |
711 |
1933 |
VCL_StackVFP(bo->vfc, bo->vcl, bo->vfp_filter_list)) { |
712 |
24 |
(bo)->htc->doclose = SC_OVERLOAD; |
713 |
24 |
vbf_cleanup(bo); |
714 |
24 |
return (F_STP_ERROR); |
715 |
|
} |
716 |
|
|
717 |
1909 |
if (oc->flags & OC_F_PRIVATE) |
718 |
627 |
AN(bo->uncacheable); |
719 |
|
|
720 |
1909 |
oc->boc->fetched_so_far = 0; |
721 |
|
|
722 |
1909 |
INIT_OBJ(ctx, VRT_CTX_MAGIC); |
723 |
1909 |
VCL_Bo2Ctx(ctx, bo); |
724 |
|
|
725 |
1909 |
if (VFP_Open(ctx, bo->vfc)) { |
726 |
53 |
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); |
727 |
53 |
bo->htc->doclose = SC_RX_BODY; |
728 |
53 |
vbf_cleanup(bo); |
729 |
53 |
return (F_STP_ERROR); |
730 |
|
} |
731 |
|
|
732 |
1856 |
if (vbf_beresp2obj(bo)) { |
733 |
3 |
bo->htc->doclose = SC_RX_BODY; |
734 |
3 |
vbf_cleanup(bo); |
735 |
3 |
return (F_STP_ERROR); |
736 |
|
} |
737 |
|
|
738 |
|
#define OBJ_FLAG(U, l, v) \ |
739 |
|
if (bo->vfc->obj_flags & OF_##U) \ |
740 |
|
ObjSetFlag(bo->wrk, oc, OF_##U, 1); |
741 |
|
#include "tbl/obj_attr.h" |
742 |
|
|
743 |
1853 |
if (!(oc->flags & OC_F_HFM) && |
744 |
1183 |
http_IsStatus(bo->beresp, 200) && ( |
745 |
1165 |
RFC2616_Strong_LM(bo->beresp, NULL, NULL) != NULL || |
746 |
1141 |
http_GetHdr(bo->beresp, H_ETag, NULL))) |
747 |
56 |
ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 1); |
748 |
|
|
749 |
1853 |
assert(oc->boc->refcount >= 1); |
750 |
|
|
751 |
1853 |
assert(oc->boc->state == BOS_REQ_DONE); |
752 |
|
|
753 |
1853 |
if (bo->do_stream) |
754 |
820 |
VBO_SetState(wrk, bo, BOS_STREAM); |
755 |
|
|
756 |
|
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", |
757 |
|
bo->htc->body_status->nbr, bo->htc->body_status->name, |
758 |
|
bo->do_stream ? "stream" : "-"); |
759 |
|
|
760 |
1853 |
if (bo->htc->body_status != BS_NONE) { |
761 |
1101 |
assert(bo->htc->body_status != BS_ERROR); |
762 |
1101 |
return (F_STP_FETCHBODY); |
763 |
|
} |
764 |
752 |
AZ(bo->vfc->failed); |
765 |
|
return (F_STP_FETCHEND); |
766 |
|
} |
767 |
|
|
768 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
769 |
1844 |
vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) |
770 |
|
{ |
771 |
|
|
772 |
|
struct objcore *oc; |
773 |
|
|
774 |
1844 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
775 |
1844 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
776 |
1844 |
oc = bo->fetch_objcore; |
777 |
1844 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
778 |
|
|
779 |
1844 |
AZ(bo->vfc->failed); |
780 |
|
|
781 |
|
/* Recycle the backend connection before setting BOS_FINISHED to |
782 |
|
give predictable backend reuse behavior for varnishtest */ |
783 |
1844 |
vbf_cleanup(bo); |
784 |
|
|
785 |
1844 |
AZ(ObjSetU64(wrk, oc, OA_LEN, oc->boc->fetched_so_far)); |
786 |
|
|
787 |
1844 |
if (bo->do_stream) |
788 |
831 |
assert(oc->boc->state == BOS_STREAM); |
789 |
|
else |
790 |
1013 |
assert(oc->boc->state == BOS_REQ_DONE); |
791 |
|
|
792 |
1844 |
VBO_SetState(wrk, bo, BOS_FINISHED); |
793 |
1844 |
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); |
794 |
1844 |
if (bo->stale_oc != NULL) { |
795 |
152 |
VSL(SLT_ExpKill, NO_VXID, "VBF_Superseded x=%ju n=%ju", |
796 |
76 |
VXID(ObjGetXID(wrk, bo->stale_oc)), |
797 |
76 |
VXID(ObjGetXID(wrk, bo->fetch_objcore))); |
798 |
76 |
HSH_Replace(bo->stale_oc, bo->fetch_objcore); |
799 |
76 |
} |
800 |
1844 |
return (F_STP_DONE); |
801 |
|
} |
802 |
|
|
803 |
|
/*-------------------------------------------------------------------- |
804 |
|
*/ |
805 |
|
|
806 |
|
struct vbf_objiter_priv { |
807 |
|
unsigned magic; |
808 |
|
#define VBF_OBITER_PRIV_MAGIC 0x3c272a17 |
809 |
|
struct busyobj *bo; |
810 |
|
// not yet allocated |
811 |
|
ssize_t l; |
812 |
|
// current allocation |
813 |
|
uint8_t *p; |
814 |
|
ssize_t pl; |
815 |
|
}; |
816 |
|
|
817 |
|
static int v_matchproto_(objiterate_f) |
818 |
32 |
vbf_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) |
819 |
|
{ |
820 |
|
struct vbf_objiter_priv *vop; |
821 |
|
ssize_t l; |
822 |
32 |
const uint8_t *ps = ptr; |
823 |
|
|
824 |
32 |
CAST_OBJ_NOTNULL(vop, priv, VBF_OBITER_PRIV_MAGIC); |
825 |
32 |
CHECK_OBJ_NOTNULL(vop->bo, BUSYOBJ_MAGIC); |
826 |
|
|
827 |
32 |
flush &= OBJ_ITER_END; |
828 |
|
|
829 |
63 |
while (len > 0) { |
830 |
32 |
if (vop->pl == 0) { |
831 |
31 |
vop->p = NULL; |
832 |
31 |
AN(vop->l); |
833 |
31 |
vop->pl = vop->l; |
834 |
62 |
if (VFP_GetStorage(vop->bo->vfc, &vop->pl, &vop->p) |
835 |
31 |
!= VFP_OK) |
836 |
1 |
return (1); |
837 |
30 |
if (vop->pl < vop->l) |
838 |
2 |
vop->l -= vop->pl; |
839 |
|
else |
840 |
28 |
vop->l = 0; |
841 |
30 |
} |
842 |
31 |
AN(vop->pl); |
843 |
31 |
AN(vop->p); |
844 |
|
|
845 |
31 |
l = vmin(vop->pl, len); |
846 |
31 |
memcpy(vop->p, ps, l); |
847 |
59 |
VFP_Extend(vop->bo->vfc, l, |
848 |
31 |
flush && l == len ? VFP_END : VFP_OK); |
849 |
31 |
ps += l; |
850 |
31 |
vop->p += l; |
851 |
31 |
len -= l; |
852 |
31 |
vop->pl -= l; |
853 |
|
} |
854 |
31 |
if (flush && vop->bo->vfc->failed == 0) |
855 |
28 |
AZ(vop->l); |
856 |
31 |
return (0); |
857 |
32 |
} |
858 |
|
|
859 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
860 |
32 |
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) |
861 |
|
{ |
862 |
|
struct boc *stale_boc; |
863 |
|
enum boc_state_e stale_state; |
864 |
|
struct objcore *oc, *stale_oc; |
865 |
|
struct vbf_objiter_priv vop[1]; |
866 |
|
|
867 |
32 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
868 |
32 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
869 |
32 |
oc = bo->fetch_objcore; |
870 |
32 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
871 |
32 |
stale_oc = bo->stale_oc; |
872 |
32 |
CHECK_OBJ_NOTNULL(stale_oc, OBJCORE_MAGIC); |
873 |
|
|
874 |
32 |
stale_boc = HSH_RefBoc(stale_oc); |
875 |
32 |
CHECK_OBJ_ORNULL(stale_boc, BOC_MAGIC); |
876 |
32 |
if (stale_boc) { |
877 |
|
/* Wait for the stale object to become fully fetched, so |
878 |
|
* that we can catch fetch errors, before we unbusy the |
879 |
|
* new object. This serves two purposes. First it helps |
880 |
|
* with request coalescing, and stops long chains of |
881 |
|
* IMS-updated short-TTL objects all streaming from a |
882 |
|
* single slow body fetch. Second it makes sure that all |
883 |
|
* the object attributes are complete when we copy them |
884 |
|
* (this would be an issue for ie OA_GZIPBITS). */ |
885 |
4 |
VSLb(bo->vsl, SLT_Notice, |
886 |
|
"vsl: Conditional fetch wait for streaming object"); |
887 |
|
/* XXX: We should have a VCL controlled timeout here */ |
888 |
4 |
ObjWaitState(stale_oc, BOS_FINISHED); |
889 |
4 |
stale_state = stale_boc->state; |
890 |
4 |
HSH_DerefBoc(bo->wrk, stale_oc); |
891 |
4 |
stale_boc = NULL; |
892 |
4 |
if (stale_state != BOS_FINISHED) { |
893 |
2 |
assert(stale_state == BOS_FAILED); |
894 |
2 |
AN(stale_oc->flags & OC_F_FAILED); |
895 |
2 |
} |
896 |
4 |
} |
897 |
|
|
898 |
32 |
AZ(stale_boc); |
899 |
32 |
if (stale_oc->flags & OC_F_FAILED) { |
900 |
2 |
(void)VFP_Error(bo->vfc, "Template object failed"); |
901 |
2 |
vbf_cleanup(bo); |
902 |
2 |
wrk->stats->fetch_failed++; |
903 |
2 |
return (F_STP_FAIL); |
904 |
|
} |
905 |
|
|
906 |
30 |
if (vbf_beresp2obj(bo)) { |
907 |
1 |
vbf_cleanup(bo); |
908 |
1 |
wrk->stats->fetch_failed++; |
909 |
1 |
return (F_STP_FAIL); |
910 |
|
} |
911 |
|
|
912 |
29 |
if (ObjHasAttr(bo->wrk, stale_oc, OA_ESIDATA)) |
913 |
1 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_ESIDATA)); |
914 |
|
|
915 |
29 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_FLAGS)); |
916 |
29 |
if (oc->flags & OC_F_HFM) |
917 |
2 |
ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 0); |
918 |
29 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_GZIPBITS)); |
919 |
|
|
920 |
29 |
if (bo->do_stream) |
921 |
28 |
VBO_SetState(wrk, bo, BOS_STREAM); |
922 |
|
|
923 |
29 |
INIT_OBJ(vop, VBF_OBITER_PRIV_MAGIC); |
924 |
29 |
vop->bo = bo; |
925 |
29 |
vop->l = ObjGetLen(bo->wrk, stale_oc); |
926 |
29 |
if (ObjIterate(wrk, stale_oc, vop, vbf_objiterate, 0)) |
927 |
1 |
(void)VFP_Error(bo->vfc, "Template object failed"); |
928 |
|
|
929 |
29 |
if (bo->vfc->failed) { |
930 |
1 |
vbf_cleanup(bo); |
931 |
1 |
wrk->stats->fetch_failed++; |
932 |
1 |
return (F_STP_FAIL); |
933 |
|
} |
934 |
28 |
return (F_STP_FETCHEND); |
935 |
32 |
} |
936 |
|
|
937 |
|
/*-------------------------------------------------------------------- |
938 |
|
* Create synth object |
939 |
|
* |
940 |
|
* replaces a stale object unless |
941 |
|
* - abandoning the bereq or |
942 |
|
* - leaving vcl_backend_error with return (deliver) |
943 |
|
* |
944 |
|
* We do want the stale replacement to avoid an object pileup with short ttl and |
945 |
|
* long grace/keep, yet there could exist cases where a cache object is |
946 |
|
* deliberately created to momentarily override a stale object. |
947 |
|
* |
948 |
|
* If this case exists, we should add a vcl veto (e.g. beresp.replace_stale with |
949 |
|
* default true) |
950 |
|
*/ |
951 |
|
|
952 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
953 |
402 |
vbf_stp_error(struct worker *wrk, struct busyobj *bo) |
954 |
|
{ |
955 |
|
ssize_t l, ll, o; |
956 |
|
vtim_real now; |
957 |
|
uint8_t *ptr; |
958 |
|
struct vsb *synth_body; |
959 |
|
struct objcore *stale, *oc; |
960 |
|
|
961 |
402 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
962 |
402 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
963 |
402 |
oc = bo->fetch_objcore; |
964 |
402 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
965 |
402 |
CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); |
966 |
402 |
assert(oc->boc->state < BOS_STREAM); |
967 |
402 |
assert(bo->director_state == DIR_S_NULL); |
968 |
|
|
969 |
402 |
if (wrk->vpi->handling != VCL_RET_ERROR) |
970 |
383 |
wrk->stats->fetch_failed++; |
971 |
|
|
972 |
402 |
now = W_TIM_real(wrk); |
973 |
402 |
VSLb_ts_busyobj(bo, "Error", now); |
974 |
|
|
975 |
402 |
if (oc->stobj->stevedore != NULL) { |
976 |
|
// replacing an already fetched object with a "synth" one |
977 |
21 |
assert(oc->boc->state < BOS_STREAM); |
978 |
21 |
oc->boc->fetched_so_far = 0; |
979 |
21 |
ObjFreeObj(bo->wrk, oc); |
980 |
21 |
} |
981 |
|
|
982 |
402 |
if (bo->storage == NULL) |
983 |
24 |
bo->storage = STV_next(); |
984 |
|
|
985 |
|
// XXX: reset all beresp flags ? |
986 |
|
|
987 |
402 |
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); |
988 |
402 |
if (bo->err_code > 0) |
989 |
64 |
http_PutResponse(bo->beresp, "HTTP/1.1", bo->err_code, |
990 |
32 |
bo->err_reason); |
991 |
|
else |
992 |
370 |
http_PutResponse(bo->beresp, "HTTP/1.1", 503, |
993 |
|
"Backend fetch failed"); |
994 |
|
|
995 |
402 |
http_TimeHeader(bo->beresp, "Date: ", now); |
996 |
402 |
http_SetHeader(bo->beresp, "Server: Varnish"); |
997 |
|
|
998 |
402 |
stale = bo->stale_oc; |
999 |
402 |
oc->t_origin = now; |
1000 |
402 |
oc->ttl = 0; |
1001 |
402 |
oc->grace = 0; |
1002 |
402 |
oc->keep = 0; |
1003 |
|
|
1004 |
402 |
synth_body = VSB_new_auto(); |
1005 |
402 |
AN(synth_body); |
1006 |
|
|
1007 |
402 |
VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body); |
1008 |
|
|
1009 |
402 |
AZ(VSB_finish(synth_body)); |
1010 |
|
|
1011 |
402 |
if (wrk->vpi->handling == VCL_RET_ABANDON || wrk->vpi->handling == VCL_RET_FAIL) { |
1012 |
28 |
VSB_destroy(&synth_body); |
1013 |
28 |
return (F_STP_FAIL); |
1014 |
|
} |
1015 |
|
|
1016 |
374 |
if (wrk->vpi->handling == VCL_RET_RETRY) { |
1017 |
10 |
VSB_destroy(&synth_body); |
1018 |
10 |
if (bo->retries++ < bo->max_retries) |
1019 |
9 |
return (F_STP_RETRY); |
1020 |
1 |
VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing"); |
1021 |
1 |
return (F_STP_FAIL); |
1022 |
|
} |
1023 |
|
|
1024 |
364 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
1025 |
|
|
1026 |
364 |
assert(bo->vfc->wrk == bo->wrk); |
1027 |
364 |
assert(bo->vfc->oc == oc); |
1028 |
364 |
assert(bo->vfc->resp == bo->beresp); |
1029 |
364 |
assert(bo->vfc->req == bo->bereq); |
1030 |
|
|
1031 |
364 |
if (vbf_beresp2obj(bo)) { |
1032 |
2 |
VSB_destroy(&synth_body); |
1033 |
2 |
return (F_STP_FAIL); |
1034 |
|
} |
1035 |
|
|
1036 |
362 |
oc->boc->transit_buffer = 0; |
1037 |
|
|
1038 |
362 |
ll = VSB_len(synth_body); |
1039 |
362 |
o = 0; |
1040 |
675 |
while (ll > 0) { |
1041 |
314 |
l = ll; |
1042 |
314 |
if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) { |
1043 |
1 |
VSB_destroy(&synth_body); |
1044 |
1 |
return (F_STP_FAIL); |
1045 |
|
} |
1046 |
313 |
l = vmin(l, ll); |
1047 |
313 |
memcpy(ptr, VSB_data(synth_body) + o, l); |
1048 |
313 |
VFP_Extend(bo->vfc, l, l == ll ? VFP_END : VFP_OK); |
1049 |
313 |
ll -= l; |
1050 |
313 |
o += l; |
1051 |
|
} |
1052 |
361 |
assert(o == VSB_len(synth_body)); |
1053 |
361 |
AZ(ObjSetU64(wrk, oc, OA_LEN, o)); |
1054 |
361 |
VSB_destroy(&synth_body); |
1055 |
361 |
if (stale != NULL && oc->ttl > 0) |
1056 |
22 |
HSH_Kill(stale); |
1057 |
361 |
VBO_SetState(wrk, bo, BOS_FINISHED); |
1058 |
361 |
return (F_STP_DONE); |
1059 |
402 |
} |
1060 |
|
|
1061 |
|
/*-------------------------------------------------------------------- |
1062 |
|
*/ |
1063 |
|
|
1064 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
1065 |
68 |
vbf_stp_fail(struct worker *wrk, struct busyobj *bo) |
1066 |
|
{ |
1067 |
|
struct objcore *oc; |
1068 |
|
|
1069 |
68 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1070 |
68 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1071 |
68 |
oc = bo->fetch_objcore; |
1072 |
68 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1073 |
|
|
1074 |
68 |
assert(oc->boc->state < BOS_FINISHED); |
1075 |
68 |
VBO_SetState(wrk, bo, BOS_FAILED); |
1076 |
68 |
HSH_Kill(oc); |
1077 |
68 |
return (F_STP_DONE); |
1078 |
|
} |
1079 |
|
|
1080 |
|
/*-------------------------------------------------------------------- |
1081 |
|
*/ |
1082 |
|
|
1083 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
1084 |
0 |
vbf_stp_done(struct worker *wrk, struct busyobj *bo) |
1085 |
|
{ |
1086 |
|
|
1087 |
0 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1088 |
0 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1089 |
0 |
WRONG("Just plain wrong"); |
1090 |
0 |
NEEDLESS(return (F_STP_DONE)); |
1091 |
|
} |
1092 |
|
|
1093 |
|
static void v_matchproto_(task_func_t) |
1094 |
2273 |
vbf_fetch_thread(struct worker *wrk, void *priv) |
1095 |
|
{ |
1096 |
|
struct vrt_ctx ctx[1]; |
1097 |
|
struct busyobj *bo; |
1098 |
|
struct objcore *oc; |
1099 |
|
const struct fetch_step *stp; |
1100 |
|
|
1101 |
2273 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1102 |
2273 |
CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); |
1103 |
2273 |
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); |
1104 |
2273 |
oc = bo->fetch_objcore; |
1105 |
2273 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1106 |
|
|
1107 |
2273 |
THR_SetBusyobj(bo); |
1108 |
2273 |
stp = F_STP_MKBEREQ; |
1109 |
2273 |
assert(isnan(bo->t_first)); |
1110 |
2273 |
assert(isnan(bo->t_prev)); |
1111 |
2273 |
VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk)); |
1112 |
|
|
1113 |
2273 |
bo->wrk = wrk; |
1114 |
2273 |
wrk->vsl = bo->vsl; |
1115 |
|
|
1116 |
|
#if 0 |
1117 |
|
if (bo->stale_oc != NULL) { |
1118 |
|
CHECK_OBJ_NOTNULL(bo->stale_oc, OBJCORE_MAGIC); |
1119 |
|
/* We don't want the oc/stevedore ops in fetching thread */ |
1120 |
|
if (!ObjCheckFlag(wrk, bo->stale_oc, OF_IMSCAND)) |
1121 |
|
(void)HSH_DerefObjCore(wrk, &bo->stale_oc, 0); |
1122 |
|
} |
1123 |
|
#endif |
1124 |
|
|
1125 |
2273 |
VCL_TaskEnter(bo->privs); |
1126 |
12268 |
while (stp != F_STP_DONE) { |
1127 |
9995 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1128 |
9995 |
assert(oc->boc->refcount >= 1); |
1129 |
9995 |
if (oc->boc->state < BOS_REQ_DONE) |
1130 |
2363 |
AN(bo->req); |
1131 |
|
else |
1132 |
7632 |
AZ(bo->req); |
1133 |
9995 |
AN(stp); |
1134 |
9995 |
AN(stp->name); |
1135 |
9995 |
AN(stp->func); |
1136 |
9995 |
stp = stp->func(wrk, bo); |
1137 |
|
} |
1138 |
|
|
1139 |
2273 |
assert(bo->director_state == DIR_S_NULL); |
1140 |
|
|
1141 |
2273 |
INIT_OBJ(ctx, VRT_CTX_MAGIC); |
1142 |
2273 |
VCL_Bo2Ctx(ctx, bo); |
1143 |
2273 |
VCL_TaskLeave(ctx, bo->privs); |
1144 |
2273 |
http_Teardown(bo->bereq); |
1145 |
2273 |
http_Teardown(bo->beresp); |
1146 |
|
// cannot make assumptions about the number of references here #3434 |
1147 |
2273 |
if (bo->bereq_body != NULL) |
1148 |
18 |
(void)HSH_DerefObjCore(bo->wrk, &bo->bereq_body); |
1149 |
|
|
1150 |
2273 |
if (oc->boc->state == BOS_FINISHED) { |
1151 |
2205 |
AZ(oc->flags & OC_F_FAILED); |
1152 |
4410 |
VSLb(bo->vsl, SLT_Length, "%ju", |
1153 |
2205 |
(uintmax_t)ObjGetLen(bo->wrk, oc)); |
1154 |
2205 |
} |
1155 |
|
// AZ(oc->boc); // XXX |
1156 |
|
|
1157 |
2273 |
if (bo->stale_oc != NULL) |
1158 |
154 |
(void)HSH_DerefObjCore(wrk, &bo->stale_oc); |
1159 |
|
|
1160 |
2273 |
wrk->vsl = NULL; |
1161 |
2273 |
HSH_DerefBoc(wrk, oc); |
1162 |
2273 |
SES_Rel(bo->sp); |
1163 |
2273 |
VBO_ReleaseBusyObj(wrk, &bo); |
1164 |
2273 |
THR_SetBusyobj(NULL); |
1165 |
2273 |
} |
1166 |
|
|
1167 |
|
/*-------------------------------------------------------------------- |
1168 |
|
*/ |
1169 |
|
|
1170 |
|
void |
1171 |
2274 |
VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, |
1172 |
|
struct objcore *oldoc, enum vbf_fetch_mode_e mode) |
1173 |
|
{ |
1174 |
|
struct boc *boc; |
1175 |
|
struct busyobj *bo; |
1176 |
|
enum task_prio prio; |
1177 |
|
const char *how; |
1178 |
|
|
1179 |
2274 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1180 |
2274 |
CHECK_OBJ_NOTNULL(req, REQ_MAGIC); |
1181 |
2274 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1182 |
2274 |
CHECK_OBJ_ORNULL(oldoc, OBJCORE_MAGIC); |
1183 |
|
|
1184 |
2274 |
bo = VBO_GetBusyObj(wrk, req); |
1185 |
2274 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1186 |
2274 |
AN(bo->vcl); |
1187 |
|
|
1188 |
2274 |
boc = HSH_RefBoc(oc); |
1189 |
2274 |
CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); |
1190 |
2274 |
assert(boc->state < BOS_STREAM); |
1191 |
2274 |
boc->transit_buffer = cache_param->transit_buffer; |
1192 |
|
|
1193 |
2274 |
switch (mode) { |
1194 |
|
case VBF_PASS: |
1195 |
784 |
prio = TASK_QUEUE_BO; |
1196 |
784 |
how = "pass"; |
1197 |
784 |
bo->uncacheable = 1; |
1198 |
784 |
break; |
1199 |
|
case VBF_NORMAL: |
1200 |
1403 |
prio = TASK_QUEUE_BO; |
1201 |
1403 |
how = "fetch"; |
1202 |
1403 |
break; |
1203 |
|
case VBF_BACKGROUND: |
1204 |
87 |
prio = TASK_QUEUE_BG; |
1205 |
87 |
how = "bgfetch"; |
1206 |
87 |
bo->is_bgfetch = 1; |
1207 |
87 |
break; |
1208 |
|
default: |
1209 |
0 |
WRONG("Wrong fetch mode"); |
1210 |
0 |
} |
1211 |
|
|
1212 |
|
#define REQ_BEREQ_FLAG(l, r, w, d) bo->l = req->l; |
1213 |
|
#include "tbl/req_bereq_flags.h" |
1214 |
|
|
1215 |
|
VSLb(bo->vsl, SLT_Begin, "bereq %ju %s", VXID(req->vsl->wid), how); |
1216 |
|
VSLbs(bo->vsl, SLT_VCL_use, TOSTRAND(VCL_Name(bo->vcl))); |
1217 |
|
VSLb(req->vsl, SLT_Link, "bereq %ju %s", VXID(bo->vsl->wid), how); |
1218 |
|
|
1219 |
|
THR_SetBusyobj(bo); |
1220 |
|
|
1221 |
|
bo->sp = req->sp; |
1222 |
|
SES_Ref(bo->sp); |
1223 |
|
|
1224 |
|
oc->boc->vary = req->vary_b; |
1225 |
|
req->vary_b = NULL; |
1226 |
|
|
1227 |
|
HSH_Ref(oc); |
1228 |
2274 |
AZ(bo->fetch_objcore); |
1229 |
|
bo->fetch_objcore = oc; |
1230 |
|
|
1231 |
2274 |
AZ(bo->stale_oc); |
1232 |
2274 |
if (oldoc != NULL) { |
1233 |
155 |
assert(oldoc->refcnt > 0); |
1234 |
155 |
HSH_Ref(oldoc); |
1235 |
155 |
bo->stale_oc = oldoc; |
1236 |
155 |
} |
1237 |
|
|
1238 |
2274 |
AZ(bo->req); |
1239 |
|
bo->req = req; |
1240 |
|
|
1241 |
|
bo->fetch_task->priv = bo; |
1242 |
|
bo->fetch_task->func = vbf_fetch_thread; |
1243 |
|
|
1244 |
2274 |
if (Pool_Task(wrk->pool, bo->fetch_task, prio)) { |
1245 |
2 |
wrk->stats->bgfetch_no_thread++; |
1246 |
2 |
VSLb(bo->vsl, SLT_FetchError, |
1247 |
|
"No thread available for bgfetch"); |
1248 |
2 |
(void)vbf_stp_fail(req->wrk, bo); |
1249 |
2 |
if (bo->stale_oc != NULL) |
1250 |
1 |
(void)HSH_DerefObjCore(wrk, &bo->stale_oc); |
1251 |
2 |
HSH_DerefBoc(wrk, oc); |
1252 |
2 |
SES_Rel(bo->sp); |
1253 |
2 |
THR_SetBusyobj(NULL); |
1254 |
2 |
VBO_ReleaseBusyObj(wrk, &bo); |
1255 |
2 |
} else { |
1256 |
2272 |
THR_SetBusyobj(NULL); |
1257 |
2272 |
bo = NULL; /* ref transferred to fetch thread */ |
1258 |
2272 |
if (mode == VBF_BACKGROUND) { |
1259 |
86 |
ObjWaitState(oc, BOS_REQ_DONE); |
1260 |
86 |
(void)VRB_Ignore(req); |
1261 |
86 |
} else { |
1262 |
2186 |
ObjWaitState(oc, BOS_STREAM); |
1263 |
2186 |
AZ(oc->flags & OC_F_BUSY); |
1264 |
2186 |
if (oc->boc->state == BOS_FAILED) |
1265 |
27 |
AN(oc->flags & OC_F_FAILED); |
1266 |
|
} |
1267 |
|
} |
1268 |
2274 |
AZ(bo); |
1269 |
|
VSLb_ts_req(req, "Fetch", W_TIM_real(wrk)); |
1270 |
2274 |
assert(oc->boc == boc); |
1271 |
|
HSH_DerefBoc(wrk, oc); |
1272 |
2274 |
if (mode == VBF_BACKGROUND) |
1273 |
87 |
(void)HSH_DerefObjCore(wrk, &oc); |
1274 |
|
} |