| | varnish-cache/bin/varnishd/cache/cache_fetch.c |
0 |
|
/*- |
1 |
|
* Copyright (c) 2006 Verdens Gang AS |
2 |
|
* Copyright (c) 2006-2015 Varnish Software AS |
3 |
|
* All rights reserved. |
4 |
|
* |
5 |
|
* Author: Poul-Henning Kamp <phk@phk.freebsd.dk> |
6 |
|
* |
7 |
|
* SPDX-License-Identifier: BSD-2-Clause |
8 |
|
* |
9 |
|
* Redistribution and use in source and binary forms, with or without |
10 |
|
* modification, are permitted provided that the following conditions |
11 |
|
* are met: |
12 |
|
* 1. Redistributions of source code must retain the above copyright |
13 |
|
* notice, this list of conditions and the following disclaimer. |
14 |
|
* 2. Redistributions in binary form must reproduce the above copyright |
15 |
|
* notice, this list of conditions and the following disclaimer in the |
16 |
|
* documentation and/or other materials provided with the distribution. |
17 |
|
* |
18 |
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
19 |
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
20 |
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
21 |
|
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE |
22 |
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
23 |
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
24 |
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
25 |
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
26 |
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
27 |
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
28 |
|
* SUCH DAMAGE. |
29 |
|
*/ |
30 |
|
|
31 |
|
#include "config.h" |
32 |
|
|
33 |
|
#include "cache_varnishd.h" |
34 |
53575 |
#include "cache_filter.h" |
35 |
53575 |
#include "cache_objhead.h" |
36 |
53575 |
#include "storage/storage.h" |
37 |
|
#include "vcl.h" |
38 |
|
#include "vtim.h" |
39 |
|
#include "vcc_interface.h" |
40 |
|
|
41 |
|
#define FETCH_STEPS \ |
42 |
|
FETCH_STEP(mkbereq, MKBEREQ) \ |
43 |
|
FETCH_STEP(retry, RETRY) \ |
44 |
|
FETCH_STEP(startfetch, STARTFETCH) \ |
45 |
|
FETCH_STEP(condfetch, CONDFETCH) \ |
46 |
|
FETCH_STEP(fetch, FETCH) \ |
47 |
|
FETCH_STEP(fetchbody, FETCHBODY) \ |
48 |
|
FETCH_STEP(fetchend, FETCHEND) \ |
49 |
|
FETCH_STEP(error, ERROR) \ |
50 |
|
FETCH_STEP(fail, FAIL) \ |
51 |
|
FETCH_STEP(done, DONE) |
52 |
|
|
53 |
|
typedef const struct fetch_step *vbf_state_f(struct worker *, struct busyobj *); |
54 |
|
|
55 |
|
struct fetch_step { |
56 |
|
const char *name; |
57 |
|
vbf_state_f *func; |
58 |
44424 |
}; |
59 |
44424 |
|
60 |
44424 |
#define FETCH_STEP(l, U) \ |
61 |
44424 |
static vbf_state_f vbf_stp_##l; \ |
62 |
|
static const struct fetch_step F_STP_##U[1] = {{ .name = "Fetch Step " #l, .func = vbf_stp_##l, }}; |
63 |
|
FETCH_STEPS |
64 |
|
#undef FETCH_STEP |
65 |
|
|
66 |
|
/*-------------------------------------------------------------------- |
67 |
|
* Allocate an object, with fall-back to Transient. |
68 |
|
* XXX: This somewhat overlaps the stuff in stevedore.c |
69 |
|
* XXX: Should this be merged over there ? |
70 |
|
*/ |
71 |
|
|
72 |
|
static int |
73 |
52950 |
vbf_allocobj(struct busyobj *bo, unsigned l) |
74 |
|
{ |
75 |
|
struct objcore *oc; |
76 |
|
const struct stevedore *stv; |
77 |
|
vtim_dur lifetime; |
78 |
|
|
79 |
52950 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
80 |
52950 |
oc = bo->fetch_objcore; |
81 |
52950 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
82 |
|
|
83 |
52950 |
lifetime = oc->ttl + oc->grace + oc->keep; |
84 |
|
|
85 |
52950 |
if (bo->uncacheable) { |
86 |
20650 |
stv = stv_transient; |
87 |
20650 |
bo->wrk->stats->beresp_uncacheable++; |
88 |
20650 |
} |
89 |
32300 |
else if (lifetime < cache_param->shortlived) { |
90 |
2951 |
stv = stv_transient; |
91 |
2951 |
bo->wrk->stats->beresp_shortlived++; |
92 |
2951 |
} |
93 |
|
else |
94 |
29349 |
stv = bo->storage; |
95 |
|
|
96 |
52950 |
bo->storage = NULL; |
97 |
|
|
98 |
52950 |
if (stv == NULL) |
99 |
25 |
return (0); |
100 |
|
|
101 |
52925 |
if (STV_NewObject(bo->wrk, oc, stv, l)) |
102 |
52750 |
return (1); |
103 |
|
|
104 |
175 |
if (stv == stv_transient) |
105 |
100 |
return (0); |
106 |
|
|
107 |
|
/* |
108 |
|
* Try to salvage the transaction by allocating a shortlived object |
109 |
|
* on Transient storage. |
110 |
|
*/ |
111 |
|
|
112 |
75 |
oc->ttl = vmin_t(float, oc->ttl, cache_param->shortlived); |
113 |
75 |
oc->grace = 0.0; |
114 |
75 |
oc->keep = 0.0; |
115 |
75 |
return (STV_NewObject(bo->wrk, oc, stv_transient, l)); |
116 |
52950 |
} |
117 |
|
|
118 |
|
static void |
119 |
48424 |
vbf_cleanup(struct busyobj *bo) |
120 |
|
{ |
121 |
|
struct vfp_ctx *vfc; |
122 |
|
|
123 |
48424 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
124 |
48424 |
vfc = bo->vfc; |
125 |
48424 |
CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); |
126 |
|
|
127 |
48424 |
bo->acct.beresp_bodybytes += VFP_Close(vfc); |
128 |
48424 |
bo->vfp_filter_list = NULL; |
129 |
|
|
130 |
48424 |
if (bo->director_state != DIR_S_NULL) |
131 |
48223 |
VDI_Finish(bo); |
132 |
48424 |
} |
133 |
|
|
134 |
|
void |
135 |
200 |
Bereq_Rollback(VRT_CTX) |
136 |
|
{ |
137 |
|
struct busyobj *bo; |
138 |
|
|
139 |
200 |
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); |
140 |
200 |
bo = ctx->bo; |
141 |
200 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
142 |
|
|
143 |
225 |
if (bo->htc != NULL && |
144 |
175 |
bo->htc->body_status != BS_NONE && |
145 |
25 |
bo->htc->body_status != BS_TAKEN) |
146 |
25 |
bo->htc->doclose = SC_RESP_CLOSE; |
147 |
|
|
148 |
200 |
vbf_cleanup(bo); |
149 |
200 |
VCL_TaskLeave(ctx, bo->privs); |
150 |
200 |
VCL_TaskEnter(bo->privs); |
151 |
200 |
HTTP_Clone(bo->bereq, bo->bereq0); |
152 |
200 |
bo->vfp_filter_list = NULL; |
153 |
200 |
bo->err_reason = NULL; |
154 |
200 |
AN(bo->ws_bo); |
155 |
200 |
WS_Rollback(bo->ws, bo->ws_bo); |
156 |
200 |
} |
157 |
|
|
158 |
|
/*-------------------------------------------------------------------- |
159 |
|
* Turn the beresp into a obj |
160 |
|
*/ |
161 |
|
|
162 |
|
static int |
163 |
52948 |
vbf_beresp2obj(struct busyobj *bo) |
164 |
|
{ |
165 |
|
unsigned l, l2; |
166 |
|
const char *b; |
167 |
|
uint8_t *bp; |
168 |
52948 |
struct vsb *vary = NULL; |
169 |
52948 |
int varyl = 0; |
170 |
|
struct objcore *oc; |
171 |
|
|
172 |
52948 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
173 |
52948 |
oc = bo->fetch_objcore; |
174 |
52948 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
175 |
|
|
176 |
52948 |
l = 0; |
177 |
|
|
178 |
|
/* Create Vary instructions */ |
179 |
52948 |
if (!(oc->flags & OC_F_PRIVATE)) { |
180 |
34650 |
varyl = VRY_Create(bo, &vary); |
181 |
34650 |
if (varyl > 0) { |
182 |
5100 |
AN(vary); |
183 |
5100 |
assert(varyl == VSB_len(vary)); |
184 |
5100 |
l += PRNDUP((intptr_t)varyl); |
185 |
34650 |
} else if (varyl < 0) { |
186 |
|
/* |
187 |
|
* Vary parse error |
188 |
|
* Complain about it, and make this a pass. |
189 |
|
*/ |
190 |
125 |
VSLb(bo->vsl, SLT_Error, |
191 |
|
"Illegal 'Vary' header from backend, " |
192 |
|
"making this a pass."); |
193 |
125 |
bo->uncacheable = 1; |
194 |
125 |
AZ(vary); |
195 |
125 |
} else |
196 |
|
/* No vary */ |
197 |
29425 |
AZ(vary); |
198 |
34650 |
} |
199 |
|
|
200 |
105896 |
l2 = http_EstimateWS(bo->beresp, |
201 |
52948 |
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS); |
202 |
52948 |
l += l2; |
203 |
|
|
204 |
52948 |
if (bo->uncacheable) |
205 |
20649 |
oc->flags |= OC_F_HFM; |
206 |
|
|
207 |
52948 |
if (!vbf_allocobj(bo, l)) { |
208 |
150 |
if (vary != NULL) |
209 |
0 |
VSB_destroy(&vary); |
210 |
150 |
AZ(vary); |
211 |
150 |
return (VFP_Error(bo->vfc, "Could not get storage")); |
212 |
|
} |
213 |
|
|
214 |
52798 |
if (vary != NULL) { |
215 |
5100 |
AN(ObjSetAttr(bo->wrk, oc, OA_VARY, varyl, VSB_data(vary))); |
216 |
5100 |
VSB_destroy(&vary); |
217 |
5100 |
} |
218 |
|
|
219 |
52798 |
AZ(ObjSetXID(bo->wrk, oc, bo->vsl->wid)); |
220 |
|
|
221 |
|
/* for HTTP_Encode() VSLH call */ |
222 |
52798 |
bo->beresp->logtag = SLT_ObjMethod; |
223 |
|
|
224 |
|
/* Filter into object */ |
225 |
52798 |
bp = ObjSetAttr(bo->wrk, oc, OA_HEADERS, l2, NULL); |
226 |
52798 |
AN(bp); |
227 |
105596 |
HTTP_Encode(bo->beresp, bp, l2, |
228 |
52798 |
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS); |
229 |
|
|
230 |
52798 |
if (http_GetHdr(bo->beresp, H_Last_Modified, &b)) |
231 |
1049 |
AZ(ObjSetDouble(bo->wrk, oc, OA_LASTMODIFIED, VTIM_parse(b))); |
232 |
|
else |
233 |
51749 |
AZ(ObjSetDouble(bo->wrk, oc, OA_LASTMODIFIED, |
234 |
|
floor(oc->t_origin))); |
235 |
|
|
236 |
52798 |
return (0); |
237 |
52948 |
} |
238 |
|
|
239 |
|
/*-------------------------------------------------------------------- |
240 |
|
* Copy req->bereq and release req if no body |
241 |
|
*/ |
242 |
|
|
243 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
244 |
53549 |
vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) |
245 |
|
{ |
246 |
|
const char *q; |
247 |
|
struct objcore *oc; |
248 |
|
|
249 |
53549 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
250 |
53549 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
251 |
53549 |
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); |
252 |
53549 |
oc = bo->fetch_objcore; |
253 |
53549 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
254 |
|
|
255 |
53549 |
assert(oc->boc->state == BOS_INVALID); |
256 |
53549 |
AZ(bo->storage); |
257 |
|
|
258 |
53549 |
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod); |
259 |
107098 |
http_FilterReq(bo->bereq0, bo->req->http, |
260 |
53549 |
bo->uncacheable ? HTTPH_R_PASS : HTTPH_R_FETCH); |
261 |
|
|
262 |
53549 |
if (bo->uncacheable) |
263 |
18300 |
AZ(bo->stale_oc); |
264 |
|
else { |
265 |
35249 |
http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET"); |
266 |
35249 |
if (cache_param->http_gzip_support) |
267 |
35123 |
http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip"); |
268 |
|
} |
269 |
53549 |
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1"); |
270 |
|
|
271 |
54224 |
if (bo->stale_oc != NULL && |
272 |
3500 |
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) && |
273 |
750 |
(bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) { |
274 |
725 |
AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); |
275 |
725 |
q = RFC2616_Strong_LM(NULL, wrk, bo->stale_oc); |
276 |
725 |
if (q != NULL) |
277 |
600 |
http_PrintfHeader(bo->bereq0, |
278 |
300 |
"If-Modified-Since: %s", q); |
279 |
725 |
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_ETag); |
280 |
725 |
if (q != NULL) |
281 |
900 |
http_PrintfHeader(bo->bereq0, |
282 |
450 |
"If-None-Match: %s", q); |
283 |
725 |
} |
284 |
|
|
285 |
53549 |
http_CopyHome(bo->bereq0); |
286 |
53549 |
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); |
287 |
53549 |
bo->ws_bo = WS_Snapshot(bo->ws); |
288 |
53549 |
HTTP_Clone(bo->bereq, bo->bereq0); |
289 |
|
|
290 |
53549 |
if (bo->req->req_body_status->avail == 0) { |
291 |
51804 |
bo->req = NULL; |
292 |
51804 |
ObjSetState(bo->wrk, oc, BOS_REQ_DONE); |
293 |
53549 |
} else if (bo->req->req_body_status == BS_CACHED) { |
294 |
425 |
AN(bo->req->body_oc); |
295 |
425 |
bo->bereq_body = bo->req->body_oc; |
296 |
425 |
HSH_Ref(bo->bereq_body); |
297 |
425 |
bo->req = NULL; |
298 |
425 |
ObjSetState(bo->wrk, oc, BOS_REQ_DONE); |
299 |
425 |
} |
300 |
53549 |
return (F_STP_STARTFETCH); |
301 |
|
} |
302 |
|
|
303 |
|
/*-------------------------------------------------------------------- |
304 |
|
* Start a new VSL transaction and try again |
305 |
|
* Prepare the busyobj and fetch processors |
306 |
|
*/ |
307 |
|
|
308 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
309 |
900 |
vbf_stp_retry(struct worker *wrk, struct busyobj *bo) |
310 |
|
{ |
311 |
900 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
312 |
900 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
313 |
|
|
314 |
900 |
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); |
315 |
|
|
316 |
900 |
if (bo->no_retry != NULL) { |
317 |
100 |
VSLb(bo->vsl, SLT_Error, |
318 |
50 |
"Retry not possible, %s", bo->no_retry); |
319 |
50 |
return (F_STP_FAIL); |
320 |
|
} |
321 |
|
|
322 |
850 |
VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk)); |
323 |
|
|
324 |
|
/* VDI_Finish (via vbf_cleanup) must have been called before */ |
325 |
850 |
assert(bo->director_state == DIR_S_NULL); |
326 |
|
|
327 |
|
/* reset other bo attributes - See VBO_GetBusyObj */ |
328 |
850 |
bo->storage = NULL; |
329 |
850 |
bo->do_esi = 0; |
330 |
850 |
bo->do_stream = 1; |
331 |
850 |
bo->was_304 = 0; |
332 |
850 |
bo->err_code = 0; |
333 |
850 |
bo->err_reason = NULL; |
334 |
850 |
if (bo->htc != NULL) |
335 |
0 |
bo->htc->doclose = SC_NULL; |
336 |
|
|
337 |
|
// XXX: BereqEnd + BereqAcct ? |
338 |
850 |
VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); |
339 |
850 |
VSLb_ts_busyobj(bo, "Start", bo->t_prev); |
340 |
850 |
http_VSL_log(bo->bereq); |
341 |
|
|
342 |
850 |
return (F_STP_STARTFETCH); |
343 |
900 |
} |
344 |
|
|
345 |
|
/*-------------------------------------------------------------------- |
346 |
|
* 304 setup logic |
347 |
|
*/ |
348 |
|
|
349 |
|
static int |
350 |
799 |
vbf_304_logic(struct busyobj *bo) |
351 |
|
{ |
352 |
799 |
if (bo->stale_oc != NULL && |
353 |
699 |
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) { |
354 |
674 |
AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); |
355 |
674 |
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) { |
356 |
|
/* |
357 |
|
* If a VFP changed C-E in the stored |
358 |
|
* object, then don't overwrite C-E from |
359 |
|
* the IMS fetch, and we must weaken any |
360 |
|
* new ETag we get. |
361 |
|
*/ |
362 |
50 |
RFC2616_Weaken_Etag(bo->beresp); |
363 |
50 |
} |
364 |
674 |
http_Unset(bo->beresp, H_Content_Encoding); |
365 |
674 |
http_Unset(bo->beresp, H_Content_Length); |
366 |
674 |
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); |
367 |
674 |
assert(http_IsStatus(bo->beresp, 200)); |
368 |
674 |
bo->was_304 = 1; |
369 |
799 |
} else if (!bo->uncacheable) { |
370 |
|
/* |
371 |
|
* Backend sent unallowed 304 |
372 |
|
*/ |
373 |
25 |
VSLb(bo->vsl, SLT_Error, |
374 |
|
"304 response but not conditional fetch"); |
375 |
25 |
bo->htc->doclose = SC_RX_BAD; |
376 |
25 |
vbf_cleanup(bo); |
377 |
25 |
return (-1); |
378 |
|
} |
379 |
774 |
return (1); |
380 |
799 |
} |
381 |
|
|
382 |
|
/*-------------------------------------------------------------------- |
383 |
|
* Setup bereq from bereq0, run vcl_backend_fetch |
384 |
|
*/ |
385 |
|
|
386 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
387 |
54397 |
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) |
388 |
|
{ |
389 |
|
int i; |
390 |
|
vtim_real now; |
391 |
|
unsigned handling; |
392 |
|
struct objcore *oc; |
393 |
|
|
394 |
54397 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
395 |
54397 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
396 |
54397 |
oc = bo->fetch_objcore; |
397 |
54397 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
398 |
|
|
399 |
54397 |
AZ(bo->storage); |
400 |
54397 |
bo->storage = bo->uncacheable ? stv_transient : STV_next(); |
401 |
|
|
402 |
54397 |
if (bo->retries > 0) |
403 |
850 |
http_Unset(bo->bereq, "\012X-Varnish:"); |
404 |
|
|
405 |
54397 |
http_PrintfHeader(bo->bereq, "X-Varnish: %ju", VXID(bo->vsl->wid)); |
406 |
|
|
407 |
54397 |
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); |
408 |
|
|
409 |
54397 |
if (wrk->vpi->handling == VCL_RET_ABANDON || |
410 |
54321 |
wrk->vpi->handling == VCL_RET_FAIL) |
411 |
196 |
return (F_STP_FAIL); |
412 |
|
|
413 |
54201 |
assert (wrk->vpi->handling == VCL_RET_FETCH || |
414 |
|
wrk->vpi->handling == VCL_RET_ERROR); |
415 |
|
|
416 |
54201 |
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); |
417 |
|
|
418 |
54201 |
assert(oc->boc->state <= BOS_REQ_DONE); |
419 |
|
|
420 |
54201 |
AZ(bo->htc); |
421 |
|
|
422 |
54201 |
VFP_Setup(bo->vfc, wrk); |
423 |
54201 |
bo->vfc->oc = oc; |
424 |
54201 |
bo->vfc->resp = bo->beresp; |
425 |
54201 |
bo->vfc->req = bo->bereq; |
426 |
|
|
427 |
54201 |
if (wrk->vpi->handling == VCL_RET_ERROR) |
428 |
200 |
return (F_STP_ERROR); |
429 |
|
|
430 |
54001 |
VSLb_ts_busyobj(bo, "Fetch", W_TIM_real(wrk)); |
431 |
54001 |
i = VDI_GetHdr(bo); |
432 |
54001 |
if (bo->htc != NULL) |
433 |
48424 |
CHECK_OBJ_NOTNULL(bo->htc->doclose, STREAM_CLOSE_MAGIC); |
434 |
|
|
435 |
54001 |
bo->t_resp = now = W_TIM_real(wrk); |
436 |
54001 |
VSLb_ts_busyobj(bo, "Beresp", now); |
437 |
|
|
438 |
54001 |
if (i) { |
439 |
5551 |
assert(bo->director_state == DIR_S_NULL); |
440 |
5551 |
return (F_STP_ERROR); |
441 |
|
} |
442 |
|
|
443 |
48450 |
if (bo->htc != NULL && bo->htc->body_status == BS_ERROR) { |
444 |
150 |
bo->htc->doclose = SC_RX_BODY; |
445 |
150 |
vbf_cleanup(bo); |
446 |
150 |
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); |
447 |
150 |
assert(bo->director_state == DIR_S_NULL); |
448 |
150 |
return (F_STP_ERROR); |
449 |
|
} |
450 |
|
|
451 |
48300 |
if (!http_GetHdr(bo->beresp, H_Date, NULL)) { |
452 |
|
/* |
453 |
|
* RFC 2616 14.18 Date: The Date general-header field |
454 |
|
* represents the date and time at which the message was |
455 |
|
* originated, having the same semantics as orig-date in |
456 |
|
* RFC 822. ... A received message that does not have a |
457 |
|
* Date header field MUST be assigned one by the recipient |
458 |
|
* if the message will be cached by that recipient or |
459 |
|
* gatewayed via a protocol which requires a Date. |
460 |
|
* |
461 |
|
* If we didn't get a Date header, we assign one here. |
462 |
|
*/ |
463 |
2300 |
http_TimeHeader(bo->beresp, "Date: ", now); |
464 |
2300 |
} |
465 |
|
|
466 |
|
/* |
467 |
|
* These two headers can be spread over multiple actual headers |
468 |
|
* and we rely on their content outside of VCL, so collect them |
469 |
|
* into one line here. |
470 |
|
*/ |
471 |
48298 |
http_CollectHdr(bo->beresp, H_Cache_Control); |
472 |
48298 |
http_CollectHdr(bo->beresp, H_Vary); |
473 |
|
|
474 |
|
/* What does RFC2616 think about TTL ? */ |
475 |
96596 |
RFC2616_Ttl(bo, now, |
476 |
48298 |
&oc->t_origin, |
477 |
48298 |
&oc->ttl, |
478 |
48298 |
&oc->grace, |
479 |
48298 |
&oc->keep); |
480 |
|
|
481 |
48298 |
AZ(bo->do_esi); |
482 |
48298 |
AZ(bo->was_304); |
483 |
|
|
484 |
48298 |
if (http_IsStatus(bo->beresp, 304) && vbf_304_logic(bo) < 0) |
485 |
25 |
return (F_STP_ERROR); |
486 |
|
|
487 |
48273 |
if (bo->htc != NULL && bo->htc->doclose == SC_NULL && |
488 |
45968 |
http_GetHdrField(bo->bereq, H_Connection, "close", NULL)) |
489 |
325 |
bo->htc->doclose = SC_REQ_CLOSE; |
490 |
|
|
491 |
48273 |
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); |
492 |
|
|
493 |
48273 |
if (bo->htc != NULL && bo->htc->doclose == SC_NULL && |
494 |
45442 |
http_GetHdrField(bo->beresp, H_Connection, "close", NULL)) |
495 |
25 |
bo->htc->doclose = SC_RESP_CLOSE; |
496 |
|
|
497 |
48223 |
if (VRG_CheckBo(bo) < 0) { |
498 |
150 |
VDI_Finish(bo); |
499 |
150 |
return (F_STP_ERROR); |
500 |
|
} |
501 |
|
|
502 |
96020 |
if (wrk->vpi->handling == VCL_RET_ABANDON || |
503 |
47997 |
wrk->vpi->handling == VCL_RET_FAIL || |
504 |
47947 |
wrk->vpi->handling == VCL_RET_ERROR) { |
505 |
|
/* do not count deliberately ending the backend connection as |
506 |
|
* fetch failure |
507 |
|
*/ |
508 |
225 |
handling = wrk->vpi->handling; |
509 |
225 |
if (bo->htc) |
510 |
225 |
bo->htc->doclose = SC_RESP_CLOSE; |
511 |
225 |
vbf_cleanup(bo); |
512 |
225 |
wrk->vpi->handling = handling; |
513 |
|
|
514 |
225 |
if (wrk->vpi->handling == VCL_RET_ERROR) |
515 |
100 |
return (F_STP_ERROR); |
516 |
|
else |
517 |
125 |
return (F_STP_FAIL); |
518 |
|
} |
519 |
|
|
520 |
47848 |
if (wrk->vpi->handling == VCL_RET_RETRY) { |
521 |
725 |
if (bo->htc && bo->htc->body_status != BS_NONE) |
522 |
175 |
bo->htc->doclose = SC_RESP_CLOSE; |
523 |
725 |
vbf_cleanup(bo); |
524 |
|
|
525 |
725 |
if (bo->retries++ < cache_param->max_retries) |
526 |
675 |
return (F_STP_RETRY); |
527 |
|
|
528 |
50 |
VSLb(bo->vsl, SLT_VCL_Error, |
529 |
|
"Too many retries, delivering 503"); |
530 |
50 |
assert(bo->director_state == DIR_S_NULL); |
531 |
50 |
return (F_STP_ERROR); |
532 |
|
} |
533 |
|
|
534 |
47123 |
VSLb_ts_busyobj(bo, "Process", W_TIM_real(wrk)); |
535 |
47123 |
assert(oc->boc->state <= BOS_REQ_DONE); |
536 |
47123 |
if (oc->boc->state != BOS_REQ_DONE) { |
537 |
825 |
bo->req = NULL; |
538 |
825 |
ObjSetState(wrk, oc, BOS_REQ_DONE); |
539 |
825 |
} |
540 |
|
|
541 |
47123 |
if (bo->do_esi) |
542 |
7675 |
bo->do_stream = 0; |
543 |
47123 |
if (wrk->vpi->handling == VCL_RET_PASS) { |
544 |
500 |
oc->flags |= OC_F_HFP; |
545 |
500 |
bo->uncacheable = 1; |
546 |
500 |
wrk->vpi->handling = VCL_RET_DELIVER; |
547 |
500 |
} |
548 |
47123 |
if (!bo->uncacheable || !bo->do_stream) |
549 |
32499 |
oc->boc->transit_buffer = 0; |
550 |
47123 |
if (bo->uncacheable) |
551 |
18074 |
oc->flags |= OC_F_HFM; |
552 |
|
|
553 |
47123 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
554 |
|
|
555 |
47123 |
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH); |
556 |
54345 |
} |
557 |
|
|
558 |
|
/*-------------------------------------------------------------------- |
559 |
|
*/ |
560 |
|
|
561 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
562 |
26848 |
vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) |
563 |
|
{ |
564 |
|
ssize_t l; |
565 |
|
uint8_t *ptr; |
566 |
26848 |
enum vfp_status vfps = VFP_ERROR; |
567 |
|
ssize_t est; |
568 |
|
struct vfp_ctx *vfc; |
569 |
|
struct objcore *oc; |
570 |
|
|
571 |
26848 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
572 |
26848 |
vfc = bo->vfc; |
573 |
26848 |
CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); |
574 |
26848 |
oc = bo->fetch_objcore; |
575 |
26848 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
576 |
|
|
577 |
26848 |
AN(vfc->vfp_nxt); |
578 |
|
|
579 |
26848 |
est = bo->htc->content_length; |
580 |
26848 |
if (est < 0) |
581 |
3850 |
est = 0; |
582 |
|
|
583 |
26848 |
do { |
584 |
1398918 |
if (oc->flags & OC_F_CANCEL) { |
585 |
|
/* |
586 |
|
* A pass object and delivery was terminated |
587 |
|
* We don't fail the fetch, in order for HitMiss |
588 |
|
* objects to be created. |
589 |
|
*/ |
590 |
73 |
AN(oc->flags & OC_F_HFM); |
591 |
73 |
VSLb(wrk->vsl, SLT_Debug, |
592 |
|
"Fetch: Pass delivery abandoned"); |
593 |
73 |
bo->htc->doclose = SC_RX_BODY; |
594 |
73 |
break; |
595 |
|
} |
596 |
1398845 |
AZ(vfc->failed); |
597 |
1398845 |
l = est; |
598 |
1398845 |
oc = bo->fetch_objcore; |
599 |
1398845 |
if (oc->boc->transit_buffer > 0) |
600 |
12110 |
l = vmin_t(ssize_t, l, oc->boc->transit_buffer); |
601 |
1398845 |
assert(l >= 0); |
602 |
1398845 |
if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { |
603 |
100 |
bo->htc->doclose = SC_RX_BODY; |
604 |
100 |
break; |
605 |
|
} |
606 |
|
|
607 |
1398745 |
AZ(vfc->failed); |
608 |
1398745 |
vfps = VFP_Suck(vfc, ptr, &l); |
609 |
1398745 |
if (l >= 0 && vfps != VFP_ERROR) { |
610 |
1397919 |
VFP_Extend(vfc, l, vfps); |
611 |
1397919 |
if (est >= l) |
612 |
70073 |
est -= l; |
613 |
|
else |
614 |
1327846 |
est = 0; |
615 |
1397919 |
} |
616 |
1398745 |
} while (vfps == VFP_OK); |
617 |
|
|
618 |
26848 |
if (vfc->failed) { |
619 |
900 |
(void)VFP_Error(vfc, "Fetch pipeline failed to process"); |
620 |
900 |
bo->htc->doclose = SC_RX_BODY; |
621 |
900 |
vbf_cleanup(bo); |
622 |
900 |
if (!bo->do_stream) { |
623 |
525 |
assert(oc->boc->state < BOS_STREAM); |
624 |
|
// XXX: doclose = ? |
625 |
525 |
return (F_STP_ERROR); |
626 |
|
} else { |
627 |
375 |
wrk->stats->fetch_failed++; |
628 |
375 |
return (F_STP_FAIL); |
629 |
|
} |
630 |
|
} |
631 |
|
|
632 |
25948 |
return (F_STP_FETCHEND); |
633 |
26848 |
} |
634 |
|
|
635 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
636 |
46474 |
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) |
637 |
|
{ |
638 |
|
struct vrt_ctx ctx[1]; |
639 |
|
struct objcore *oc; |
640 |
|
|
641 |
46474 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
642 |
46474 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
643 |
46474 |
oc = bo->fetch_objcore; |
644 |
46474 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
645 |
|
|
646 |
46474 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
647 |
|
|
648 |
46474 |
if (bo->htc == NULL) { |
649 |
50 |
(void)VFP_Error(bo->vfc, "No backend connection (rollback?)"); |
650 |
50 |
vbf_cleanup(bo); |
651 |
50 |
return (F_STP_ERROR); |
652 |
|
} |
653 |
|
|
654 |
|
/* No body -> done */ |
655 |
46424 |
if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { |
656 |
17600 |
http_Unset(bo->beresp, H_Content_Encoding); |
657 |
17600 |
bo->do_gzip = bo->do_gunzip = 0; |
658 |
17600 |
bo->do_stream = 0; |
659 |
17600 |
bo->vfp_filter_list = ""; |
660 |
46424 |
} else if (bo->vfp_filter_list == NULL) { |
661 |
28675 |
bo->vfp_filter_list = VBF_Get_Filter_List(bo); |
662 |
28675 |
} |
663 |
|
|
664 |
46424 |
if (bo->vfp_filter_list == NULL || |
665 |
46424 |
VCL_StackVFP(bo->vfc, bo->vcl, bo->vfp_filter_list)) { |
666 |
600 |
(bo)->htc->doclose = SC_OVERLOAD; |
667 |
600 |
vbf_cleanup(bo); |
668 |
600 |
return (F_STP_ERROR); |
669 |
|
} |
670 |
|
|
671 |
45824 |
if (oc->flags & OC_F_PRIVATE) |
672 |
15273 |
AN(bo->uncacheable); |
673 |
|
|
674 |
45824 |
oc->boc->fetched_so_far = 0; |
675 |
|
|
676 |
45824 |
INIT_OBJ(ctx, VRT_CTX_MAGIC); |
677 |
45824 |
VCL_Bo2Ctx(ctx, bo); |
678 |
|
|
679 |
45824 |
if (VFP_Open(ctx, bo->vfc)) { |
680 |
1325 |
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); |
681 |
1325 |
bo->htc->doclose = SC_RX_BODY; |
682 |
1325 |
vbf_cleanup(bo); |
683 |
1325 |
return (F_STP_ERROR); |
684 |
|
} |
685 |
|
|
686 |
44499 |
if (vbf_beresp2obj(bo)) { |
687 |
75 |
bo->htc->doclose = SC_RX_BODY; |
688 |
75 |
vbf_cleanup(bo); |
689 |
75 |
return (F_STP_ERROR); |
690 |
|
} |
691 |
|
|
692 |
|
#define OBJ_FLAG(U, l, v) \ |
693 |
|
if (bo->vfc->obj_flags & OF_##U) \ |
694 |
|
ObjSetFlag(bo->wrk, oc, OF_##U, 1); |
695 |
|
#include "tbl/obj_attr.h" |
696 |
|
|
697 |
71652 |
if (!(oc->flags & OC_F_HFM) && |
698 |
28199 |
http_IsStatus(bo->beresp, 200) && ( |
699 |
27799 |
RFC2616_Strong_LM(bo->beresp, NULL, NULL) != NULL || |
700 |
27228 |
http_GetHdr(bo->beresp, H_ETag, NULL))) |
701 |
1246 |
ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 1); |
702 |
|
|
703 |
44424 |
assert(oc->boc->refcount >= 1); |
704 |
|
|
705 |
44424 |
assert(oc->boc->state == BOS_REQ_DONE); |
706 |
|
|
707 |
44424 |
if (bo->do_stream) { |
708 |
19924 |
ObjSetState(wrk, oc, BOS_PREP_STREAM); |
709 |
19924 |
HSH_Unbusy(wrk, oc); |
710 |
19924 |
ObjSetState(wrk, oc, BOS_STREAM); |
711 |
19924 |
} |
712 |
|
|
713 |
88848 |
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", |
714 |
44424 |
bo->htc->body_status->nbr, bo->htc->body_status->name, |
715 |
44424 |
bo->do_stream ? "stream" : "-"); |
716 |
|
|
717 |
44424 |
if (bo->htc->body_status != BS_NONE) { |
718 |
26874 |
assert(bo->htc->body_status != BS_ERROR); |
719 |
26874 |
return (F_STP_FETCHBODY); |
720 |
|
} |
721 |
17550 |
AZ(bo->vfc->failed); |
722 |
17550 |
return (F_STP_FETCHEND); |
723 |
46474 |
} |
724 |
|
|
725 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
726 |
44073 |
vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) |
727 |
|
{ |
728 |
|
|
729 |
|
struct objcore *oc; |
730 |
|
|
731 |
44073 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
732 |
44073 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
733 |
44073 |
oc = bo->fetch_objcore; |
734 |
44073 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
735 |
|
|
736 |
44073 |
AZ(bo->vfc->failed); |
737 |
|
|
738 |
|
/* Recycle the backend connection before setting BOS_FINISHED to |
739 |
|
give predictable backend reuse behavior for varnishtest */ |
740 |
44073 |
vbf_cleanup(bo); |
741 |
|
|
742 |
44073 |
AZ(ObjSetU64(wrk, oc, OA_LEN, oc->boc->fetched_so_far)); |
743 |
|
|
744 |
44073 |
if (bo->do_stream) |
745 |
20073 |
assert(oc->boc->state == BOS_STREAM); |
746 |
|
else { |
747 |
24000 |
assert(oc->boc->state == BOS_REQ_DONE); |
748 |
24000 |
ObjSetState(wrk, oc, BOS_PREP_STREAM); |
749 |
24000 |
HSH_Unbusy(wrk, oc); |
750 |
|
} |
751 |
|
|
752 |
44073 |
ObjSetState(wrk, oc, BOS_FINISHED); |
753 |
44073 |
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); |
754 |
44073 |
if (bo->stale_oc != NULL) |
755 |
1724 |
HSH_Kill(bo->stale_oc); |
756 |
44073 |
return (F_STP_DONE); |
757 |
|
} |
758 |
|
|
759 |
|
/*-------------------------------------------------------------------- |
760 |
|
*/ |
761 |
|
|
762 |
|
struct vbf_objiter_priv { |
763 |
|
unsigned magic; |
764 |
|
#define VBF_OBITER_PRIV_MAGIC 0x3c272a17 |
765 |
|
struct busyobj *bo; |
766 |
|
// not yet allocated |
767 |
|
ssize_t l; |
768 |
|
// current allocation |
769 |
|
uint8_t *p; |
770 |
|
ssize_t pl; |
771 |
|
}; |
772 |
|
|
773 |
|
static int v_matchproto_(objiterate_f) |
774 |
599 |
vbf_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) |
775 |
|
{ |
776 |
|
struct vbf_objiter_priv *vop; |
777 |
|
ssize_t l; |
778 |
599 |
const uint8_t *ps = ptr; |
779 |
|
|
780 |
599 |
CAST_OBJ_NOTNULL(vop, priv, VBF_OBITER_PRIV_MAGIC); |
781 |
599 |
CHECK_OBJ_NOTNULL(vop->bo, BUSYOBJ_MAGIC); |
782 |
|
|
783 |
599 |
flush &= OBJ_ITER_END; |
784 |
|
|
785 |
1198 |
while (len > 0) { |
786 |
599 |
if (vop->pl == 0) { |
787 |
574 |
vop->p = NULL; |
788 |
574 |
AN(vop->l); |
789 |
574 |
vop->pl = vop->l; |
790 |
1148 |
if (VFP_GetStorage(vop->bo->vfc, &vop->pl, &vop->p) |
791 |
574 |
!= VFP_OK) |
792 |
0 |
return (1); |
793 |
574 |
if (vop->pl < vop->l) |
794 |
25 |
vop->l -= vop->pl; |
795 |
|
else |
796 |
549 |
vop->l = 0; |
797 |
574 |
} |
798 |
599 |
AN(vop->pl); |
799 |
599 |
AN(vop->p); |
800 |
|
|
801 |
599 |
l = vmin(vop->pl, len); |
802 |
599 |
memcpy(vop->p, ps, l); |
803 |
1148 |
VFP_Extend(vop->bo->vfc, l, |
804 |
599 |
flush && l == len ? VFP_END : VFP_OK); |
805 |
599 |
ps += l; |
806 |
599 |
vop->p += l; |
807 |
599 |
len -= l; |
808 |
599 |
vop->pl -= l; |
809 |
|
} |
810 |
599 |
if (flush) |
811 |
549 |
AZ(vop->l); |
812 |
599 |
return (0); |
813 |
599 |
} |
814 |
|
|
815 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
816 |
649 |
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) |
817 |
|
{ |
818 |
|
struct boc *stale_boc; |
819 |
|
enum boc_state_e stale_state; |
820 |
|
struct objcore *oc, *stale_oc; |
821 |
|
struct vbf_objiter_priv vop[1]; |
822 |
|
|
823 |
649 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
824 |
649 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
825 |
649 |
oc = bo->fetch_objcore; |
826 |
649 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
827 |
649 |
stale_oc = bo->stale_oc; |
828 |
649 |
CHECK_OBJ_NOTNULL(stale_oc, OBJCORE_MAGIC); |
829 |
|
|
830 |
649 |
stale_boc = HSH_RefBoc(stale_oc); |
831 |
649 |
CHECK_OBJ_ORNULL(stale_boc, BOC_MAGIC); |
832 |
649 |
if (stale_boc) { |
833 |
|
/* Wait for the stale object to become fully fetched, so |
834 |
|
* that we can catch fetch errors, before we unbusy the |
835 |
|
* new object. This serves two purposes. First it helps |
836 |
|
* with request coalesching, and stops long chains of |
837 |
|
* IMS-updated short-TTL objects all streaming from a |
838 |
|
* single slow body fetch. Second it makes sure that all |
839 |
|
* the object attributes are complete when we copy them |
840 |
|
* (this would be an issue for ie OA_GZIPBITS). */ |
841 |
75 |
VSLb(bo->vsl, SLT_Notice, |
842 |
|
"vsl: Conditional fetch wait for streaming object"); |
843 |
75 |
ObjWaitState(stale_oc, BOS_FINISHED); |
844 |
75 |
stale_state = stale_boc->state; |
845 |
75 |
HSH_DerefBoc(bo->wrk, stale_oc); |
846 |
75 |
stale_boc = NULL; |
847 |
75 |
if (stale_state != BOS_FINISHED) { |
848 |
50 |
assert(stale_state == BOS_FAILED); |
849 |
50 |
AN(stale_oc->flags & OC_F_FAILED); |
850 |
50 |
} |
851 |
75 |
} |
852 |
|
|
853 |
649 |
AZ(stale_boc); |
854 |
649 |
if (stale_oc->flags & OC_F_FAILED) { |
855 |
50 |
(void)VFP_Error(bo->vfc, "Template object failed"); |
856 |
50 |
vbf_cleanup(bo); |
857 |
50 |
wrk->stats->fetch_failed++; |
858 |
50 |
return (F_STP_FAIL); |
859 |
|
} |
860 |
|
|
861 |
599 |
if (vbf_beresp2obj(bo)) { |
862 |
25 |
vbf_cleanup(bo); |
863 |
25 |
wrk->stats->fetch_failed++; |
864 |
25 |
return (F_STP_FAIL); |
865 |
|
} |
866 |
|
|
867 |
574 |
if (ObjHasAttr(bo->wrk, stale_oc, OA_ESIDATA)) |
868 |
25 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_ESIDATA)); |
869 |
|
|
870 |
574 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_FLAGS)); |
871 |
574 |
if (oc->flags & OC_F_HFM) |
872 |
50 |
ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 0); |
873 |
574 |
AZ(ObjCopyAttr(bo->wrk, oc, stale_oc, OA_GZIPBITS)); |
874 |
|
|
875 |
574 |
if (bo->do_stream) { |
876 |
549 |
ObjSetState(wrk, oc, BOS_PREP_STREAM); |
877 |
549 |
HSH_Unbusy(wrk, oc); |
878 |
549 |
ObjSetState(wrk, oc, BOS_STREAM); |
879 |
549 |
} |
880 |
|
|
881 |
574 |
INIT_OBJ(vop, VBF_OBITER_PRIV_MAGIC); |
882 |
574 |
vop->bo = bo; |
883 |
574 |
vop->l = ObjGetLen(bo->wrk, stale_oc); |
884 |
574 |
if (ObjIterate(wrk, stale_oc, vop, vbf_objiterate, 0)) |
885 |
0 |
(void)VFP_Error(bo->vfc, "Template object failed"); |
886 |
|
|
887 |
574 |
if (bo->vfc->failed) { |
888 |
0 |
vbf_cleanup(bo); |
889 |
0 |
wrk->stats->fetch_failed++; |
890 |
0 |
return (F_STP_FAIL); |
891 |
|
} |
892 |
574 |
return (F_STP_FETCHEND); |
893 |
649 |
} |
894 |
|
|
895 |
|
/*-------------------------------------------------------------------- |
896 |
|
* Create synth object |
897 |
|
* |
898 |
|
* replaces a stale object unless |
899 |
|
* - abandoning the bereq or |
900 |
|
* - leaving vcl_backend_error with return (deliver) and beresp.ttl == 0s or |
901 |
|
* - there is a waitinglist on this object because in this case the default ttl |
902 |
|
* would be 1s, so we might be looking at the same case as the previous |
903 |
|
* |
904 |
|
* We do want the stale replacement to avoid an object pileup with short ttl and |
905 |
|
* long grace/keep, yet there could exist cases where a cache object is |
906 |
|
* deliberately created to momentarily override a stale object. |
907 |
|
* |
908 |
|
* If this case exists, we should add a vcl veto (e.g. beresp.replace_stale with |
909 |
|
* default true) |
910 |
|
*/ |
911 |
|
|
912 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
913 |
8801 |
vbf_stp_error(struct worker *wrk, struct busyobj *bo) |
914 |
|
{ |
915 |
|
ssize_t l, ll, o; |
916 |
|
vtim_real now; |
917 |
|
uint8_t *ptr; |
918 |
|
struct vsb *synth_body; |
919 |
|
struct objcore *stale, *oc; |
920 |
|
|
921 |
8801 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
922 |
8801 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
923 |
8801 |
oc = bo->fetch_objcore; |
924 |
8801 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
925 |
8801 |
AN(oc->flags & OC_F_BUSY); |
926 |
8801 |
assert(bo->director_state == DIR_S_NULL); |
927 |
|
|
928 |
8801 |
if (wrk->vpi->handling != VCL_RET_ERROR) |
929 |
8500 |
wrk->stats->fetch_failed++; |
930 |
|
|
931 |
8801 |
now = W_TIM_real(wrk); |
932 |
8801 |
VSLb_ts_busyobj(bo, "Error", now); |
933 |
|
|
934 |
8801 |
if (oc->stobj->stevedore != NULL) { |
935 |
525 |
oc->boc->fetched_so_far = 0; |
936 |
525 |
ObjFreeObj(bo->wrk, oc); |
937 |
525 |
} |
938 |
|
|
939 |
8801 |
if (bo->storage == NULL) |
940 |
600 |
bo->storage = STV_next(); |
941 |
|
|
942 |
|
// XXX: reset all beresp flags ? |
943 |
|
|
944 |
8801 |
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); |
945 |
8801 |
if (bo->err_code > 0) |
946 |
502 |
http_PutResponse(bo->beresp, "HTTP/1.1", bo->err_code, |
947 |
251 |
bo->err_reason); |
948 |
|
else |
949 |
8550 |
http_PutResponse(bo->beresp, "HTTP/1.1", 503, |
950 |
|
"Backend fetch failed"); |
951 |
|
|
952 |
8801 |
http_TimeHeader(bo->beresp, "Date: ", now); |
953 |
8801 |
http_SetHeader(bo->beresp, "Server: Varnish"); |
954 |
|
|
955 |
8801 |
stale = bo->stale_oc; |
956 |
8801 |
oc->t_origin = now; |
957 |
8801 |
if (!VTAILQ_EMPTY(&oc->objhead->waitinglist)) { |
958 |
|
/* |
959 |
|
* If there is a waitinglist, it means that there is no |
960 |
|
* grace-able object, so cache the error return for a |
961 |
|
* short time, so the waiting list can drain, rather than |
962 |
|
* each objcore on the waiting list sequentially attempt |
963 |
|
* to fetch from the backend. |
964 |
|
*/ |
965 |
69 |
oc->ttl = 1; |
966 |
69 |
oc->grace = 5; |
967 |
69 |
oc->keep = 5; |
968 |
69 |
stale = NULL; |
969 |
69 |
} else { |
970 |
8732 |
oc->ttl = 0; |
971 |
8732 |
oc->grace = 0; |
972 |
8732 |
oc->keep = 0; |
973 |
|
} |
974 |
|
|
975 |
8801 |
synth_body = VSB_new_auto(); |
976 |
8801 |
AN(synth_body); |
977 |
|
|
978 |
8801 |
VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body); |
979 |
|
|
980 |
8801 |
AZ(VSB_finish(synth_body)); |
981 |
|
|
982 |
8801 |
if (wrk->vpi->handling == VCL_RET_ABANDON || wrk->vpi->handling == VCL_RET_FAIL) { |
983 |
700 |
VSB_destroy(&synth_body); |
984 |
700 |
return (F_STP_FAIL); |
985 |
|
} |
986 |
|
|
987 |
8101 |
if (wrk->vpi->handling == VCL_RET_RETRY) { |
988 |
250 |
VSB_destroy(&synth_body); |
989 |
250 |
if (bo->retries++ < cache_param->max_retries) |
990 |
225 |
return (F_STP_RETRY); |
991 |
25 |
VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing"); |
992 |
25 |
return (F_STP_FAIL); |
993 |
|
} |
994 |
|
|
995 |
7851 |
assert(wrk->vpi->handling == VCL_RET_DELIVER); |
996 |
|
|
997 |
7851 |
assert(bo->vfc->wrk == bo->wrk); |
998 |
7851 |
assert(bo->vfc->oc == oc); |
999 |
7851 |
assert(bo->vfc->resp == bo->beresp); |
1000 |
7851 |
assert(bo->vfc->req == bo->bereq); |
1001 |
|
|
1002 |
7851 |
if (vbf_beresp2obj(bo)) { |
1003 |
50 |
VSB_destroy(&synth_body); |
1004 |
50 |
return (F_STP_FAIL); |
1005 |
|
} |
1006 |
|
|
1007 |
7801 |
oc->boc->transit_buffer = 0; |
1008 |
|
|
1009 |
7801 |
ll = VSB_len(synth_body); |
1010 |
7801 |
o = 0; |
1011 |
14402 |
while (ll > 0) { |
1012 |
6601 |
l = ll; |
1013 |
6601 |
if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) |
1014 |
0 |
break; |
1015 |
6601 |
l = vmin(l, ll); |
1016 |
6601 |
memcpy(ptr, VSB_data(synth_body) + o, l); |
1017 |
6601 |
VFP_Extend(bo->vfc, l, l == ll ? VFP_END : VFP_OK); |
1018 |
6601 |
ll -= l; |
1019 |
6601 |
o += l; |
1020 |
|
} |
1021 |
7801 |
AZ(ObjSetU64(wrk, oc, OA_LEN, o)); |
1022 |
7801 |
VSB_destroy(&synth_body); |
1023 |
7801 |
ObjSetState(wrk, oc, BOS_PREP_STREAM); |
1024 |
7801 |
HSH_Unbusy(wrk, oc); |
1025 |
7801 |
if (stale != NULL && oc->ttl > 0) |
1026 |
525 |
HSH_Kill(stale); |
1027 |
7801 |
ObjSetState(wrk, oc, BOS_FINISHED); |
1028 |
7801 |
return (F_STP_DONE); |
1029 |
8801 |
} |
1030 |
|
|
1031 |
|
/*-------------------------------------------------------------------- |
1032 |
|
*/ |
1033 |
|
|
1034 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
1035 |
1650 |
vbf_stp_fail(struct worker *wrk, struct busyobj *bo) |
1036 |
|
{ |
1037 |
|
struct objcore *oc; |
1038 |
|
|
1039 |
1650 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1040 |
1650 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1041 |
1650 |
oc = bo->fetch_objcore; |
1042 |
1650 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1043 |
|
|
1044 |
1650 |
assert(oc->boc->state < BOS_FINISHED); |
1045 |
1650 |
HSH_Fail(oc); |
1046 |
1650 |
if (!(oc->flags & OC_F_BUSY)) |
1047 |
375 |
HSH_Kill(oc); |
1048 |
1650 |
ObjSetState(wrk, oc, BOS_FAILED); |
1049 |
1650 |
return (F_STP_DONE); |
1050 |
|
} |
1051 |
|
|
1052 |
|
/*-------------------------------------------------------------------- |
1053 |
|
*/ |
1054 |
|
|
1055 |
|
static const struct fetch_step * v_matchproto_(vbf_state_f) |
1056 |
0 |
vbf_stp_done(struct worker *wrk, struct busyobj *bo) |
1057 |
|
{ |
1058 |
|
|
1059 |
0 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1060 |
0 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1061 |
0 |
WRONG("Just plain wrong"); |
1062 |
0 |
NEEDLESS(return (F_STP_DONE)); |
1063 |
|
} |
1064 |
|
|
1065 |
|
static void v_matchproto_(task_func_t) |
1066 |
53549 |
vbf_fetch_thread(struct worker *wrk, void *priv) |
1067 |
|
{ |
1068 |
|
struct vrt_ctx ctx[1]; |
1069 |
|
struct busyobj *bo; |
1070 |
|
struct objcore *oc; |
1071 |
|
const struct fetch_step *stp; |
1072 |
|
|
1073 |
53549 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1074 |
53549 |
CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); |
1075 |
53549 |
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); |
1076 |
53549 |
oc = bo->fetch_objcore; |
1077 |
53549 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1078 |
|
|
1079 |
53549 |
THR_SetBusyobj(bo); |
1080 |
53549 |
stp = F_STP_MKBEREQ; |
1081 |
53549 |
assert(isnan(bo->t_first)); |
1082 |
53549 |
assert(isnan(bo->t_prev)); |
1083 |
53549 |
VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk)); |
1084 |
|
|
1085 |
53549 |
bo->wrk = wrk; |
1086 |
53549 |
wrk->vsl = bo->vsl; |
1087 |
|
|
1088 |
|
#if 0 |
1089 |
|
if (bo->stale_oc != NULL) { |
1090 |
|
CHECK_OBJ_NOTNULL(bo->stale_oc, OBJCORE_MAGIC); |
1091 |
|
/* We don't want the oc/stevedore ops in fetching thread */ |
1092 |
|
if (!ObjCheckFlag(wrk, bo->stale_oc, OF_IMSCAND)) |
1093 |
|
(void)HSH_DerefObjCore(wrk, &bo->stale_oc, 0); |
1094 |
|
} |
1095 |
|
#endif |
1096 |
|
|
1097 |
53549 |
VCL_TaskEnter(bo->privs); |
1098 |
290884 |
while (stp != F_STP_DONE) { |
1099 |
237335 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1100 |
237335 |
assert(oc->boc->refcount >= 1); |
1101 |
237335 |
if (oc->boc->state < BOS_REQ_DONE) |
1102 |
55268 |
AN(bo->req); |
1103 |
|
else |
1104 |
182067 |
AZ(bo->req); |
1105 |
237335 |
AN(stp); |
1106 |
237335 |
AN(stp->name); |
1107 |
237335 |
AN(stp->func); |
1108 |
237335 |
stp = stp->func(wrk, bo); |
1109 |
|
} |
1110 |
|
|
1111 |
53549 |
assert(bo->director_state == DIR_S_NULL); |
1112 |
|
|
1113 |
53549 |
INIT_OBJ(ctx, VRT_CTX_MAGIC); |
1114 |
53549 |
VCL_Bo2Ctx(ctx, bo); |
1115 |
53549 |
VCL_TaskLeave(ctx, bo->privs); |
1116 |
53549 |
http_Teardown(bo->bereq); |
1117 |
53549 |
http_Teardown(bo->beresp); |
1118 |
|
// can not make assumptions about the number of references here #3434 |
1119 |
53549 |
if (bo->bereq_body != NULL) |
1120 |
300 |
(void) HSH_DerefObjCore(bo->wrk, &bo->bereq_body, 0); |
1121 |
|
|
1122 |
53549 |
if (oc->boc->state == BOS_FINISHED) { |
1123 |
51874 |
AZ(oc->flags & OC_F_FAILED); |
1124 |
103748 |
VSLb(bo->vsl, SLT_Length, "%ju", |
1125 |
51874 |
(uintmax_t)ObjGetLen(bo->wrk, oc)); |
1126 |
51874 |
} |
1127 |
|
// AZ(oc->boc); // XXX |
1128 |
|
|
1129 |
53549 |
if (bo->stale_oc != NULL) |
1130 |
3500 |
(void)HSH_DerefObjCore(wrk, &bo->stale_oc, 0); |
1131 |
|
|
1132 |
53549 |
wrk->vsl = NULL; |
1133 |
53549 |
HSH_DerefBoc(wrk, oc); |
1134 |
53549 |
SES_Rel(bo->sp); |
1135 |
53549 |
VBO_ReleaseBusyObj(wrk, &bo); |
1136 |
53549 |
THR_SetBusyobj(NULL); |
1137 |
53549 |
} |
1138 |
|
|
1139 |
|
/*-------------------------------------------------------------------- |
1140 |
|
*/ |
1141 |
|
|
1142 |
|
void |
1143 |
53575 |
VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, |
1144 |
|
struct objcore *oldoc, enum vbf_fetch_mode_e mode) |
1145 |
|
{ |
1146 |
|
struct boc *boc; |
1147 |
|
struct busyobj *bo; |
1148 |
|
enum task_prio prio; |
1149 |
|
const char *how; |
1150 |
|
|
1151 |
53575 |
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); |
1152 |
53575 |
CHECK_OBJ_NOTNULL(req, REQ_MAGIC); |
1153 |
53575 |
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); |
1154 |
53575 |
AN(oc->flags & OC_F_BUSY); |
1155 |
53575 |
CHECK_OBJ_ORNULL(oldoc, OBJCORE_MAGIC); |
1156 |
|
|
1157 |
53575 |
bo = VBO_GetBusyObj(wrk, req); |
1158 |
53575 |
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); |
1159 |
53575 |
AN(bo->vcl); |
1160 |
|
|
1161 |
53575 |
boc = HSH_RefBoc(oc); |
1162 |
53575 |
CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); |
1163 |
|
|
1164 |
53575 |
switch (mode) { |
1165 |
|
case VBF_PASS: |
1166 |
18300 |
prio = TASK_QUEUE_BO; |
1167 |
18300 |
how = "pass"; |
1168 |
18300 |
bo->uncacheable = 1; |
1169 |
18300 |
break; |
1170 |
|
case VBF_NORMAL: |
1171 |
33175 |
prio = TASK_QUEUE_BO; |
1172 |
33175 |
how = "fetch"; |
1173 |
33175 |
break; |
1174 |
|
case VBF_BACKGROUND: |
1175 |
2100 |
prio = TASK_QUEUE_BG; |
1176 |
2100 |
how = "bgfetch"; |
1177 |
2100 |
bo->is_bgfetch = 1; |
1178 |
2100 |
break; |
1179 |
|
default: |
1180 |
0 |
WRONG("Wrong fetch mode"); |
1181 |
0 |
} |
1182 |
|
|
1183 |
|
#define REQ_BEREQ_FLAG(l, r, w, d) bo->l = req->l; |
1184 |
|
#include "tbl/req_bereq_flags.h" |
1185 |
|
|
1186 |
53575 |
VSLb(bo->vsl, SLT_Begin, "bereq %ju %s", VXID(req->vsl->wid), how); |
1187 |
53575 |
VSLbs(bo->vsl, SLT_VCL_use, TOSTRAND(VCL_Name(bo->vcl))); |
1188 |
53575 |
VSLb(req->vsl, SLT_Link, "bereq %ju %s", VXID(bo->vsl->wid), how); |
1189 |
|
|
1190 |
53575 |
THR_SetBusyobj(bo); |
1191 |
|
|
1192 |
53575 |
bo->sp = req->sp; |
1193 |
53575 |
SES_Ref(bo->sp); |
1194 |
|
|
1195 |
53575 |
oc->boc->vary = req->vary_b; |
1196 |
53575 |
req->vary_b = NULL; |
1197 |
|
|
1198 |
53575 |
HSH_Ref(oc); |
1199 |
53575 |
AZ(bo->fetch_objcore); |
1200 |
53575 |
bo->fetch_objcore = oc; |
1201 |
|
|
1202 |
53575 |
AZ(bo->stale_oc); |
1203 |
53575 |
if (oldoc != NULL) { |
1204 |
3525 |
assert(oldoc->refcnt > 0); |
1205 |
3525 |
HSH_Ref(oldoc); |
1206 |
3525 |
bo->stale_oc = oldoc; |
1207 |
3525 |
} |
1208 |
|
|
1209 |
53575 |
AZ(bo->req); |
1210 |
53575 |
bo->req = req; |
1211 |
|
|
1212 |
53575 |
bo->fetch_task->priv = bo; |
1213 |
53575 |
bo->fetch_task->func = vbf_fetch_thread; |
1214 |
|
|
1215 |
53575 |
if (Pool_Task(wrk->pool, bo->fetch_task, prio)) { |
1216 |
57 |
wrk->stats->bgfetch_no_thread++; |
1217 |
57 |
(void)vbf_stp_fail(req->wrk, bo); |
1218 |
57 |
if (bo->stale_oc != NULL) |
1219 |
25 |
(void)HSH_DerefObjCore(wrk, &bo->stale_oc, 0); |
1220 |
57 |
HSH_DerefBoc(wrk, oc); |
1221 |
57 |
SES_Rel(bo->sp); |
1222 |
57 |
THR_SetBusyobj(NULL); |
1223 |
57 |
VBO_ReleaseBusyObj(wrk, &bo); |
1224 |
57 |
} else { |
1225 |
53518 |
THR_SetBusyobj(NULL); |
1226 |
53518 |
bo = NULL; /* ref transferred to fetch thread */ |
1227 |
53518 |
if (mode == VBF_BACKGROUND) { |
1228 |
2075 |
ObjWaitState(oc, BOS_REQ_DONE); |
1229 |
2075 |
(void)VRB_Ignore(req); |
1230 |
2075 |
} else { |
1231 |
51443 |
ObjWaitState(oc, BOS_STREAM); |
1232 |
51443 |
if (oc->boc->state == BOS_FAILED) { |
1233 |
675 |
AN((oc->flags & OC_F_FAILED)); |
1234 |
675 |
} else { |
1235 |
50768 |
AZ(oc->flags & OC_F_BUSY); |
1236 |
|
} |
1237 |
|
} |
1238 |
|
} |
1239 |
53575 |
AZ(bo); |
1240 |
53575 |
VSLb_ts_req(req, "Fetch", W_TIM_real(wrk)); |
1241 |
53575 |
assert(oc->boc == boc); |
1242 |
53575 |
HSH_DerefBoc(wrk, oc); |
1243 |
53575 |
if (mode == VBF_BACKGROUND) |
1244 |
2100 |
(void)HSH_DerefObjCore(wrk, &oc, HSH_RUSH_POLICY); |
1245 |
53575 |
} |