static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
ssize_t l, ll, o;
double now;
uint8_t *ptr;
struct vsb *synth_body;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
AN(bo->fetch_objcore->flags & OC_F_BUSY);
assert(bo->director_state == DIR_S_NULL);
wrk->stats->fetch_failed++;
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Error", now);
if (bo->fetch_objcore->stobj->stevedore != NULL)
ObjFreeObj(bo->wrk, bo->fetch_objcore);
if (bo->storage == NULL)
bo->storage = STV_next();
// XXX: reset all beresp flags ?
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
http_TimeHeader(bo->beresp, "Date: ", now);
http_SetHeader(bo->beresp, "Server: Varnish");
bo->fetch_objcore->t_origin = now;
if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) {
/*
* If there is a waitinglist, it means that there is no
* grace-able object, so cache the error return for a
* short time, so the waiting list can drain, rather than
* each objcore on the waiting list sequentially attempt
* to fetch from the backend.
*/
bo->fetch_objcore->ttl = 1;
bo->fetch_objcore->grace = 5;
bo->fetch_objcore->keep = 5;
} else {
bo->fetch_objcore->ttl = 0;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
}
synth_body = VSB_new_auto();
AN(synth_body);
VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);
AZ(VSB_finish(synth_body));
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
VSB_destroy(&synth_body);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
VSB_destroy(&synth_body);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing");
return (F_STP_FAIL);
}
assert(wrk->handling == VCL_RET_DELIVER);
assert(bo->vfc->wrk == bo->wrk);
assert(bo->vfc->oc == bo->fetch_objcore);
assert(bo->vfc->resp == bo->beresp);
assert(bo->vfc->req == bo->bereq);
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
VSB_destroy(&synth_body);
return (F_STP_FAIL);
}
ll = VSB_len(synth_body);
o = 0;
while (ll > 0) {
l = ll;
if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
break;
if (l > ll)
l = ll;
memcpy(ptr, VSB_data(synth_body) + o, l);
VFP_Extend(bo->vfc, l);
ll -= l;
o += l;
}
AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o));
VSB_destroy(&synth_body);
HSH_Unbusy(wrk, bo->fetch_objcore);
ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage);
bo->storage = bo->do_pass ? stv_transient : STV_next();
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
AZ(bo->htc);
VFP_Setup(bo->vfc, wrk);
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->resp = bo->beresp;
bo->vfc->req = bo->bereq;
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
if (bo->fetch_objcore->flags & OC_F_PRIVATE) {
/* private objects have negative TTL */
bo->fetch_objcore->t_origin = now;
bo->fetch_objcore->ttl = -1.;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
} else {
/* What does RFC2616 think about TTL ? */
RFC2616_Ttl(bo, now,
&bo->fetch_objcore->t_origin,
&bo->fetch_objcore->ttl,
&bo->fetch_objcore->grace,
&bo->fetch_objcore->keep
);
}
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
}
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
if (bo->fetch_objcore->boc->state != BOS_REQ_DONE) {
bo->req = NULL;
ObjSetState(wrk, bo->fetch_objcore, BOS_REQ_DONE);
}
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_PASS) {
bo->fetch_objcore->flags |= OC_F_HFP;
bo->uncacheable = 1;
wrk->handling = VCL_RET_DELIVER;
}
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
ssize_t l, ll, o;
double now;
uint8_t *ptr;
struct vsb *synth_body;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(bo->director_state == DIR_S_NULL);
if(bo->fetch_objcore->stobj->stevedore != NULL)
ObjFreeObj(bo->wrk, bo->fetch_objcore);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Error", now);
AN(bo->fetch_objcore->flags & OC_F_BUSY);
synth_body = VSB_new_auto();
AN(synth_body);
// XXX: reset all beresp flags ?
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
http_TimeHeader(bo->beresp, "Date: ", now);
http_SetHeader(bo->beresp, "Server: Varnish");
EXP_Clr(&bo->fetch_objcore->exp);
bo->fetch_objcore->exp.t_origin = bo->t_prev;
VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);
AZ(VSB_finish(synth_body));
if (wrk->handling == VCL_RET_RETRY ||
wrk->handling == VCL_RET_ABANDON) {
VSB_delete(synth_body);
if (bo->director_state != DIR_S_NULL) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
}
return (F_STP_FAIL);
}
assert(wrk->handling == VCL_RET_DELIVER);
bo->vfc->bo = bo;
bo->vfc->wrk = bo->wrk;
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->http = bo->beresp;
bo->vfc->esi_req = bo->bereq;
if (vbf_beresp2obj(bo)) {
VSB_delete(synth_body);
return (F_STP_FAIL);
}
ll = VSB_len(synth_body);
o = 0;
while (ll > 0) {
l = ll;
if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
break;
memcpy(ptr, VSB_data(synth_body) + o, l);
VBO_extend(bo, l);
ll -= l;
o += l;
}
VSB_delete(synth_body);
HSH_Unbusy(wrk, bo->fetch_objcore);
VBO_setstate(bo, BOS_FINISHED);
return (F_STP_DONE);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
struct http *h;
double now;
struct vsb *synth_body;
ssize_t sz, szl;
uint8_t *ptr;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats->s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
http_TimeHeader(h, "Date: ", now);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
synth_body = VSB_new_auto();
AN(synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, synth_body);
AZ(VSB_finish(synth_body));
http_Unset(h, H_Content_Length);
http_PrintfHeader(req->resp, "Content-Length: %zd",
VSB_len(synth_body));
/* Discard any lingering request body before delivery */
(void)VRB_Ignore(req);
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_destroy(&synth_body);
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
req->objcore = HSH_Private(wrk);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
szl = -1;
if (STV_NewObject(wrk, req->objcore, TRANSIENT_STORAGE, 1024)) {
szl = VSB_len(synth_body);
assert(szl >= 0);
sz = szl;
if (sz > 0 &&
ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) {
memcpy(ptr, VSB_data(synth_body), szl);
ObjExtend(wrk, req->objcore, szl);
} else if (sz > 0) {
szl = -1;
}
}
if (szl >= 0)
AZ(ObjSetU64(wrk, req->objcore, OA_LEN, szl));
HSH_DerefBoc(wrk, req->objcore);
VSB_destroy(&synth_body);
if (szl < 0) {
VSLb(req->vsl, SLT_Error, "Could not get storage");
req->doclose = SC_OVERLOAD;
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
req->req_step = R_STP_TRANSMIT;
return (REQ_FSM_MORE);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
AZ(bo->htc);
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
*/
if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
wrk->stats->fetch_head++;
bo->htc->body_status = BS_NONE;
} else if (http_GetStatus(bo->beresp) <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
* ... but we should never see them.
*/
wrk->stats->fetch_1xx++;
bo->htc->body_status = BS_ERROR;
} else if (http_IsStatus(bo->beresp, 204)) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC7230 3.3.1 p28 and 3.3.2 p30]
*/
wrk->stats->fetch_204++;
if (http_GetHdr(bo->beresp, H_Content_Length, NULL) ||
http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
bo->htc->body_status = BS_ERROR;
else
bo->htc->body_status = BS_NONE;
} else if (http_IsStatus(bo->beresp, 304)) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
wrk->stats->fetch_304++;
bo->htc->body_status = BS_NONE;
} else if (bo->htc->body_status == BS_CHUNKED) {
wrk->stats->fetch_chunked++;
} else if (bo->htc->body_status == BS_LENGTH) {
assert(bo->htc->content_length > 0);
wrk->stats->fetch_length++;
} else if (bo->htc->body_status == BS_EOF) {
wrk->stats->fetch_eof++;
} else if (bo->htc->body_status == BS_ERROR) {
wrk->stats->fetch_bad++;
} else if (bo->htc->body_status == BS_NONE) {
wrk->stats->fetch_none++;
} else {
WRONG("wrong bodystatus");
}
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
}
bo->vfc->bo = bo;
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->wrk = bo->wrk;
bo->vfc->http = bo->beresp;
bo->vfc->esi_req = bo->bereq;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
