static enum fetch_step
vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
{
AZ(bo->vfc->failed);
VFP_Close(bo->vfc);
AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
bo->fetch_objcore->boc->len_so_far));
if (bo->do_stream)
assert(bo->fetch_objcore->boc->state == BOS_STREAM);
else {
assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
HSH_Unbusy(wrk, bo->fetch_objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->stale_oc != NULL)
HSH_Kill(bo->stale_oc);
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(vbf_beresp2obj(bo));
if (ObjHasAttr(bo->wrk, bo->stale_oc, OA_ESIDATA))
AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc,
OA_ESIDATA));
AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));
if (bo->do_stream) {
ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
HSH_Unbusy(wrk, bo->fetch_objcore);
ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
}
if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0))
(void)VFP_Error(bo->vfc, "Template object failed");
if (bo->stale_oc->flags & OC_F_FAILED)
(void)VFP_Error(bo->vfc, "Template object failed");
if (bo->vfc->failed) {
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
bo->fetch_objcore->boc->len_so_far));
if (!bo->do_stream)
HSH_Unbusy(wrk, bo->fetch_objcore);
HSH_Kill(bo->stale_oc);
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
ssize_t l, ll, o;
double now;
uint8_t *ptr;
struct vsb *synth_body;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
AN(bo->fetch_objcore->flags & OC_F_BUSY);
assert(bo->director_state == DIR_S_NULL);
wrk->stats->fetch_failed++;
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Error", now);
if (bo->fetch_objcore->stobj->stevedore != NULL)
ObjFreeObj(bo->wrk, bo->fetch_objcore);
if (bo->storage == NULL)
bo->storage = STV_next();
// XXX: reset all beresp flags ?
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
http_TimeHeader(bo->beresp, "Date: ", now);
http_SetHeader(bo->beresp, "Server: Varnish");
bo->fetch_objcore->t_origin = now;
if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) {
/*
* If there is a waitinglist, it means that there is no
* grace-able object, so cache the error return for a
* short time, so the waiting list can drain, rather than
* each objcore on the waiting list sequentially attempt
* to fetch from the backend.
*/
bo->fetch_objcore->ttl = 1;
bo->fetch_objcore->grace = 5;
bo->fetch_objcore->keep = 5;
} else {
bo->fetch_objcore->ttl = 0;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
}
synth_body = VSB_new_auto();
AN(synth_body);
VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);
AZ(VSB_finish(synth_body));
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
VSB_destroy(&synth_body);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
VSB_destroy(&synth_body);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing");
return (F_STP_FAIL);
}
assert(wrk->handling == VCL_RET_DELIVER);
assert(bo->vfc->wrk == bo->wrk);
assert(bo->vfc->oc == bo->fetch_objcore);
assert(bo->vfc->resp == bo->beresp);
assert(bo->vfc->req == bo->bereq);
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
VSB_destroy(&synth_body);
return (F_STP_FAIL);
}
ll = VSB_len(synth_body);
o = 0;
while (ll > 0) {
l = ll;
if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
break;
if (l > ll)
l = ll;
memcpy(ptr, VSB_data(synth_body) + o, l);
VFP_Extend(bo->vfc, l);
ll -= l;
o += l;
}
AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o));
VSB_destroy(&synth_body);
HSH_Unbusy(wrk, bo->fetch_objcore);
ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
return (F_STP_DONE);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
struct http *h;
double now;
struct vsb *synth_body;
ssize_t sz, szl;
uint8_t *ptr;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats->s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
http_TimeHeader(h, "Date: ", now);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
synth_body = VSB_new_auto();
AN(synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, synth_body);
AZ(VSB_finish(synth_body));
http_Unset(h, H_Content_Length);
http_PrintfHeader(req->resp, "Content-Length: %zd",
VSB_len(synth_body));
/* Discard any lingering request body before delivery */
(void)VRB_Ignore(req);
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_destroy(&synth_body);
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
req->objcore = HSH_Private(wrk);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
szl = -1;
if (STV_NewObject(wrk, req->objcore, TRANSIENT_STORAGE, 1024)) {
szl = VSB_len(synth_body);
assert(szl >= 0);
sz = szl;
if (sz > 0 &&
ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) {
memcpy(ptr, VSB_data(synth_body), szl);
ObjExtend(wrk, req->objcore, szl);
} else if (sz > 0) {
szl = -1;
}
}
if (szl >= 0)
AZ(ObjSetU64(wrk, req->objcore, OA_LEN, szl));
HSH_DerefBoc(wrk, req->objcore);
VSB_destroy(&synth_body);
if (szl < 0) {
VSLb(req->vsl, SLT_Error, "Could not get storage");
req->doclose = SC_OVERLOAD;
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
req->req_step = R_STP_TRANSMIT;
return (REQ_FSM_MORE);
}
static ssize_t
vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
{
ssize_t l, r = 0, yet;
struct vfp_ctx *vfc;
uint8_t *ptr;
enum vfp_status vfps = VFP_ERROR;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC);
CHECK_OBJ_NOTNULL(req->htc->vfc, VFP_CTX_MAGIC);
vfc = req->htc->vfc;
req->body_oc = HSH_Private(req->wrk);
AN(req->body_oc);
XXXAN(STV_NewObject(req->wrk, req->body_oc, TRANSIENT_STORAGE, 8));
vfc->oc = req->body_oc;
if (VFP_Open(vfc) < 0) {
req->req_body_status = REQ_BODY_FAIL;
HSH_DerefBoc(req->wrk, req->body_oc);
AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
return (-1);
}
AZ(req->req_bodybytes);
AN(req->htc);
yet = req->htc->content_length;
if (yet < 0)
yet = 0;
do {
AZ(vfc->failed);
if (maxsize >= 0 && req->req_bodybytes > maxsize) {
(void)VFP_Error(vfc, "Request body too big to cache");
break;
}
l = yet;
if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK)
break;
AZ(vfc->failed);
AN(ptr);
AN(l);
vfps = VFP_Suck(vfc, ptr, &l);
if (l > 0 && vfps != VFP_ERROR) {
req->req_bodybytes += l;
req->acct.req_bodybytes += l;
if (yet >= l)
yet -= l;
if (func != NULL) {
r = func(priv, 1, ptr, l);
if (r)
break;
} else {
ObjExtend(req->wrk, req->body_oc, l);
}
}
} while (vfps == VFP_OK);
VFP_Close(vfc);
VSLb_ts_req(req, "ReqBody", VTIM_real());
if (func != NULL) {
HSH_DerefBoc(req->wrk, req->body_oc);
AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
if (vfps != VFP_END) {
req->req_body_status = REQ_BODY_FAIL;
if (r == 0)
r = -1;
}
return (r);
}
ObjTrimStore(req->wrk, req->body_oc);
AZ(ObjSetU64(req->wrk, req->body_oc, OA_LEN, req->req_bodybytes));
HSH_DerefBoc(req->wrk, req->body_oc);
if (vfps != VFP_END) {
req->req_body_status = REQ_BODY_FAIL;
AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
return (-1);
}
assert(req->req_bodybytes >= 0);
if (req->req_bodybytes != req->htc->content_length) {
/* We must update also the "pristine" req.* copy */
http_Unset(req->http0, H_Content_Length);
http_Unset(req->http0, H_Transfer_Encoding);
http_PrintfHeader(req->http0, "Content-Length: %ju",
(uintmax_t)req->req_bodybytes);
http_Unset(req->http, H_Content_Length);
http_Unset(req->http, H_Transfer_Encoding);
http_PrintfHeader(req->http, "Content-Length: %ju",
(uintmax_t)req->req_bodybytes);
}
req->req_body_status = REQ_BODY_CACHED;
return (req->req_bodybytes);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
const char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
if (bo->htc->content_length == 0)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->htc->body_status != BS_NONE) {
bo->is_gzip =
http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip =
!http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
}
/* We won't gunzip unless it is non-empty and gzip'ed */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gunzip && !bo->is_gzip))
bo->do_gunzip = 0;
/* We wont gzip unless it is non-empty and ungzip'ed */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gzip && !bo->is_gunzip))
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
vbf_vfp_push(bo, &vfp_gunzip, 1);
if (bo->htc->content_length != 0) {
if (bo->do_esi && bo->do_gzip) {
vbf_vfp_push(bo, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
vbf_vfp_push(bo, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
vbf_vfp_push(bo, &vfp_esi, 1);
} else if (bo->do_gzip) {
vbf_vfp_push(bo, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
vbf_vfp_push(bo, &vfp_testgunzip, 1);
}
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc->body_status == BS_NONE)
bo->do_stream = 0;
bo->fetch_objcore->boc->len_so_far = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (bo->do_esi)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);
if (bo->do_gzip || bo->do_gunzip)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);
if (!(bo->fetch_objcore->flags & OC_F_PASS) &&
http_IsStatus(bo->beresp, 200) && (
http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
http_GetHdr(bo->beresp, H_ETag, &p)))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);
if (bo->htc->body_status != BS_NONE &&
VDI_GetBody(bo->wrk, bo) != 0) {
(void)VFP_Error(bo->vfc,
"GetBody failed - workspace_backend overflow?");
bo->htc->doclose = SC_OVERLOAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->refcount >= 1);
assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
if (bo->do_stream) {
ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
HSH_Unbusy(wrk, bo->fetch_objcore);
ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc->body_status, body_status_2str(bo->htc->body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc->body_status != BS_NONE) {
assert(bo->htc->body_status != BS_ERROR);
vbf_fetch_body_helper(bo);
}
VFP_Close(bo->vfc);
if (bo->vfc->failed) {
VDI_Finish(bo->wrk, bo);
if (!bo->do_stream) {
assert(bo->fetch_objcore->boc->state < BOS_STREAM);
// XXX: doclose = ?
return (F_STP_ERROR);
} else {
return (F_STP_FAIL);
}
}
AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
bo->fetch_objcore->boc->len_so_far));
if (bo->do_stream)
assert(bo->fetch_objcore->boc->state == BOS_STREAM);
else {
assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
HSH_Unbusy(wrk, bo->fetch_objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->stale_oc != NULL)
HSH_Kill(bo->stale_oc);
return (F_STP_DONE);
}
