static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi) {
bo->vfp = &vfp_esi;
/*
* The one case were we do not weaken Etag is where
* incoming obj is not gzip'ed and we don't gzip either
* If we ESI expand it on deliver, we weaken there.
*/
if (bo->is_gzip || bo->do_gzip | bo->do_gunzip)
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_gunzip) {
bo->vfp = &vfp_gunzip;
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_gzip) {
bo->vfp = &vfp_gzip;
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip) {
bo->vfp = &vfp_testgzip;
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
V1F_fetch_body(bo);
break;
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
bo->vfp = NULL;
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}