static enum fetch_step
vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
{
int i, do_ims;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
xxxassert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
if (!bo->do_pass && bo->req != NULL)
vbf_release_req(bo); /* XXX: retry ?? */
assert(bo->state <= BOS_REQ_DONE);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
if (bo->do_pass && bo->req != NULL)
vbf_release_req(bo); /* XXX : retry ?? */
AZ(bo->req);
if (i) {
AZ(bo->vbc);
make_it_503(bo);
} else {
AN(bo->vbc);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
if (i && bo->htc.body_status == BS_LENGTH)
bo->htc.body_status = BS_NONE;
bo->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
RFC2616_Ttl(bo);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
bo->beresp->status = 200;
http_PrintfHeader(bo->beresp, "Content-Length: %jd",
(intmax_t)bo->ims_obj->len);
do_ims = 1;
} else
do_ims = 0;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass)
bo->fetch_objcore->flags |= OC_F_PASS;
if (wrk->handling == VCL_RET_DELIVER)
return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
if (wrk->handling == VCL_RET_RETRY) {
assert(bo->state == BOS_REQ_DONE);
bo->retries++;
if (bo->retries <= cache_param->max_retries) {
VDI_CloseFd(&bo->vbc);
return (F_STP_STARTFETCH);
}
// XXX: wrk->handling = VCL_RET_SYNTH;
}
INCOMPL();
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i, do_ims;
double now;
char time_str[VTIM_FORMAT_SIZE];
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->vbc);
assert(bo->doclose == SC_NULL);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
HTTP_Copy(bo->bereq, bo->bereq0);
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
AZ(bo->vbc);
return (F_STP_ERROR);
}
AN(bo->vbc);
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
VTIM_format(now, time_str);
http_PrintfHeader(bo->beresp, "Date: %s", time_str);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
if (bo->htc.body_status == BS_ERROR) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
http_Merge(bo->ims_obj->http, bo->beresp,
bo->ims_obj->changed_gzip);
assert(bo->beresp->status == 200);
do_ims = 1;
} else
do_ims = 0;
VFP_Setup(bo->vfc);
bo->vfc->bo = bo;
bo->vfc->http = bo->beresp;
bo->vfc->vsl = bo->vsl;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
if (wrk->handling == VCL_RET_RETRY) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
bo->doclose = SC_NULL;
bo->retries++;
if (bo->retries <= cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
}
static void
vbf_fetch_thread(struct worker *wrk, void *priv)
{
struct busyobj *bo;
enum fetch_step stp;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
THR_SetBusyobj(bo);
stp = F_STP_MKBEREQ;
assert(bo->doclose == SC_NULL);
assert(isnan(bo->t_first));
assert(isnan(bo->t_prev));
VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk));
bo->stats = &wrk->stats;
while (stp != F_STP_DONE) {
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(bo->refcount >= 1);
switch(stp) {
#define FETCH_STEP(l, U, arg) \
case F_STP_##U: \
stp = vbf_stp_##l arg; \
break;
#include "tbl/steps.h"
#undef FETCH_STEP
default:
WRONG("Illegal fetch_step");
}
}
assert(WRW_IsReleased(wrk));
bo->stats = NULL;
if (bo->vbc != NULL) {
if (bo->doclose != SC_NULL)
VDI_CloseFd(&bo->vbc, &bo->acct);
else
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
if (bo->state == BOS_FINISHED) {
AZ(bo->fetch_objcore->flags & OC_F_FAILED);
HSH_Complete(bo->fetch_objcore);
VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &bo->fetch_obj->body->list, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= bo->fetch_obj->len);
else
assert(uu == bo->fetch_obj->len);
}
}
AZ(bo->fetch_objcore->busyobj);
if (bo->ims_obj != NULL)
(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
VBO_DerefBusyObj(wrk, &bo);
THR_SetBusyobj(NULL);
}
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(req->handling == VCL_RET_DELIVER);
if (req->objcore->objhead == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
bo->exp.ttl = -1.;
} else if (bo->do_pass) {
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->do_esi || req->esi_level > 0)
bo->do_stream = 0;
if (!req->wantbody)
bo->do_stream = 0;
/* No reason to try streaming a non-existing body */
if (bo->body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (req->objcore->objhead != NULL) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
vary = VRY_Create(req, bo->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (bo->exp.ttl < cache_param->shortlived ||
req->objcore == NULL)
req->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
nhttp);
if (req->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
l, nhttp);
}
bo->stats = NULL;
if (req->obj == NULL) {
req->err_code = 503;
req->req_step = R_STP_ERROR;
VDI_CloseFd(&bo->vbc);
VBO_DerefBusyObj(wrk, &req->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
req->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = req->obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
req->obj->gziped = 1;
if (vary != NULL) {
req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
VSB_data(vary), varyl);
AN(req->obj->vary);
VRY_Validate(req->obj->vary);
VSB_delete(vary);
}
req->obj->vxid = bo->vsl->wid;
req->obj->response = req->err_code;
WS_Assert(req->obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = req->obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
req->obj->last_modified = VTIM_parse(b);
else
req->obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (req->obj->response == 200 &&
req->http->conds &&
RFC2616_Do_Cond(req))
bo->do_stream = 0;
/*
* Ready to fetch the body
*/
bo->fetch_task.func = FetchBody;
bo->fetch_task.priv = bo;
assert(bo->refcount == 2); /* one for each thread */
if (req->obj->objcore->objhead != NULL) {
EXP_Insert(req->obj);
AN(req->obj->objcore->ban);
AZ(req->obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, req->obj->objcore);
}
if (!bo->do_stream ||
Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
FetchBody(wrk, bo);
if (req->obj->objcore->objhead != NULL)
HSH_Ref(req->obj->objcore);
if (bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &req->busyobj);
} else if (bo->state == BOS_FAILED) {
/* handle early failures */
HSH_Deref(&wrk->stats, NULL, &req->obj);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->err_code = 503;
req->req_step = R_STP_ERROR;
return (0);
}
assert(WRW_IsReleased(wrk));
req->req_step = R_STP_PREPRESP;
return (0);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
ssize_t est;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
AN(bo->vbc);
est = V1F_Setup_Fetch(bo);
if (est == 0) {
/*
* If the length is known to be zero, it's not gziped.
* A similar issue exists for chunked encoding but we
* don't handle that. See #1320.
*/
http_Unset(bo->beresp, H_Content_Encoding);
bo->is_gzip = 0;
bo->is_gunzip = 1;
}
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* Fix Content-Encoding, as appropriate */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
else if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc, &bo->acct);
return (F_STP_ERROR);
}
assert(WRW_IsReleased(wrk));
obj = bo->fetch_obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (bo->do_gzip || bo->do_gunzip)
obj->changed_gzip = 1;
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
assert (bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc.body_status, body_status_2str(bo->htc.body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc.body_status != BS_NONE) {
assert(bo->htc.body_status != BS_ERROR);
VFP_Fetch_Body(bo, est);
}
if (bo->failed && !bo->do_stream) {
assert(bo->state < BOS_STREAM);
if (bo->fetch_obj != NULL) {
oc_freeobj(bo->fetch_objcore);
bo->fetch_obj = NULL;
bo->stats->n_object--;
}
return (F_STP_ERROR);
}
if (bo->failed)
return (F_STP_FAIL);
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && !(bo->should_close)) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
{
int i;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
xxxassert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
if (!bo->do_pass)
vbf_release_req(bo); /* XXX: retry ?? */
assert(bo->state == BOS_INVALID);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
if (bo->do_pass)
vbf_release_req(bo); /* XXX : retry ?? */
AZ(bo->req);
if (i) {
AZ(bo->vbc);
bo->err_code = 503;
http_SetH(bo->beresp, HTTP_HDR_PROTO, "HTTP/1.1");
http_SetResp(bo->beresp,
"HTTP/1.1", 503, "Backend fetch failed");
http_SetHeader(bo->beresp, "Content-Length: 0");
http_SetHeader(bo->beresp, "Connection: close");
} else {
AN(bo->vbc);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
bo->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
bo->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(bo);
/* pass from vclrecv{} has negative TTL */
if (bo->fetch_objcore->objhead == NULL)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_DELIVER)
return (F_STP_FETCH);
if (wrk->handling == VCL_RET_RETRY) {
assert(bo->state == BOS_INVALID);
bo->retries++;
if (bo->retries <= cache_param->max_retries) {
VDI_CloseFd(&bo->vbc);
return (F_STP_STARTFETCH);
}
// XXX: wrk->handling = VCL_RET_SYNTH;
}
return (F_STP_NOTYET);
}
void
PipeRequest(struct req *req)
{
struct vbc *vc;
struct worker *wrk;
struct pollfd fds[2];
struct busyobj *bo;
int i;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
wrk = req->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
vc = VDI_GetFd(NULL, bo);
if (vc == NULL)
return;
bo->vbc = vc; /* For panic dumping */
(void)VTCP_blocking(vc->fd);
WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);
req->acct_req.hdrbytes += HTTP1_Write(wrk, bo->bereq, 0);
if (req->htc->pipeline.b != NULL)
req->acct_req.bodybytes +=
WRW_Write(wrk, req->htc->pipeline.b,
Tlen(req->htc->pipeline));
i = WRW_FlushRelease(wrk);
if (i) {
SES_Close(req->sp, SC_TX_PIPE);
VDI_CloseFd(&vc);
return;
}
req->t_resp = VTIM_real();
memset(fds, 0, sizeof fds);
// XXX: not yet (void)VTCP_linger(vc->fd, 0);
fds[0].fd = vc->fd;
fds[0].events = POLLIN | POLLERR;
// XXX: not yet (void)VTCP_linger(req->sp->fd, 0);
fds[1].fd = req->sp->fd;
fds[1].events = POLLIN | POLLERR;
while (fds[0].fd > -1 || fds[1].fd > -1) {
fds[0].revents = 0;
fds[1].revents = 0;
i = poll(fds, 2, cache_param->pipe_timeout * 1000);
if (i < 1)
break;
if (fds[0].revents && rdf(vc->fd, req->sp->fd)) {
if (fds[1].fd == -1)
break;
(void)shutdown(vc->fd, SHUT_RD);
(void)shutdown(req->sp->fd, SHUT_WR);
fds[0].events = 0;
fds[0].fd = -1;
}
if (fds[1].revents && rdf(req->sp->fd, vc->fd)) {
if (fds[0].fd == -1)
break;
(void)shutdown(req->sp->fd, SHUT_RD);
(void)shutdown(vc->fd, SHUT_WR);
fds[1].events = 0;
fds[1].fd = -1;
}
}
SES_Close(req->sp, SC_TX_PIPE);
VDI_CloseFd(&vc);
bo->vbc = NULL;
}
int
FetchBody(struct worker *wrk, struct object *obj)
{
int cls;
struct storage *st;
int mklen;
ssize_t cl;
struct http_conn *htc;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
bo = wrk->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->fetch_obj);
CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
htc = &bo->htc;
if (bo->vfp == NULL)
bo->vfp = &vfp_nop;
AssertObjCorePassOrBusy(obj->objcore);
AZ(bo->vgz_rx);
AZ(VTAILQ_FIRST(&obj->store));
bo->fetch_obj = obj;
bo->fetch_failed = 0;
/* XXX: pick up estimate from objdr ? */
cl = 0;
switch (bo->body_status) {
case BS_NONE:
cls = 0;
mklen = 0;
break;
case BS_ZERO:
cls = 0;
mklen = 1;
break;
case BS_LENGTH:
cl = fetch_number(bo->h_content_length, 10);
bo->vfp->begin(wrk, cl > 0 ? cl : 0);
cls = fetch_straight(wrk, htc, cl);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_CHUNKED:
bo->vfp->begin(wrk, cl);
cls = fetch_chunked(wrk, htc);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_EOF:
bo->vfp->begin(wrk, cl);
cls = fetch_eof(wrk, htc);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_ERROR:
cls = 1;
mklen = 0;
break;
default:
cls = 0;
mklen = 0;
INCOMPL();
}
AZ(bo->vgz_rx);
/*
* It is OK for ->end to just leave the last storage segment
* sitting on wrk->storage, we will always call vfp_nop_end()
* to get it trimmed or thrown out if empty.
*/
AZ(vfp_nop_end(wrk));
bo->fetch_obj = NULL;
WSLB(wrk, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
bo->body_status, body_status(bo->body_status),
cls, mklen);
if (bo->body_status == BS_ERROR) {
VDI_CloseFd(wrk, &bo->vbc);
return (__LINE__);
}
if (cls < 0) {
wrk->stats.fetch_failed++;
/* XXX: Wouldn't this store automatically be released ? */
while (!VTAILQ_EMPTY(&obj->store)) {
st = VTAILQ_FIRST(&obj->store);
VTAILQ_REMOVE(&obj->store, st, list);
STV_free(st);
}
VDI_CloseFd(wrk, &bo->vbc);
obj->len = 0;
return (__LINE__);
}
AZ(bo->fetch_failed);
if (cls == 0 && bo->should_close)
cls = 1;
WSLB(wrk, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert (uu <= obj->len);
else
assert(uu == obj->len);
}
if (mklen > 0) {
http_Unset(obj->http, H_Content_Length);
http_PrintfHeader(wrk, bo->vbc->vsl_id, obj->http,
"Content-Length: %zd", obj->len);
}
if (cls)
VDI_CloseFd(wrk, &bo->vbc);
else
VDI_RecycleFd(wrk, &bo->vbc);
return (0);
}
int
V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
{
struct vbc *vc;
struct http *hp;
enum htc_status_e hs;
int retry = -1;
int i, first;
struct http_conn *htc;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_ORNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
htc = &bo->htc;
AN(bo->director);
hp = bo->bereq;
bo->vbc = VDI_GetFd(NULL, bo);
if (bo->vbc == NULL) {
VSLb(bo->vsl, SLT_FetchError, "no backend connection");
return (-1);
}
vc = bo->vbc;
if (vc->recycled)
retry = 1;
/*
* Now that we know our backend, we can set a default Host:
* header if one is necessary. This cannot be done in the VCL
* because the backend may be chosen by a director.
*/
if (!http_GetHdr(bo->bereq, H_Host, NULL))
VDI_AddHostHeader(bo->bereq, vc);
(void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */
WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_fetch);
bo->t_send = VTIM_mono();
(void)HTTP1_Write(wrk, hp, 0); /* XXX: stats ? */
/* Deal with any message-body the request might (still) have */
i = 0;
if (req != NULL) {
i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
if (req->req_body_status == REQ_BODY_DONE)
retry = -1;
if (req->req_body_status == REQ_BODY_FAIL) {
VSLb(bo->vsl, SLT_FetchError,
"req.body read error: %d (%s)",
errno, strerror(errno));
req->doclose = SC_RX_BODY;
retry = -1;
}
}
if (WRW_FlushRelease(wrk) || i != 0) {
VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
errno, strerror(errno));
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (retry);
}
VSC_C_main->backend_req++;
bo->t_sent = VTIM_mono();
/* Receive response */
HTTP1_Init(htc, bo->ws, vc->fd, vc->vsl,
cache_param->http_resp_size,
cache_param->http_resp_hdr_len);
VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
first = 1;
do {
hs = HTTP1_Rx(htc);
if (hs == HTTP1_OVERFLOW) {
VSLb(bo->vsl, SLT_FetchError,
"http %sread error: overflow",
first ? "first " : "");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (-1);
}
if (hs == HTTP1_ERROR_EOF) {
VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
first ? "first " : "");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (retry);
}
if (first) {
retry = -1;
first = 0;
VTCP_set_read_timeout(vc->fd,
vc->between_bytes_timeout);
}
} while (hs != HTTP1_COMPLETE);
bo->t_hdr = VTIM_mono();
hp = bo->beresp;
if (HTTP1_DissectResponse(hp, htc)) {
VSLb(bo->vsl, SLT_FetchError, "http format error");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (-1);
}
return (0);
}
static int
cnt_fetchbody(struct sess *sp)
{
int i;
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
sp->handling == VCL_RET_DELIVER);
if (wrk->objcore == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
wrk->busyobj->exp.ttl = -1.;
} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
/* pass from vcl_fetch{} -> hit-for-pass */
/* XXX: the bereq was not filtered pass... */
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
wrk->busyobj->is_gzip =
http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
wrk->busyobj->is_gunzip =
!http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
wrk->busyobj->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (wrk->busyobj->do_gunzip)
http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
wrk->busyobj->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (wrk->busyobj->do_gzip)
http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
"Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (wrk->busyobj->do_esi)
wrk->busyobj->vfp = &vfp_esi;
else if (wrk->busyobj->do_gunzip)
wrk->busyobj->vfp = &vfp_gunzip;
else if (wrk->busyobj->do_gzip)
wrk->busyobj->vfp = &vfp_gzip;
else if (wrk->busyobj->is_gzip)
wrk->busyobj->vfp = &vfp_testgzip;
if (wrk->busyobj->do_esi || sp->esi_level > 0)
wrk->busyobj->do_stream = 0;
if (!sp->wantbody)
wrk->busyobj->do_stream = 0;
l = http_EstimateWS(wrk->busyobj->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
vary = VRY_Create(sp, wrk->busyobj->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
wrk->objcore == NULL)
wrk->storage_hint = TRANSIENT_STORAGE;
wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
if (wrk->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
if (wrk->busyobj->exp.ttl > cache_param->shortlived)
wrk->busyobj->exp.ttl = cache_param->shortlived;
wrk->busyobj->exp.grace = 0.0;
wrk->busyobj->exp.keep = 0.0;
}
if (wrk->obj == NULL) {
sp->err_code = 503;
sp->step = STP_ERROR;
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
wrk->storage_hint = NULL;
if (wrk->busyobj->do_gzip ||
(wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
wrk->obj->gziped = 1;
if (vary != NULL) {
wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
AN(wrk->obj->vary);
memcpy(wrk->obj->vary, VSB_data(vary), varyl);
VRY_Validate(wrk->obj->vary);
VSB_delete(vary);
}
wrk->obj->xid = sp->xid;
wrk->obj->response = sp->err_code;
WS_Assert(wrk->obj->ws_o);
/* Filter into object */
hp = wrk->busyobj->beresp;
hp2 = wrk->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(wrk, sp->vsl_id, hp2, hp,
pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(wrk, sp->vsl_id, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
wrk->obj->last_modified = VTIM_parse(b);
else
wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (wrk->obj->response == 200 &&
sp->http->conds &&
RFC2616_Do_Cond(sp))
wrk->busyobj->do_stream = 0;
AssertObjCorePassOrBusy(wrk->obj->objcore);
if (wrk->busyobj->do_stream) {
sp->step = STP_PREPRESP;
return (0);
}
/* Use unmodified headers*/
i = FetchBody(wrk, wrk->obj);
http_Setup(wrk->busyobj->bereq, NULL);
http_Setup(wrk->busyobj->beresp, NULL);
wrk->busyobj->vfp = NULL;
assert(WRW_IsReleased(wrk));
AZ(wrk->busyobj->vbc);
AN(sp->director);
if (i) {
HSH_Drop(wrk);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
AZ(wrk->obj);
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (wrk->obj->objcore != NULL) {
EXP_Insert(wrk->obj);
AN(wrk->obj->objcore);
AN(wrk->obj->objcore->ban);
HSH_Unbusy(wrk);
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
wrk->acct_tmp.fetch++;
sp->step = STP_PREPRESP;
return (0);
}
int
FetchHdr(struct sess *sp, int need_host_hdr)
{
struct vbc *vc;
struct worker *wrk;
struct http *hp;
int retry = -1;
int i;
struct http_conn *htc;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
htc = &wrk->busyobj->htc;
AN(sp->req->director);
AZ(sp->req->obj);
if (sp->req->objcore != NULL) { /* pass has no objcore */
CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC);
AN(sp->req->objcore->flags & OC_F_BUSY);
}
hp = wrk->busyobj->bereq;
sp->wrk->busyobj->vbc = VDI_GetFd(NULL, sp);
if (sp->wrk->busyobj->vbc == NULL) {
WSP(sp, SLT_FetchError, "no backend connection");
return (-1);
}
vc = sp->wrk->busyobj->vbc;
if (vc->recycled)
retry = 1;
/*
* Now that we know our backend, we can set a default Host:
* header if one is necessary. This cannot be done in the VCL
* because the backend may be chosen by a director.
*/
if (need_host_hdr)
VDI_AddHostHeader(sp->wrk, vc);
(void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */
WRW_Reserve(wrk, &vc->fd);
(void)http_Write(wrk, vc->vsl_id, hp, 0); /* XXX: stats ? */
/* Deal with any message-body the request might have */
i = FetchReqBody(sp);
if (WRW_FlushRelease(wrk) || i > 0) {
WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
errno, strerror(errno));
VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
/* XXX: other cleanup ? */
return (retry);
}
/* Checkpoint the vsl.here */
WSL_Flush(wrk, 0);
/* XXX is this the right place? */
VSC_C_main->backend_req++;
/* Receive response */
HTC_Init(htc, wrk->ws, vc->fd, vc->vsl_id,
cache_param->http_resp_size,
cache_param->http_resp_hdr_len);
VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
i = HTC_Rx(htc);
if (i < 0) {
WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
i, errno, strerror(errno));
VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
/* XXX: other cleanup ? */
/* Retryable if we never received anything */
return (i == -1 ? retry : -1);
}
VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
while (i == 0) {
i = HTC_Rx(htc);
if (i < 0) {
WSP(sp, SLT_FetchError,
"http first read error: %d %d (%s)",
i, errno, strerror(errno));
VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
/* XXX: other cleanup ? */
return (-1);
}
}
hp = wrk->busyobj->beresp;
if (http_DissectResponse(wrk, htc, hp)) {
WSP(sp, SLT_FetchError, "http format error");
VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
/* XXX: other cleanup ? */
return (-1);
}
return (0);
}
static int
cnt_fetch(struct sess *sp)
{
int i, need_host_hdr;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
AN(sp->director);
AZ(wrk->busyobj->vbc);
AZ(wrk->busyobj->should_close);
AZ(wrk->storage_hint);
http_Setup(wrk->busyobj->beresp, wrk->ws);
need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL);
i = FetchHdr(sp, need_host_hdr);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = FetchHdr(sp, need_host_hdr);
}
if (i) {
sp->handling = VCL_RET_ERROR;
sp->err_code = 503;
} else {
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(wrk->busyobj->beresp, H_Cache_Control);
http_CollectHdr(wrk->busyobj->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
wrk->busyobj->body_status = RFC2616_Body(sp);
sp->err_code = http_GetStatus(wrk->busyobj->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&wrk->busyobj->exp);
wrk->busyobj->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(sp);
/* pass from vclrecv{} has negative TTL */
if (wrk->objcore == NULL)
wrk->busyobj->exp.ttl = -1.;
AZ(wrk->busyobj->do_esi);
VCL_fetch_method(sp);
switch (sp->handling) {
case VCL_RET_HIT_FOR_PASS:
if (wrk->objcore != NULL)
wrk->objcore->flags |= OC_F_PASS;
sp->step = STP_FETCHBODY;
return (0);
case VCL_RET_DELIVER:
AssertObjCorePassOrBusy(wrk->objcore);
sp->step = STP_FETCHBODY;
return (0);
default:
break;
}
/* We are not going to fetch the body, Close the connection */
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
}
/* Clean up partial fetch */
AZ(wrk->busyobj->vbc);
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
AZ(HSH_Deref(wrk, wrk->objcore, NULL));
wrk->objcore = NULL;
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
sp->director = NULL;
wrk->storage_hint = NULL;
switch (sp->handling) {
case VCL_RET_RESTART:
sp->restarts++;
sp->step = STP_RECV;
return (0);
case VCL_RET_ERROR:
sp->step = STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
}
static int
cnt_prepresp(struct sess *sp)
{
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
if (wrk->busyobj != NULL) {
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
AN(wrk->busyobj->do_stream);
AssertObjCorePassOrBusy(wrk->obj->objcore);
}
wrk->res_mode = 0;
if (wrk->busyobj == NULL)
wrk->res_mode |= RES_LEN;
if (wrk->busyobj != NULL &&
(wrk->busyobj->h_content_length != NULL ||
!wrk->busyobj->do_stream) &&
!wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
wrk->res_mode |= RES_LEN;
if (!sp->disable_esi && wrk->obj->esidata != NULL) {
/* In ESI mode, we don't know the aggregate length */
wrk->res_mode &= ~RES_LEN;
wrk->res_mode |= RES_ESI;
}
if (sp->esi_level > 0) {
wrk->res_mode &= ~RES_LEN;
wrk->res_mode |= RES_ESI_CHILD;
}
if (cache_param->http_gzip_support && wrk->obj->gziped &&
!RFC2616_Req_Gzip(sp)) {
/*
* We don't know what it uncompresses to
* XXX: we could cache that
*/
wrk->res_mode &= ~RES_LEN;
wrk->res_mode |= RES_GUNZIP;
}
if (!(wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
if (wrk->obj->len == 0 &&
(wrk->busyobj == NULL || !wrk->busyobj->do_stream))
/*
* If the object is empty, neither ESI nor GUNZIP
* can make it any different size
*/
wrk->res_mode |= RES_LEN;
else if (!sp->wantbody) {
/* Nothing */
} else if (sp->http->protover >= 11) {
wrk->res_mode |= RES_CHUNKED;
} else {
wrk->res_mode |= RES_EOF;
sp->doclose = "EOF mode";
}
}
sp->t_resp = W_TIM_real(wrk);
if (wrk->obj->objcore != NULL) {
if ((sp->t_resp - wrk->obj->last_lru) >
cache_param->lru_timeout &&
EXP_Touch(wrk->obj->objcore))
wrk->obj->last_lru = sp->t_resp;
wrk->obj->last_use = sp->t_resp; /* XXX: locking ? */
}
http_Setup(wrk->resp, wrk->ws);
RES_BuildHttp(sp);
VCL_deliver_method(sp);
switch (sp->handling) {
case VCL_RET_DELIVER:
break;
case VCL_RET_RESTART:
if (sp->restarts >= cache_param->max_restarts)
break;
if (wrk->busyobj != NULL) {
AN(wrk->busyobj->do_stream);
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
HSH_Drop(wrk);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
} else {
(void)HSH_Deref(wrk, NULL, &wrk->obj);
}
AZ(wrk->obj);
sp->restarts++;
sp->director = NULL;
http_Setup(wrk->resp, NULL);
sp->step = STP_RECV;
return (0);
default:
WRONG("Illegal action in vcl_deliver{}");
}
if (wrk->busyobj != NULL && wrk->busyobj->do_stream) {
AssertObjCorePassOrBusy(wrk->obj->objcore);
sp->step = STP_STREAMBODY;
} else {
sp->step = STP_DELIVER;
}
return (0);
}
void
PipeSession(struct sess *sp)
{
struct vbc *vc;
struct worker *w;
struct pollfd fds[2];
int i;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
w = sp->wrk;
sp->vbc = VDI_GetFd(NULL, sp);
if (sp->vbc == NULL)
return;
vc = sp->vbc;
(void)VTCP_blocking(vc->fd);
WRW_Reserve(w, &vc->fd);
sp->wrk->acct_tmp.hdrbytes +=
http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
if (sp->htc->pipeline.b != NULL)
sp->wrk->acct_tmp.bodybytes +=
WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
i = WRW_FlushRelease(w);
if (i) {
SES_Close(sp, "pipe");
VDI_CloseFd(sp);
return;
}
sp->t_resp = VTIM_real();
memset(fds, 0, sizeof fds);
// XXX: not yet (void)VTCP_linger(vc->fd, 0);
fds[0].fd = vc->fd;
fds[0].events = POLLIN | POLLERR;
// XXX: not yet (void)VTCP_linger(sp->fd, 0);
fds[1].fd = sp->fd;
fds[1].events = POLLIN | POLLERR;
while (fds[0].fd > -1 || fds[1].fd > -1) {
fds[0].revents = 0;
fds[1].revents = 0;
i = poll(fds, 2, params->pipe_timeout * 1000);
if (i < 1)
break;
if (fds[0].revents && rdf(vc->fd, sp->fd)) {
if (fds[1].fd == -1)
break;
(void)shutdown(vc->fd, SHUT_RD);
(void)shutdown(sp->fd, SHUT_WR);
fds[0].events = 0;
fds[0].fd = -1;
}
if (fds[1].revents && rdf(sp->fd, vc->fd)) {
if (fds[0].fd == -1)
break;
(void)shutdown(sp->fd, SHUT_RD);
(void)shutdown(vc->fd, SHUT_WR);
fds[1].events = 0;
fds[1].fd = -1;
}
}
SES_Close(sp, "pipe");
VDI_CloseFd(sp);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
void
V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
{
int cls;
struct storage *st;
ssize_t cl;
struct http_conn *htc;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
htc = &bo->htc;
CHECK_OBJ_ORNULL(bo->vbc, VBC_MAGIC);
obj = bo->fetch_obj;
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
assert(bo->state == BOS_FETCHING);
/*
* XXX: The busyobj needs a dstat, but it is not obvious which one
* XXX: it should be (own/borrowed). For now borrow the wrk's.
*/
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->vfp);
AZ(bo->vgz_rx);
assert(VTAILQ_EMPTY(&obj->store));
/* XXX: pick up estimate from objdr ? */
cl = 0;
cls = bo->should_close;
switch (htc->body_status) {
case BS_NONE:
break;
case BS_ZERO:
break;
case BS_LENGTH:
cl = vbf_fetch_number(bo->h_content_length, 10);
bo->vfp->begin(bo, cl);
if (bo->state == BOS_FETCHING && cl > 0)
cls |= vbf_fetch_straight(bo, htc, cl);
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_CHUNKED:
bo->vfp->begin(bo, cl > 0 ? cl : 0);
if (bo->state == BOS_FETCHING)
cls |= vbf_fetch_chunked(bo, htc);
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_EOF:
bo->vfp->begin(bo, cl > 0 ? cl : 0);
if (bo->state == BOS_FETCHING)
vbf_fetch_eof(bo, htc);
cls = 1;
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_ERROR:
cls |= VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
INCOMPL();
}
bo->t_body = VTIM_mono();
AZ(bo->vgz_rx);
/*
* Trim or delete the last segment, if any
*/
st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
/* XXX: Temporary: Only trim if we are not streaming */
if (st != NULL && !bo->do_stream) {
/* XXX: is any of this safe under streaming ? */
if (st->len == 0) {
VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
STV_free(st);
} else if (st->len < st->space) {
STV_trim(st, st->len, 1);
}
}
bo->vfp = NULL;
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d",
htc->body_status, body_status_2str(htc->body_status), cls);
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
if (bo->vbc != NULL) {
if (cls)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
}
AZ(bo->vbc);
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
bo->stats = NULL;
}
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
unsigned l;
uint16_t nhttp;
struct object *obj;
struct objiter *oi;
void *sp;
ssize_t sl, al, tl, vl;
struct storage *st;
enum objiter_status ois;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
l = 0;
if (bo->ims_obj->vary != NULL) {
vl = VRY_Validate(bo->ims_obj->vary);
l += vl;
} else
vl = 0;
l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
obj->gziped = bo->ims_obj->gziped;
obj->gzip_start = bo->ims_obj->gzip_start;
obj->gzip_last = bo->ims_obj->gzip_last;
obj->gzip_stop = bo->ims_obj->gzip_stop;
/* XXX: ESI */
if (bo->ims_obj->vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
bo->ims_obj->vary, vl);
assert(vl == VRY_Validate(obj->vary));
}
obj->vxid = bo->vsl->wid;
obj->http->logtag = HTTP_Obj;
/* XXX: we should have our own HTTP_A_CONDFETCH */
http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
http_CopyHome(obj->http);
AZ(WS_Overflowed(bo->ws_o));
VBO_setstate(bo, BOS_FETCHING);
HSH_Unbusy(&wrk->stats, obj->objcore);
if (!(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
st = NULL;
al = 0;
oi = ObjIterBegin(wrk, bo->ims_obj);
do {
ois = ObjIter(oi, &sp, &sl);
while (sl > 0) {
if (st == NULL) {
st = VFP_GetStorage(bo, bo->ims_obj->len - al);
XXXAN(st);
}
tl = sl;
if (tl > st->space - st->len)
tl = st->space - st->len;
memcpy(st->ptr + st->len, sp, tl);
al += tl;
sp = (char *)sp + tl;
sl -= tl;
VBO_extend(bo, al);
if (st->len == st->space)
st = NULL;
}
} while (ois == OIS_DATA || ois == OIS_STREAM);
ObjIterEnd(&oi);
bo->stats = NULL;
assert(al == bo->ims_obj->len);
assert(obj->len == al);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
HSH_Complete(obj->objcore);
return (F_STP_DONE);
}
static int
cnt_fetch(struct worker *wrk, struct req *req)
{
int i, need_host_hdr;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AN(req->director);
AZ(bo->vbc);
AZ(bo->should_close);
AZ(req->storage_hint);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
req->acct_req.fetch++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
}
if (i) {
req->handling = VCL_RET_ERROR;
req->err_code = 503;
} else {
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->body_status = RFC2616_Body(bo, &wrk->stats);
req->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
bo->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(bo);
/* pass from vclrecv{} has negative TTL */
if (req->objcore->objhead == NULL)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->do_pass);
VCL_fetch_method(req);
if (bo->do_pass)
req->objcore->flags |= OC_F_PASS;
switch (req->handling) {
case VCL_RET_DELIVER:
req->req_step = R_STP_FETCHBODY;
return (0);
default:
break;
}
/* We are not going to fetch the body, Close the connection */
VDI_CloseFd(&bo->vbc);
}
/* Clean up partial fetch */
AZ(bo->vbc);
if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
req->objcore = NULL;
}
assert(bo->refcount == 2);
VBO_DerefBusyObj(wrk, &bo);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->director = NULL;
req->storage_hint = NULL;
switch (req->handling) {
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
return (0);
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
int
FetchBody(struct sess *sp)
{
int cls;
struct storage *st;
struct worker *w;
int mklen;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
w = sp->wrk;
CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
if (w->vfp == NULL)
w->vfp = &vfp_nop;
AN(sp->director);
AssertObjCorePassOrBusy(sp->obj->objcore);
AZ(w->vgz_rx);
AZ(VTAILQ_FIRST(&sp->obj->store));
switch (w->body_status) {
case BS_NONE:
cls = 0;
mklen = 0;
break;
case BS_ZERO:
cls = 0;
mklen = 1;
break;
case BS_LENGTH:
cls = fetch_straight(sp, w->htc,
w->h_content_length);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_CHUNKED:
cls = fetch_chunked(sp, w->htc);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_EOF:
cls = fetch_eof(sp, w->htc);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_ERROR:
cls = 1;
mklen = 0;
break;
default:
cls = 0;
mklen = 0;
INCOMPL();
}
AZ(w->vgz_rx);
/*
* It is OK for ->end to just leave the last storage segment
* sitting on w->storage, we will always call vfp_nop_end()
* to get it trimmed or thrown out if empty.
*/
AZ(vfp_nop_end(sp));
WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
w->body_status, body_status(w->body_status),
cls, mklen);
if (w->body_status == BS_ERROR) {
VDI_CloseFd(sp);
return (__LINE__);
}
if (cls < 0) {
w->stats.fetch_failed++;
/* XXX: Wouldn't this store automatically be released ? */
while (!VTAILQ_EMPTY(&sp->obj->store)) {
st = VTAILQ_FIRST(&sp->obj->store);
VTAILQ_REMOVE(&sp->obj->store, st, list);
STV_free(st);
}
VDI_CloseFd(sp);
sp->obj->len = 0;
return (__LINE__);
}
if (cls == 0 && w->do_close)
cls = 1;
WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &sp->obj->store, list)
uu += st->len;
if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
/* Streaming might have started freeing stuff */
assert (uu <= sp->obj->len);
else
assert(uu == sp->obj->len);
}
if (mklen > 0) {
http_Unset(sp->obj->http, H_Content_Length);
http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
"Content-Length: %jd", (intmax_t)sp->obj->len);
}
if (cls)
VDI_CloseFd(sp);
else
VDI_RecycleFd(sp);
return (0);
}
