static enum body_status
http1_body_status(const struct http *hp, struct http_conn *htc, int request)
{
ssize_t cl;
const char *b;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
htc->content_length = -1;
cl = http_GetContentLength(hp);
if (cl == -2)
return (BS_ERROR);
if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
if (strcasecmp(b, "chunked"))
return (BS_ERROR);
if (cl != -1) {
/*
* RFC7230 3.3.3 allows more lenient handling
* but we're going to be strict.
*/
return (BS_ERROR);
}
return (BS_CHUNKED);
}
if (cl >= 0) {
htc->content_length = cl;
return (cl == 0 ? BS_NONE : BS_LENGTH);
}
if (hp->protover == 11 && (request || !http_HdrIs(hp, H_Connection, "close")))
return (BS_NONE);
if (http_HdrIs(hp, H_Connection, "keep-alive")) {
/*
* Keep alive with neither TE=Chunked or C-Len is impossible.
* We assume a zero length body.
*/
return (BS_NONE);
}
/*
* Fall back to EOF transfer.
*/
return (BS_EOF);
}
static int
vbf_figure_out_vfp(struct busyobj *bo)
{
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* No body or no GZIP supprt -> done */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
!cache_param->http_gzip_support) {
http_Unset(bo->beresp, H_Content_Encoding);
bo->do_gzip = bo->do_gunzip = 0;
bo->do_stream = 0;
return (0);
}
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless if it already is gzip'ed */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL)
return (-1);
if (bo->do_esi && (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)))
return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0);
if (bo->do_esi)
return (VFP_Push(bo->vfc, &VFP_esi) == NULL ? -1 : 0);
if (bo->do_gzip)
return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0);
if (bo->is_gzip && !bo->do_gunzip)
return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0);
return (0);
}
V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
{
int err = 0;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
if (sendbody) {
if (http_GetHdr(req->resp, H_Content_Length, NULL))
req->res_mode |= RES_LEN;
else if (req->http->protover == 11) {
req->res_mode |= RES_CHUNKED;
http_SetHeader(req->resp, "Transfer-Encoding: chunked");
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
if (req->doclose) {
if (!http_HdrIs(req->resp, H_Connection, "close")) {
http_Unset(req->resp, H_Connection);
http_SetHeader(req->resp, "Connection: close");
}
} else if (!http_GetHdr(req->resp, H_Connection, NULL))
http_SetHeader(req->resp, "Connection: keep-alive");
if (sendbody && req->resp_len != 0)
VDP_push(req, v1d_bytes, NULL, 1);
AZ(req->wrk->v1l);
V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev);
if (WS_Overflowed(req->ws)) {
v1d_error(req, "workspace_client overflow");
AZ(req->wrk->v1l);
return;
}
req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
if (DO_DEBUG(DBG_FLUSH_HEAD))
(void)V1L_Flush(req->wrk);
if (sendbody && req->resp_len != 0) {
if (req->res_mode & RES_CHUNKED)
V1L_Chunked(req->wrk);
err = VDP_DeliverObj(req);
if (!err && (req->res_mode & RES_CHUNKED))
V1L_EndChunk(req->wrk);
}
if ((V1L_FlushRelease(req->wrk) || err) && req->sp->fd >= 0)
SES_Close(req->sp, SC_REM_CLOSE);
AZ(req->wrk->v1l);
VDP_close(req);
}
vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
{
struct vgz *vg;
CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
if (http_HdrIs(vc->http, H_Content_Length, "0")) {
http_Unset(vc->http, H_Content_Encoding);
return (VFP_NULL);
}
if (vfe->vfp->priv2 == VFP_GZIP) {
if (http_GetHdr(vc->http, H_Content_Encoding, NULL))
return (VFP_NULL);
vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
} else {
if (!http_HdrIs(vc->http, H_Content_Encoding, "gzip"))
return (VFP_NULL);
vg = VGZ_NewUngzip(vc->wrk->vsl, vfe->vfp->priv1);
}
if (vg == NULL)
return (VFP_ERROR);
vfe->priv1 = vg;
if (vgz_getmbuf(vg))
return (VFP_ERROR);
VGZ_Ibuf(vg, vg->m_buf, 0);
AZ(vg->m_len);
if (vfe->vfp->priv2 == VFP_GUNZIP || vfe->vfp->priv2 == VFP_GZIP) {
http_Unset(vc->http, H_Content_Encoding);
http_Unset(vc->http, H_Content_Length);
RFC2616_Weaken_Etag(vc->http);
}
if (vfe->vfp->priv2 == VFP_GZIP)
http_SetHeader(vc->http, "Content-Encoding: gzip");
if (vfe->vfp->priv2 == VFP_GZIP || vfe->vfp->priv2 == VFP_TESTGUNZIP)
RFC2616_Vary_AE(vc->http);
return (VFP_OK);
}
void
http_ForceHeader(struct http *to, const char *hdr, const char *val)
{
CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
if (http_HdrIs(to, hdr, val))
return;
http_Unset(to, hdr);
http_PrintfHeader(to, "%s %s", hdr + 1, val);
}
static enum req_fsm_nxt
cnt_deliver(struct worker *wrk, struct req *req)
{
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore->objhead, OBJHEAD_MAGIC);
AN(req->vcl);
assert(req->objcore->refcnt > 0);
if (req->objcore->exp_flags & OC_EF_EXP)
EXP_Touch(req->objcore, req->t_prev);
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
AZ(HTTP_Decode(req->resp,
ObjGetattr(req->wrk, req->objcore, OA_HEADERS, NULL)));
http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
if (req->is_hit)
http_PrintfHeader(req->resp,
"X-Varnish: %u %u", VXID(req->vsl->wid),
ObjGetXID(wrk, req->objcore));
else
http_PrintfHeader(req->resp,
"X-Varnish: %u", VXID(req->vsl->wid));
/* We base Age calculation upon the last timestamp taken during
client request processing. This gives some inaccuracy, but
since Age is only full second resolution that shouldn't
matter. (Last request timestamp could be a Start timestamp
taken before the object entered into cache leading to negative
age. Truncate to zero in that case).
*/
http_PrintfHeader(req->resp, "Age: %.0f",
fmax(0., req->t_prev - req->objcore->exp.t_origin));
http_SetHeader(req->resp, "Via: 1.1 varnish-v4");
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http))
RFC2616_Weaken_Etag(req->resp);
VCL_deliver_method(req->vcl, wrk, req, NULL, NULL);
VSLb_ts_req(req, "Process", W_TIM_real(wrk));
/* Stop the insanity before it turns "Hotel California" on us */
if (req->restarts >= cache_param->max_restarts)
wrk->handling = VCL_RET_DELIVER;
if (wrk->handling != VCL_RET_DELIVER) {
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
switch (wrk->handling) {
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
break;
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
break;
default:
INCOMPL();
}
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (!(req->objcore->flags & OC_F_PASS)
&& req->esi_level == 0
&& http_IsStatus(req->resp, 200)
&& req->http->conds && RFC2616_Do_Cond(req))
http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
/* Grab a ref to the bo if there is one, and hand it down */
bo = HSH_RefBusy(req->objcore);
if (bo != NULL) {
if (req->esi_level == 0 && bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &bo);
} else if (!bo->do_stream) {
VBO_waitstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo);
}
}
cnt_vdp(req, bo);
if (bo != NULL)
VBO_DerefBusyObj(wrk, &bo);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
if ((req->objcore->flags & OC_F_PASS) && bo != NULL) {
VBO_waitstate(bo, BOS_FINISHED);
ObjSlim(wrk, req->objcore);
}
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
AN(bo->vbc);
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);
if (bo->do_esi && bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
(void)VFP_Push(bo->vfc, &vfp_esi, 1);
} else if (bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open");
bo->doclose = SC_RX_BODY;
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->doclose = SC_RX_BODY;
return (F_STP_ERROR);
}
assert(WRW_IsReleased(wrk));
obj = bo->fetch_obj;
bo->vfc->body = obj->body;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (bo->do_gzip || bo->do_gunzip)
obj->changed_gzip = 1;
if (bo->htc.body_status != BS_NONE)
V1F_Setup_Fetch(bo);
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
assert (bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc.body_status, body_status_2str(bo->htc.body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc.body_status != BS_NONE) {
assert(bo->htc.body_status != BS_ERROR);
VFP_Fetch_Body(bo);
bo->acct.beresp_bodybytes = bo->vfc->bodybytes;
}
if (bo->vfc->failed && !bo->do_stream) {
assert(bo->state < BOS_STREAM);
if (bo->fetch_obj != NULL) {
ObjFreeObj(bo->fetch_objcore, bo->stats);
bo->fetch_obj = NULL;
}
return (F_STP_ERROR);
}
if (bo->vfc->failed)
return (F_STP_FAIL);
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && bo->doclose == SC_NULL) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->ims_obj != NULL)
EXP_Rearm(bo->ims_obj->objcore,
bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
return (F_STP_DONE);
}
void
HTTP1_Session(struct worker *wrk, struct req *req)
{
enum htc_status_e hs;
struct sess *sp;
const char *st;
int i;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
sp = req->sp;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
/*
* Whenever we come in from the acceptor or waiter, we need to set
* blocking mode. It would be simpler to do this in the acceptor
* or waiter, but we'd rather do the syscall in the worker thread.
* On systems which return errors for ioctl, we close early
*/
if (http1_getstate(sp) == H1NEWREQ && VTCP_blocking(sp->fd)) {
if (errno == ECONNRESET)
SES_Close(sp, SC_REM_CLOSE);
else
SES_Close(sp, SC_TX_ERROR);
AN(Req_Cleanup(sp, wrk, req));
return;
}
while (1) {
st = http1_getstate(sp);
if (st == H1NEWREQ) {
assert(isnan(req->t_prev));
assert(isnan(req->t_req));
AZ(req->vcl);
AZ(req->esi_level);
hs = HTC_RxStuff(req->htc, HTTP1_Complete,
&req->t_first, &req->t_req,
sp->t_idle + cache_param->timeout_linger,
sp->t_idle + cache_param->timeout_idle,
cache_param->http_req_size);
XXXAZ(req->htc->ws->r);
if (hs < HTC_S_EMPTY) {
req->acct.req_hdrbytes +=
req->htc->rxbuf_e - req->htc->rxbuf_b;
CNT_AcctLogCharge(wrk->stats, req);
Req_Release(req);
switch(hs) {
case HTC_S_CLOSE:
SES_Delete(sp, SC_REM_CLOSE, NAN);
return;
case HTC_S_TIMEOUT:
SES_Delete(sp, SC_RX_TIMEOUT, NAN);
return;
case HTC_S_OVERFLOW:
SES_Delete(sp, SC_RX_OVERFLOW, NAN);
return;
case HTC_S_EOF:
SES_Delete(sp, SC_REM_CLOSE, NAN);
return;
default:
WRONG("htc_status (bad)");
}
}
if (hs == HTC_S_IDLE) {
wrk->stats->sess_herd++;
Req_Release(req);
SES_Wait(sp, &HTTP1_transport);
return;
}
if (hs != HTC_S_COMPLETE)
WRONG("htc_status (nonbad)");
if (H2_prism_complete(req->htc) == HTC_S_COMPLETE) {
if (!FEATURE(FEATURE_HTTP2)) {
VSLb(req->vsl, SLT_Debug,
"H2 attempt");
SES_Close(req->sp, req->doclose);
http1_setstate(sp, H1CLEANUP);
continue;
}
VSLb(req->vsl, SLT_Debug,
"H2 Prior Knowledge Upgrade");
http1_setstate(sp, NULL);
req->err_code = 1;
SES_SetTransport(wrk, sp, req, &H2_transport);
return;
}
i = http1_dissect(wrk, req);
req->acct.req_hdrbytes +=
req->htc->rxbuf_e - req->htc->rxbuf_b;
if (i) {
SES_Close(req->sp, req->doclose);
http1_setstate(sp, H1CLEANUP);
continue;
}
if (req->htc->body_status == BS_NONE && /* XXX */
http_HdrIs(req->http, H_Upgrade, "h2c")) {
if (!FEATURE(FEATURE_HTTP2)) {
VSLb(req->vsl, SLT_Debug,
"H2 upgrade attempt");
SES_Close(req->sp, req->doclose);
http1_setstate(sp, H1CLEANUP);
continue;
}
VSLb(req->vsl, SLT_Debug,
"H2 Upgrade");
http1_setstate(sp, NULL);
req->err_code = 2;
SES_SetTransport(wrk, sp, req, &H2_transport);
return;
}
req->req_step = R_STP_RECV;
http1_setstate(sp, H1PROC);
} else if (st == H1BUSY) {
/*
* Return from waitinglist.
* Check to see if the remote has left.
*/
if (VTCP_check_hup(sp->fd)) {
AN(req->hash_objhead);
(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
AZ(req->hash_objhead);
SES_Close(sp, SC_REM_CLOSE);
AN(Req_Cleanup(sp, wrk, req));
return;
}
http1_setstate(sp, H1PROC);
} else if (st == H1PROC) {
req->transport = &HTTP1_transport;
req->task.func = http1_req;
req->task.priv = req;
if (CNT_Request(wrk, req) == REQ_FSM_DISEMBARK)
return;
req->transport = NULL;
req->task.func = NULL;
req->task.priv = NULL;
http1_setstate(sp, H1CLEANUP);
} else if (st == H1CLEANUP) {
if (Req_Cleanup(sp, wrk, req))
return;
HTC_RxInit(req->htc, req->ws);
if (req->htc->rxbuf_e != req->htc->rxbuf_b)
wrk->stats->sess_readahead++;
http1_setstate(sp, H1NEWREQ);
} else {
WRONG("Wrong H1 session state");
}
}
}
void
V1D_Deliver(struct req *req, struct busyobj *bo)
{
const char *r;
enum objiter_status ois;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
req->res_mode = 0;
/*
* Determine ESI status first. Not dependent on wantbody, because
* we want ESI to supress C-L in HEAD too.
*/
if (!req->disable_esi &&
ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL)
req->res_mode |= RES_ESI;
/*
* ESI-childen don't care about headers -> early escape
*/
if (req->esi_level > 0) {
ESI_DeliverChild(req, bo);
return;
}
if (req->res_mode & RES_ESI) {
RFC2616_Weaken_Etag(req->resp);
} else if (http_IsStatus(req->resp, 304)) {
http_Unset(req->resp, H_Content_Length);
req->wantbody = 0;
} else if (bo == NULL &&
!http_GetHdr(req->resp, H_Content_Length, NULL)) {
http_PrintfHeader(req->resp,
"Content-Length: %ju", (uintmax_t)ObjGetLen(
req->wrk, req->objcore));
}
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http)) {
/*
* We don't know what it uncompresses to
* XXX: we could cache that, but would still deliver
* XXX: with multiple writes because of the gunzip buffer
*/
req->res_mode |= RES_GUNZIP;
VDP_push(req, VDP_gunzip, NULL, 0);
}
if (req->res_mode & RES_ESI) {
/* Gunzip could have added back a C-L */
http_Unset(req->resp, H_Content_Length);
}
/*
* Range comes after the others and pushes on bottom because it
* can generate a correct C-L header.
*/
if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) {
http_SetHeader(req->resp, "Accept-Ranges: bytes");
if (req->wantbody && http_GetHdr(req->http, H_Range, &r))
VRG_dorange(req, bo, r);
}
if (http_GetHdr(req->resp, H_Content_Length, NULL))
req->res_mode |= RES_LEN;
if (req->wantbody && !(req->res_mode & RES_LEN)) {
if (req->http->protover >= 11) {
req->res_mode |= RES_CHUNKED;
http_SetHeader(req->resp, "Transfer-Encoding: chunked");
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
if (req->doclose) {
if (!http_HdrIs(req->resp, H_Connection, "close")) {
http_Unset(req->resp, H_Connection);
http_SetHeader(req->resp, "Connection: close");
}
} else if (!http_GetHdr(req->resp, H_Connection, NULL))
http_SetHeader(req->resp, "Connection: keep-alive");
VDP_push(req, v1d_bytes, NULL, 1);
V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev);
req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
if (DO_DEBUG(DBG_FLUSH_HEAD))
(void)V1L_Flush(req->wrk);
ois = OIS_DONE;
if (req->wantbody) {
if (req->res_mode & RES_CHUNKED)
V1L_Chunked(req->wrk);
ois = VDP_DeliverObj(req);
(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
if (ois == OIS_DONE && (req->res_mode & RES_CHUNKED))
V1L_EndChunk(req->wrk);
}
if ((V1L_FlushRelease(req->wrk) || ois != OIS_DONE) && req->sp->fd >= 0)
SES_Close(req->sp, SC_REM_CLOSE);
VDP_close(req);
}
static void
HTTP1_Session(struct worker *wrk, struct req *req)
{
enum htc_status_e hs;
struct sess *sp;
const char *st;
int i;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
sp = req->sp;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
/*
* Whenever we come in from the acceptor or waiter, we need to set
* blocking mode. It would be simpler to do this in the acceptor
* or waiter, but we'd rather do the syscall in the worker thread.
*/
if (http1_getstate(sp) == H1NEWREQ)
VTCP_blocking(sp->fd);
req->transport = &HTTP1_transport;
while (1) {
st = http1_getstate(sp);
if (st == H1NEWREQ) {
CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
assert(isnan(req->t_prev));
assert(isnan(req->t_req));
AZ(req->vcl);
AZ(req->esi_level);
AN(req->htc->ws->r);
hs = HTC_RxStuff(req->htc, HTTP1_Complete,
&req->t_first, &req->t_req,
sp->t_idle + cache_param->timeout_linger,
sp->t_idle + cache_param->timeout_idle,
NAN,
cache_param->http_req_size);
AZ(req->htc->ws->r);
if (hs < HTC_S_EMPTY) {
req->acct.req_hdrbytes +=
req->htc->rxbuf_e - req->htc->rxbuf_b;
Req_Cleanup(sp, wrk, req);
Req_Release(req);
switch (hs) {
case HTC_S_CLOSE:
SES_Delete(sp, SC_REM_CLOSE, NAN);
return;
case HTC_S_TIMEOUT:
SES_Delete(sp, SC_RX_TIMEOUT, NAN);
return;
case HTC_S_OVERFLOW:
SES_Delete(sp, SC_RX_OVERFLOW, NAN);
return;
case HTC_S_EOF:
SES_Delete(sp, SC_REM_CLOSE, NAN);
return;
default:
WRONG("htc_status (bad)");
}
}
if (hs == HTC_S_IDLE) {
wrk->stats->sess_herd++;
Req_Cleanup(sp, wrk, req);
Req_Release(req);
SES_Wait(sp, &HTTP1_transport);
return;
}
if (hs != HTC_S_COMPLETE)
WRONG("htc_status (nonbad)");
if (H2_prism_complete(req->htc) == HTC_S_COMPLETE) {
if (!FEATURE(FEATURE_HTTP2)) {
SES_Close(req->sp, SC_REQ_HTTP20);
AZ(req->ws->r);
AZ(wrk->aws->r);
http1_setstate(sp, H1CLEANUP);
continue;
}
http1_setstate(sp, NULL);
H2_PU_Sess(wrk, sp, req);
return;
}
i = http1_dissect(wrk, req);
req->acct.req_hdrbytes +=
req->htc->rxbuf_e - req->htc->rxbuf_b;
if (i) {
assert(req->doclose > 0);
SES_Close(req->sp, req->doclose);
AZ(req->ws->r);
AZ(wrk->aws->r);
http1_setstate(sp, H1CLEANUP);
continue;
}
if (http_HdrIs(req->http, H_Upgrade, "h2c")) {
if (!FEATURE(FEATURE_HTTP2)) {
VSLb(req->vsl, SLT_Debug,
"H2 upgrade attempt");
} else if (req->htc->body_status != BS_NONE) {
VSLb(req->vsl, SLT_Debug,
"H2 upgrade attempt has body");
} else {
http1_setstate(sp, NULL);
req->err_code = 2;
H2_OU_Sess(wrk, sp, req);
return;
}
}
req->req_step = R_STP_TRANSPORT;
http1_setstate(sp, H1PROC);
} else if (st == H1PROC) {
req->task.func = http1_req;
req->task.priv = req;
wrk->stats->client_req++;
CNT_Embark(wrk, req);
if (req->req_step == R_STP_TRANSPORT)
VCL_TaskEnter(req->vcl, req->privs);
if (CNT_Request(req) == REQ_FSM_DISEMBARK)
return;
AZ(req->vcl0);
req->task.func = NULL;
req->task.priv = NULL;
AZ(req->ws->r);
AZ(wrk->aws->r);
http1_setstate(sp, H1CLEANUP);
} else if (st == H1CLEANUP) {
AZ(wrk->aws->r);
AZ(req->ws->r);
if (sp->fd >= 0 && req->doclose != SC_NULL)
SES_Close(sp, req->doclose);
if (sp->fd < 0) {
wrk->stats->sess_closed++;
Req_Cleanup(sp, wrk, req);
Req_Release(req);
SES_Delete(sp, SC_NULL, NAN);
return;
}
Req_Cleanup(sp, wrk, req);
HTC_RxInit(req->htc, req->ws);
if (req->htc->rxbuf_e != req->htc->rxbuf_b)
wrk->stats->sess_readahead++;
http1_setstate(sp, H1NEWREQ);
} else {
WRONG("Wrong H1 session state");
}
}
}
static int
cnt_fetchbody(struct sess *sp)
{
int i;
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
sp->handling == VCL_RET_DELIVER);
if (wrk->objcore == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
wrk->busyobj->exp.ttl = -1.;
} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
/* pass from vcl_fetch{} -> hit-for-pass */
/* XXX: the bereq was not filtered pass... */
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
wrk->busyobj->is_gzip =
http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
wrk->busyobj->is_gunzip =
!http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
wrk->busyobj->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (wrk->busyobj->do_gunzip)
http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
wrk->busyobj->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (wrk->busyobj->do_gzip)
http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
"Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (wrk->busyobj->do_esi)
wrk->busyobj->vfp = &vfp_esi;
else if (wrk->busyobj->do_gunzip)
wrk->busyobj->vfp = &vfp_gunzip;
else if (wrk->busyobj->do_gzip)
wrk->busyobj->vfp = &vfp_gzip;
else if (wrk->busyobj->is_gzip)
wrk->busyobj->vfp = &vfp_testgzip;
if (wrk->busyobj->do_esi || sp->esi_level > 0)
wrk->busyobj->do_stream = 0;
if (!sp->wantbody)
wrk->busyobj->do_stream = 0;
l = http_EstimateWS(wrk->busyobj->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
vary = VRY_Create(sp, wrk->busyobj->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
wrk->objcore == NULL)
wrk->storage_hint = TRANSIENT_STORAGE;
wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
if (wrk->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
if (wrk->busyobj->exp.ttl > cache_param->shortlived)
wrk->busyobj->exp.ttl = cache_param->shortlived;
wrk->busyobj->exp.grace = 0.0;
wrk->busyobj->exp.keep = 0.0;
}
if (wrk->obj == NULL) {
sp->err_code = 503;
sp->step = STP_ERROR;
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
wrk->storage_hint = NULL;
if (wrk->busyobj->do_gzip ||
(wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
wrk->obj->gziped = 1;
if (vary != NULL) {
wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
AN(wrk->obj->vary);
memcpy(wrk->obj->vary, VSB_data(vary), varyl);
VRY_Validate(wrk->obj->vary);
VSB_delete(vary);
}
wrk->obj->xid = sp->xid;
wrk->obj->response = sp->err_code;
WS_Assert(wrk->obj->ws_o);
/* Filter into object */
hp = wrk->busyobj->beresp;
hp2 = wrk->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(wrk, sp->vsl_id, hp2, hp,
pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(wrk, sp->vsl_id, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
wrk->obj->last_modified = VTIM_parse(b);
else
wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (wrk->obj->response == 200 &&
sp->http->conds &&
RFC2616_Do_Cond(sp))
wrk->busyobj->do_stream = 0;
AssertObjCorePassOrBusy(wrk->obj->objcore);
if (wrk->busyobj->do_stream) {
sp->step = STP_PREPRESP;
return (0);
}
/* Use unmodified headers*/
i = FetchBody(wrk, wrk->obj);
http_Setup(wrk->busyobj->bereq, NULL);
http_Setup(wrk->busyobj->beresp, NULL);
wrk->busyobj->vfp = NULL;
assert(WRW_IsReleased(wrk));
AZ(wrk->busyobj->vbc);
AN(sp->director);
if (i) {
HSH_Drop(wrk);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
AZ(wrk->obj);
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (wrk->obj->objcore != NULL) {
EXP_Insert(wrk->obj);
AN(wrk->obj->objcore);
AN(wrk->obj->objcore->ban);
HSH_Unbusy(wrk);
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
wrk->acct_tmp.fetch++;
sp->step = STP_PREPRESP;
return (0);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
char date[40];
struct http *h;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats->s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
VTIM_format(now, date);
http_PrintfHeader(h, "Date: %s", date);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
AZ(req->synth_body);
req->synth_body = VSB_new_auto();
AN(req->synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws);
http_Unset(h, H_Content_Length);
AZ(VSB_finish(req->synth_body));
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
/* Discard any lingering request body before delivery */
(void)VRB_Ignore(req);
V1D_Deliver_Synth(req);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->err_code = 0;
req->err_reason = NULL;
return (REQ_FSM_DONE);
}
static enum req_fsm_nxt
cnt_deliver(struct worker *wrk, struct req *req)
{
char time_str[30];
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(req->obj->objcore, OBJCORE_MAGIC);
CHECK_OBJ_NOTNULL(req->obj->objcore->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
assert(WRW_IsReleased(wrk));
assert(req->obj->objcore->refcnt > 0);
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->obj->objcore->exp_flags & OC_EF_EXP)
EXP_Touch(req->obj->objcore, now);
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
http_ClrHeader(req->resp);
http_FilterResp(req->obj->http, req->resp, 0);
http_Unset(req->resp, H_Date);
VTIM_format(now, time_str);
http_PrintfHeader(req->resp, "Date: %s", time_str);
if (req->wrk->stats.cache_hit)
http_PrintfHeader(req->resp,
"X-Varnish: %u %u", req->vsl->wid & VSL_IDENTMASK,
req->obj->vxid & VSL_IDENTMASK);
else
http_PrintfHeader(req->resp,
"X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
http_PrintfHeader(req->resp, "Age: %.0f",
now - req->obj->exp.t_origin);
http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
if (cache_param->http_gzip_support && req->obj->gziped &&
!RFC2616_Req_Gzip(req->http))
RFC2616_Weaken_Etag(req->resp);
VCL_deliver_method(req->vcl, wrk, req, NULL, req->http->ws);
/* Stop the insanity before it turns "Hotel California" on us */
if (req->restarts >= cache_param->max_restarts)
wrk->handling = VCL_RET_DELIVER;
if (wrk->handling == VCL_RET_RESTART) {
(void)HSH_DerefObj(&wrk->stats, &req->obj);
AZ(req->obj);
http_Teardown(req->resp);
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (!(req->obj->objcore->flags & OC_F_PASS)
&& req->esi_level == 0
&& http_GetStatus(req->obj->http) == 200
&& req->http->conds && RFC2616_Do_Cond(req))
http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
V1D_Deliver(req);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
if (req->obj->objcore->flags & OC_F_PASS) {
/*
* No point in saving the body if it is hit-for-pass,
* but we can't yank it until the fetching thread has
* finished/abandoned also.
*/
while (req->obj->objcore->busyobj != NULL)
(void)usleep(100000);
STV_Freestore(req->obj);
}
assert(WRW_IsReleased(wrk));
VSLb(req->vsl, SLT_Debug, "XXX REF %d", req->obj->objcore->refcnt);
(void)HSH_DerefObj(&wrk->stats, &req->obj);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
struct http *h;
double now;
struct vsb *synth_body;
ssize_t sz, szl;
uint8_t *ptr;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats->s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
http_TimeHeader(h, "Date: ", now);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
synth_body = VSB_new_auto();
AN(synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, synth_body);
http_Unset(h, H_Content_Length);
AZ(VSB_finish(synth_body));
/* Discard any lingering request body before delivery */
(void)VRB_Ignore(req);
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_delete(synth_body);
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
req->objcore = HSH_Private(wrk);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
if (STV_NewObject(req->objcore, wrk, TRANSIENT_STORAGE, 1024)) {
szl = VSB_len(synth_body);
assert(szl >= 0);
} else
szl = -1;
if (szl > 0) {
sz = szl;
if (ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) {
memcpy(ptr, VSB_data(synth_body), szl);
ObjExtend(wrk, req->objcore, szl);
} else
szl = -1;
}
if (szl < 0) {
VSLb(req->vsl, SLT_Error, "Could not get storage");
req->doclose = SC_OVERLOAD;
} else
cnt_vdp(req, NULL);
(void)HSH_DerefObjCore(wrk, &req->objcore);
VSB_delete(synth_body);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
req->err_code = 0;
req->err_reason = NULL;
return (REQ_FSM_DONE);
}
static enum req_fsm_nxt
cnt_deliver(struct worker *wrk, struct req *req)
{
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(req->obj->objcore, OBJCORE_MAGIC);
CHECK_OBJ_NOTNULL(req->obj->objcore->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
assert(WRW_IsReleased(wrk));
assert(req->obj->objcore->refcnt > 0);
if (req->obj->objcore->exp_flags & OC_EF_EXP)
EXP_Touch(req->obj->objcore, req->t_prev);
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
http_FilterResp(req->obj->http, req->resp, 0);
http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
if (req->wrk->stats.cache_hit)
http_PrintfHeader(req->resp,
"X-Varnish: %u %u", VXID(req->vsl->wid),
VXID(req->obj->vxid));
else
http_PrintfHeader(req->resp,
"X-Varnish: %u", VXID(req->vsl->wid));
/* We base Age calculation upon the last timestamp taken during
client request processing. This gives some inaccuracy, but
since Age is only full second resolution that shouldn't
matter. (Last request timestamp could be a Start timestamp
taken before the object entered into cache leading to negative
age. Truncate to zero in that case).
*/
http_PrintfHeader(req->resp, "Age: %.0f",
fmax(0., req->t_prev - req->obj->objcore->exp.t_origin));
http_SetHeader(req->resp, "Via: 1.1 varnish-v4");
if (cache_param->http_gzip_support && req->obj->gziped &&
!RFC2616_Req_Gzip(req->http))
RFC2616_Weaken_Etag(req->resp);
VCL_deliver_method(req->vcl, wrk, req, NULL, req->http->ws);
VSLb_ts_req(req, "Process", W_TIM_real(wrk));
/* Stop the insanity before it turns "Hotel California" on us */
if (req->restarts >= cache_param->max_restarts)
wrk->handling = VCL_RET_DELIVER;
if (wrk->handling != VCL_RET_DELIVER) {
(void)HSH_DerefObj(&wrk->stats, &req->obj);
AZ(req->obj);
http_Teardown(req->resp);
switch (wrk->handling) {
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
break;
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
break;
default:
INCOMPL();
}
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (!(req->obj->objcore->flags & OC_F_PASS)
&& req->esi_level == 0
&& http_GetStatus(req->obj->http) == 200
&& req->http->conds && RFC2616_Do_Cond(req)) {
http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
req->wantbody = 0;
}
/* Grab a ref to the bo if there is one, and hand it down */
bo = HSH_RefBusy(req->obj->objcore);
V1D_Deliver(req, bo);
if (bo != NULL)
VBO_DerefBusyObj(req->wrk, &bo);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
if (req->obj->objcore->flags & OC_F_PASS) {
/*
* No point in saving the body if it is hit-for-pass,
* but we can't yank it until the fetching thread has
* finished/abandoned also.
*/
while (req->obj->objcore->busyobj != NULL)
(void)usleep(100000);
STV_Freestore(req->obj);
}
assert(WRW_IsReleased(wrk));
VSLb(req->vsl, SLT_Debug, "XXX REF %d", req->obj->objcore->refcnt);
(void)HSH_DerefObj(&wrk->stats, &req->obj);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
const char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
if (bo->htc->content_length == 0)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->htc->body_status != BS_NONE) {
bo->is_gzip =
http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip =
!http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
}
/* We won't gunzip unless it is non-empty and gzip'ed */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gunzip && !bo->is_gzip))
bo->do_gunzip = 0;
/* We wont gzip unless it is non-empty and ungziped */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gzip && !bo->is_gunzip))
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);
if (bo->htc->content_length != 0) {
if (bo->do_esi && bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
(void)VFP_Push(bo->vfc, &vfp_esi, 1);
} else if (bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
}
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc->body_status == BS_NONE)
bo->do_stream = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (bo->do_esi)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);
if (bo->do_gzip || bo->do_gunzip)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);
if (http_IsStatus(bo->beresp, 200) && (
http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
http_GetHdr(bo->beresp, H_ETag, &p)))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);
if (bo->htc->body_status != BS_NONE)
AZ(VDI_GetBody(bo->wrk, bo));
assert(bo->refcount >= 1);
assert(bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(wrk, bo->fetch_objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc->body_status, body_status_2str(bo->htc->body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc->body_status != BS_NONE) {
assert(bo->htc->body_status != BS_ERROR);
vbf_fetch_body_helper(bo);
}
if (bo->vfc->failed) {
VDI_Finish(bo->wrk, bo);
if (!bo->do_stream) {
assert(bo->state < BOS_STREAM);
// XXX: doclose = ?
return (F_STP_ERROR);
} else {
return (F_STP_FAIL);
}
}
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(wrk, bo->fetch_objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->stale_oc != NULL)
EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);
return (F_STP_DONE);
}
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(req->handling == VCL_RET_DELIVER);
if (req->objcore->objhead == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
bo->exp.ttl = -1.;
} else if (bo->do_pass) {
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->do_esi || req->esi_level > 0)
bo->do_stream = 0;
if (!req->wantbody)
bo->do_stream = 0;
/* No reason to try streaming a non-existing body */
if (bo->body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (req->objcore->objhead != NULL) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
vary = VRY_Create(req, bo->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (bo->exp.ttl < cache_param->shortlived ||
req->objcore == NULL)
req->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
nhttp);
if (req->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
l, nhttp);
}
bo->stats = NULL;
if (req->obj == NULL) {
req->err_code = 503;
req->req_step = R_STP_ERROR;
VDI_CloseFd(&bo->vbc);
VBO_DerefBusyObj(wrk, &req->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
req->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = req->obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
req->obj->gziped = 1;
if (vary != NULL) {
req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
VSB_data(vary), varyl);
AN(req->obj->vary);
VRY_Validate(req->obj->vary);
VSB_delete(vary);
}
req->obj->vxid = bo->vsl->wid;
req->obj->response = req->err_code;
WS_Assert(req->obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = req->obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
req->obj->last_modified = VTIM_parse(b);
else
req->obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (req->obj->response == 200 &&
req->http->conds &&
RFC2616_Do_Cond(req))
bo->do_stream = 0;
/*
* Ready to fetch the body
*/
bo->fetch_task.func = FetchBody;
bo->fetch_task.priv = bo;
assert(bo->refcount == 2); /* one for each thread */
if (req->obj->objcore->objhead != NULL) {
EXP_Insert(req->obj);
AN(req->obj->objcore->ban);
AZ(req->obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, req->obj->objcore);
}
if (!bo->do_stream ||
Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
FetchBody(wrk, bo);
if (req->obj->objcore->objhead != NULL)
HSH_Ref(req->obj->objcore);
if (bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &req->busyobj);
} else if (bo->state == BOS_FAILED) {
/* handle early failures */
HSH_Deref(&wrk->stats, NULL, &req->obj);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->err_code = 503;
req->req_step = R_STP_ERROR;
return (0);
}
assert(WRW_IsReleased(wrk));
req->req_step = R_STP_PREPRESP;
return (0);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
ssize_t est;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
AN(bo->vbc);
est = V1F_Setup_Fetch(bo);
if (est == 0) {
/*
* If the length is known to be zero, it's not gziped.
* A similar issue exists for chunked encoding but we
* don't handle that. See #1320.
*/
http_Unset(bo->beresp, H_Content_Encoding);
bo->is_gzip = 0;
bo->is_gunzip = 1;
}
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* Fix Content-Encoding, as appropriate */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
else if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc, &bo->acct);
return (F_STP_ERROR);
}
assert(WRW_IsReleased(wrk));
obj = bo->fetch_obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (bo->do_gzip || bo->do_gunzip)
obj->changed_gzip = 1;
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
assert (bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc.body_status, body_status_2str(bo->htc.body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc.body_status != BS_NONE) {
assert(bo->htc.body_status != BS_ERROR);
VFP_Fetch_Body(bo, est);
}
if (bo->failed && !bo->do_stream) {
assert(bo->state < BOS_STREAM);
if (bo->fetch_obj != NULL) {
oc_freeobj(bo->fetch_objcore);
bo->fetch_obj = NULL;
bo->stats->n_object--;
}
return (F_STP_ERROR);
}
if (bo->failed)
return (F_STP_FAIL);
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && !(bo->should_close)) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
return (F_STP_DONE);
}
enum body_status
RFC2616_Body(struct busyobj *bo, struct dstat *stats)
{
struct http *hp;
char *b;
hp = bo->beresp;
if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
bo->should_close = 1;
else if (http_HdrIs(hp, H_Connection, "close"))
bo->should_close = 1;
else
bo->should_close = 0;
if (!strcasecmp(http_GetReq(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
stats->fetch_head++;
return (BS_NONE);
}
if (hp->status <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
*/
stats->fetch_1xx++;
return (BS_NONE);
}
if (hp->status == 204) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC2616 10.2.5 p60]
*/
stats->fetch_204++;
return (BS_NONE);
}
if (hp->status == 304) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
stats->fetch_304++;
return (BS_NONE);
}
if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
stats->fetch_chunked++;
return (BS_CHUNKED);
}
if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
stats->fetch_bad++;
return (BS_ERROR);
}
if (http_GetHdr(hp, H_Content_Length, &bo->h_content_length)) {
stats->fetch_length++;
return (BS_LENGTH);
}
if (http_HdrIs(hp, H_Connection, "keep-alive")) {
/*
* Keep alive with neither TE=Chunked or C-Len is impossible.
* We assume a zero length body.
*/
stats->fetch_zero++;
return (BS_ZERO);
}
if (http_HdrIs(hp, H_Connection, "close")) {
/*
* In this case, it is safe to just read what comes.
*/
stats->fetch_close++;
return (BS_EOF);
}
if (hp->protover < 11) {
/*
* With no Connection header, assume EOF.
*/
stats->fetch_oldhttp++;
return (BS_EOF);
}
/*
* Fall back to EOF transfer.
*/
stats->fetch_eof++;
return (BS_EOF);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
char date[40];
struct http *h;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
req->acct_req.synth++;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
req->t_resp = VTIM_real();
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
http_ClrHeader(h);
http_PutProtocol(h, "HTTP/1.1");
http_PutStatus(h, req->err_code);
VTIM_format(req->t_resp, date);
http_PrintfHeader(h, "Date: %s", date);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp,
"X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
if (req->err_reason != NULL)
http_PutResponse(h, req->err_reason);
else
http_PutResponse(h, http_StatusMessage(req->err_code));
AZ(req->synth_body);
req->synth_body = VSB_new_auto();
AN(req->synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws);
http_Unset(h, H_Content_Length);
AZ(VSB_finish(req->synth_body));
if (wrk->handling == VCL_RET_RESTART) {
http_ClrHeader(h);
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
V1D_Deliver_Synth(req);
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->err_code = 0;
req->err_reason = NULL;
return (REQ_FSM_DONE);
}
