static void pan_htc(struct vsb *vsb, const struct http_conn *htc) { VSB_printf(vsb, "http_conn = %p {\n", htc); if (pan_already(vsb, htc)) return; VSB_indent(vsb, 2); VSB_printf(vsb, "fd = %d,\n", htc->fd); VSB_printf(vsb, "doclose = %s,\n", sess_close_2str(htc->doclose, 0)); VSB_printf(vsb, "ws = %p,\n", htc->ws); VSB_printf(vsb, "{rxbuf_b, rxbuf_e} = {%p, %p},\n", htc->rxbuf_b, htc->rxbuf_e); VSB_printf(vsb, "{pipeline_b, pipeline_e} = {%p, %p},\n", htc->pipeline_b, htc->pipeline_e); VSB_printf(vsb, "content_length = %jd,\n", (intmax_t)htc->content_length); VSB_printf(vsb, "body_status = %s,\n", body_status_2str(htc->body_status)); VSB_printf(vsb, "first_byte_timeout = %f,\n", htc->first_byte_timeout); VSB_printf(vsb, "between_bytes_timeout = %f,\n", htc->between_bytes_timeout); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); }
static void pan_busyobj(const struct busyobj *bo) { struct vfp_entry *vfe; VSB_printf(pan_vsp, " busyobj = %p {\n", bo); pan_ws(bo->ws, 4); VSB_printf(pan_vsp, " refcnt = %u\n", bo->refcount); VSB_printf(pan_vsp, " retries = %d\n", bo->retries); VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed); VSB_printf(pan_vsp, " state = %d\n", (int)bo->state); #define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, " is_" #l "\n"); #include "tbl/bo_flags.h" #undef BO_FLAG VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", bo->htc.body_status, body_status_2str(bo->htc.body_status)); if (!VTAILQ_EMPTY(&bo->vfc->vfp)) { VSB_printf(pan_vsp, " filters ="); VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list) VSB_printf(pan_vsp, " %s=%d", vfe->vfp->name, (int)vfe->closed); VSB_printf(pan_vsp, "\n"); } VSB_printf(pan_vsp, " },\n"); if (VALID_OBJ(bo->vbc, BACKEND_MAGIC)) pan_vbc(bo->vbc); if (bo->bereq->ws != NULL) pan_http("bereq", bo->bereq, 4); if (bo->beresp->ws != NULL) pan_http("beresp", bo->beresp, 4); pan_ws(bo->ws_o, 4); if (bo->fetch_objcore) pan_objcore("FETCH", bo->fetch_objcore); if (bo->fetch_obj) pan_object("FETCH", bo->fetch_obj); if (bo->ims_obj) pan_object("IMS", bo->ims_obj); VSB_printf(pan_vsp, " }\n"); }
static void pan_busyobj(const struct busyobj *bo) { VSB_printf(pan_vsp, " busyobj = %p {\n", bo); pan_ws(bo->ws, 4); if (bo->is_gzip) VSB_printf(pan_vsp, " is_gzip\n"); if (bo->is_gunzip) VSB_printf(pan_vsp, " is_gunzip\n"); if (bo->do_gzip) VSB_printf(pan_vsp, " do_gzip\n"); if (bo->do_gunzip) VSB_printf(pan_vsp, " do_gunzip\n"); if (bo->do_esi) VSB_printf(pan_vsp, " do_esi\n"); if (bo->do_stream) VSB_printf(pan_vsp, " do_stream\n"); if (bo->should_close) VSB_printf(pan_vsp, " should_close\n"); VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", bo->htc.body_status, body_status_2str(bo->htc.body_status)); VSB_printf(pan_vsp, " },\n"); if (VALID_OBJ(bo->vbc, BACKEND_MAGIC)) pan_vbc(bo->vbc); if (bo->bereq->ws != NULL) pan_http("bereq", bo->bereq, 4); if (bo->beresp->ws != NULL) pan_http("beresp", bo->beresp, 4); VSB_printf(pan_vsp, " }\n"); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); if (vbf_figure_out_vfp(bo)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); bo->fetch_objcore->boc->len_so_far = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->htc->doclose = SC_RX_BODY; VFP_Close(bo->vfc); VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (bo->do_esi) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (!(bo->fetch_objcore->flags & OC_F_PASS) && http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE && VDI_GetBody(bo->wrk, bo) != 0) { (void)VFP_Error(bo->vfc, "GetBody failed - workspace_backend overflow?"); VFP_Close(bo->vfc); bo->htc->doclose = SC_OVERLOAD; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } assert(bo->fetch_objcore->boc->refcount >= 1); assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); if (bo->do_stream) { ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); return (F_STP_FETCHBODY); } AZ(bo->vfc->failed); return (F_STP_FETCHEND); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; if (bo->htc->content_length == 0) http_Unset(bo->beresp, H_Content_Encoding); if (bo->htc->body_status != BS_NONE) { bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(bo->is_gzip == 0 || bo->is_gunzip == 0); } /* We won't gunzip unless it is non-empty and gzip'ed */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gunzip && !bo->is_gzip)) bo->do_gunzip = 0; /* We wont gzip unless it is non-empty and ungziped */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gzip && !bo->is_gunzip)) bo->do_gzip = 0; /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->htc->content_length != 0) { if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc->body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (bo->do_esi) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) AZ(VDI_GetBody(bo->wrk, bo)); assert(bo->refcount >= 1); assert(bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); vbf_fetch_body_helper(bo); } if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); if (!bo->do_stream) { assert(bo->state < BOS_STREAM); // XXX: doclose = ? return (F_STP_ERROR); } else { return (F_STP_FAIL); } } if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(wrk, bo->fetch_objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->stale_oc != NULL) EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct http *hp, *hp2; char *b; uint16_t nhttp; unsigned l; struct vsb *vary = NULL; int varyl = 0; struct object *obj; ssize_t est = -1; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* If we do gunzip, remove the C-E header */ if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; /* If we do gzip, add the C-E header */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->vbc != NULL) est = V1F_Setup_Fetch(bo); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) { RFC2616_Weaken_Etag(bo->beresp); VFP_Push(bo, vfp_gunzip_pull, 0); } if (bo->do_esi && bo->do_gzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi) { VFP_Push(bo, vfp_esi_pull, 0); } else if (bo->do_gzip) { VFP_Push(bo, vfp_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_testgunzip_pull, 0); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; l = 0; /* Create Vary instructions */ if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) { varyl = VRY_Create(bo, &vary); if (varyl > 0) { AN(vary); assert(varyl == VSB_len(vary)); l += varyl; } else if (varyl < 0) { /* * Vary parse error * Complain about it, and make this a pass. */ VSLb(bo->vsl, SLT_Error, "Illegal 'Vary' header from backend, " "making this a pass."); bo->uncacheable = 1; AZ(vary); } else /* No vary */ AZ(vary); } l += http_EstimateWS(bo->beresp, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); if (bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; if (bo->uncacheable || bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived) bo->storage_hint = TRANSIENT_STORAGE; AZ(bo->stats); bo->stats = &wrk->stats; AN(bo->fetch_objcore); obj = STV_NewObject(bo, bo->storage_hint, l, nhttp); if (obj == NULL) { /* * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ if (bo->exp.ttl > cache_param->shortlived) bo->exp.ttl = cache_param->shortlived; bo->exp.grace = 0.0; bo->exp.keep = 0.0; obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp); } if (obj == NULL) { bo->stats = NULL; (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc); return (F_STP_DONE); } CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); bo->storage_hint = NULL; AZ(bo->fetch_obj); bo->fetch_obj = obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (vary != NULL) { obj->vary = (void *)WS_Copy(obj->http->ws, VSB_data(vary), varyl); AN(obj->vary); (void)VRY_Validate(obj->vary); VSB_delete(vary); } obj->vxid = bo->vsl->wid; obj->response = bo->err_code; WS_Assert(bo->ws_o); /* Filter into object */ hp = bo->beresp; hp2 = obj->http; hp2->logtag = HTTP_Obj; http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS); http_CopyHome(hp2); if (http_GetHdr(hp, H_Last_Modified, &b)) obj->last_modified = VTIM_parse(b); else obj->last_modified = floor(bo->exp.t_origin); assert(WRW_IsReleased(wrk)); /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); if (bo->do_stream) HSH_Unbusy(&wrk->stats, obj->objcore); assert(bo->state == BOS_REQ_DONE); VBO_setstate(bo, BOS_FETCHING); switch (bo->htc.body_status) { case BS_NONE: break; case BS_ERROR: /* XXX: Why not earlier ? */ bo->should_close |= VFP_Error(bo, "error incompatible Transfer-Encoding"); break; default: if (bo->vbc == NULL) (void)VFP_Error(bo, "Backend connection gone"); else VFP_Fetch_Body(bo, est); } bo->stats = NULL; bo->t_body = VTIM_mono(); if (bo->vbc != NULL) { if (bo->should_close) VDI_CloseFd(&bo->vbc); else VDI_RecycleFd(&bo->vbc); AZ(bo->vbc); } http_Teardown(bo->bereq); http_Teardown(bo->beresp); VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)", bo->htc.body_status, body_status_2str(bo->htc.body_status)); if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; } else { assert(bo->state == BOS_FETCHING); VSLb(bo->vsl, SLT_Length, "%zd", obj->len); { /* Sanity check fetch methods accounting */ ssize_t uu; struct storage *st; uu = 0; VTAILQ_FOREACH(st, &obj->store, list) uu += st->len; if (bo->do_stream) /* Streaming might have started freeing stuff */ assert(uu <= obj->len); else assert(uu == obj->len); } } if (!bo->do_stream && bo->state != BOS_FAILED) HSH_Unbusy(&wrk->stats, obj->objcore); if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) { EXP_Insert(obj->objcore); AN(obj->objcore->ban); } HSH_Complete(obj->objcore); assert(bo->refcount >= 1); if (bo->state != BOS_FAILED) VBO_setstate(bo, BOS_FINISHED); VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; bo->vfc->body = obj->body; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; if (bo->htc.body_status != BS_NONE) V1F_Setup_Fetch(bo); /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo); bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { ObjFreeObj(bo->fetch_objcore, bo->stats); bo->fetch_obj = NULL; } return (F_STP_ERROR); } if (bo->vfc->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && bo->doclose == SC_NULL) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->ims_obj != NULL) EXP_Rearm(bo->ims_obj->objcore, bo->ims_obj->objcore->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; ssize_t est; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); est = V1F_Setup_Fetch(bo); if (est == 0) { /* * If the length is known to be zero, it's not gziped. * A similar issue exists for chunked encoding but we * don't handle that. See #1320. */ http_Unset(bo->beresp, H_Content_Encoding); bo->is_gzip = 0; bo->is_gunzip = 1; } /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); /* Fix Content-Encoding, as appropriate */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); else if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) { RFC2616_Weaken_Etag(bo->beresp); VFP_Push(bo, vfp_gunzip_pull, 0); } if (bo->do_esi && bo->do_gzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi) { VFP_Push(bo, vfp_esi_pull, 0); } else if (bo->do_gzip) { VFP_Push(bo, vfp_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_testgunzip_pull, 0); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc, &bo->acct); return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo, est); } if (bo->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { oc_freeobj(bo->fetch_objcore); bo->fetch_obj = NULL; bo->stats->n_object--; } return (F_STP_ERROR); } if (bo->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && !(bo->should_close)) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
void V1F_fetch_body(struct worker *wrk, struct busyobj *bo) { int cls; struct storage *st; ssize_t cl; struct http_conn *htc; struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); htc = &bo->htc; CHECK_OBJ_ORNULL(bo->vbc, VBC_MAGIC); obj = bo->fetch_obj; CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC); assert(bo->state == BOS_FETCHING); /* * XXX: The busyobj needs a dstat, but it is not obvious which one * XXX: it should be (own/borrowed). For now borrow the wrk's. */ AZ(bo->stats); bo->stats = &wrk->stats; AN(bo->vfp); AZ(bo->vgz_rx); assert(VTAILQ_EMPTY(&obj->store)); /* XXX: pick up estimate from objdr ? */ cl = 0; cls = bo->should_close; switch (htc->body_status) { case BS_NONE: break; case BS_ZERO: break; case BS_LENGTH: cl = vbf_fetch_number(bo->h_content_length, 10); bo->vfp->begin(bo, cl); if (bo->state == BOS_FETCHING && cl > 0) cls |= vbf_fetch_straight(bo, htc, cl); if (bo->vfp->end(bo)) assert(bo->state == BOS_FAILED); break; case BS_CHUNKED: bo->vfp->begin(bo, cl > 0 ? cl : 0); if (bo->state == BOS_FETCHING) cls |= vbf_fetch_chunked(bo, htc); if (bo->vfp->end(bo)) assert(bo->state == BOS_FAILED); break; case BS_EOF: bo->vfp->begin(bo, cl > 0 ? cl : 0); if (bo->state == BOS_FETCHING) vbf_fetch_eof(bo, htc); cls = 1; if (bo->vfp->end(bo)) assert(bo->state == BOS_FAILED); break; case BS_ERROR: cls |= VFP_Error(bo, "error incompatible Transfer-Encoding"); break; default: INCOMPL(); } bo->t_body = VTIM_mono(); AZ(bo->vgz_rx); /* * Trim or delete the last segment, if any */ st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead); /* XXX: Temporary: Only trim if we are not streaming */ if (st != NULL && !bo->do_stream) { /* XXX: is any of this safe under streaming ? */ if (st->len == 0) { VTAILQ_REMOVE(&bo->fetch_obj->store, st, list); STV_free(st); } else if (st->len < st->space) { STV_trim(st, st->len, 1); } } bo->vfp = NULL; VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d", htc->body_status, body_status_2str(htc->body_status), cls); http_Teardown(bo->bereq); http_Teardown(bo->beresp); if (bo->vbc != NULL) { if (cls) VDI_CloseFd(&bo->vbc); else VDI_RecycleFd(&bo->vbc); } AZ(bo->vbc); if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; } else { assert(bo->state == BOS_FETCHING); VSLb(bo->vsl, SLT_Length, "%zd", obj->len); { /* Sanity check fetch methods accounting */ ssize_t uu; uu = 0; VTAILQ_FOREACH(st, &obj->store, list) uu += st->len; if (bo->do_stream) /* Streaming might have started freeing stuff */ assert(uu <= obj->len); else assert(uu == obj->len); } } bo->stats = NULL; }