static void pan_ws(const struct ws *ws, int indent) { VSB_printf(pan_vsp, "%*sws = %p {", indent, "", ws); if (!VALID_OBJ(ws, WS_MAGIC)) { if (ws != NULL) VSB_printf(pan_vsp, " BAD_MAGIC(0x%08x) ", ws->magic); } else { if (WS_Overflowed(ws)) VSB_printf(pan_vsp, " OVERFLOW"); VSB_printf(pan_vsp, "\n%*sid = \"%s\",\n", indent + 2, "", ws->id); VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s); if (ws->f > ws->s) VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s)); else VSB_printf(pan_vsp, ",%p", ws->f); if (ws->r > ws->s) VSB_printf(pan_vsp, ",+%ld", (long) (ws->r - ws->s)); else VSB_printf(pan_vsp, ",%p", ws->r); if (ws->e > ws->s) VSB_printf(pan_vsp, ",+%ld", (long) (ws->e - ws->s)); else VSB_printf(pan_vsp, ",%p", ws->e); } VSB_printf(pan_vsp, "},\n"); VSB_printf(pan_vsp, "%*s},\n", indent, "" ); }
static void pan_ws(struct vsb *vsb, const struct ws *ws) { VSB_printf(vsb, "ws = %p {\n", ws); VSB_indent(vsb, 2); if (WS_Overflowed(ws)) VSB_printf(vsb, "OVERFLOW "); VSB_printf(vsb, "id = \"%s\",\n", ws->id); VSB_printf(vsb, "{s,f,r,e} = {%p", ws->s); if (ws->f > ws->s) VSB_printf(vsb, ",+%ld", (long) (ws->f - ws->s)); else VSB_printf(vsb, ",%p", ws->f); if (ws->r > ws->s) VSB_printf(vsb, ",+%ld", (long) (ws->r - ws->s)); else VSB_printf(vsb, ",%p", ws->r); if (ws->e > ws->s) VSB_printf(vsb, ",+%ld", (long) (ws->e - ws->s)); else VSB_printf(vsb, ",%p", ws->e); VSB_printf(vsb, "},\n"); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); }
void SES_Delete(struct sess *sp, enum sess_close reason, vtim_real now) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); if (reason != SC_NULL) SES_Close(sp, reason); assert(sp->fd < 0); if (isnan(now)) now = VTIM_real(); AZ(isnan(sp->t_open)); if (now < sp->t_open) { VSL(SLT_Debug, sp->vxid, "Clock step (now=%f < t_open=%f)", now, sp->t_open); if (now + cache_param->clock_step < sp->t_open) WRONG("Clock step detected"); now = sp->t_open; /* Do not log negatives */ } if (reason == SC_NULL) reason = (enum sess_close)-sp->fd; VSL(SLT_SessClose, sp->vxid, "%s %.3f", sess_close_2str(reason, 0), now - sp->t_open); VSL(SLT_End, sp->vxid, "%s", ""); if (WS_Overflowed(sp->ws)) VSC_C_main->ws_session_overflow++; SES_Rel(sp); }
V1D_Deliver(struct req *req, struct boc *boc, int sendbody) { int err = 0; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_ORNULL(boc, BOC_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); if (sendbody) { if (http_GetHdr(req->resp, H_Content_Length, NULL)) req->res_mode |= RES_LEN; else if (req->http->protover == 11) { req->res_mode |= RES_CHUNKED; http_SetHeader(req->resp, "Transfer-Encoding: chunked"); } else { req->res_mode |= RES_EOF; req->doclose = SC_TX_EOF; } } VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode); if (req->doclose) { if (!http_HdrIs(req->resp, H_Connection, "close")) { http_Unset(req->resp, H_Connection); http_SetHeader(req->resp, "Connection: close"); } } else if (!http_GetHdr(req->resp, H_Connection, NULL)) http_SetHeader(req->resp, "Connection: keep-alive"); if (sendbody && req->resp_len != 0) VDP_push(req, v1d_bytes, NULL, 1); AZ(req->wrk->v1l); V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev); if (WS_Overflowed(req->ws)) { v1d_error(req, "workspace_client overflow"); AZ(req->wrk->v1l); return; } req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp); if (DO_DEBUG(DBG_FLUSH_HEAD)) (void)V1L_Flush(req->wrk); if (sendbody && req->resp_len != 0) { if (req->res_mode & RES_CHUNKED) V1L_Chunked(req->wrk); err = VDP_DeliverObj(req); if (!err && (req->res_mode & RES_CHUNKED)) V1L_EndChunk(req->wrk); } if ((V1L_FlushRelease(req->wrk) || err) && req->sp->fd >= 0) SES_Close(req->sp, SC_REM_CLOSE); AZ(req->wrk->v1l); VDP_close(req); }
void Req_Rollback(struct req *req) { VCL_TaskLeave(req->vcl, req->privs); VCL_TaskEnter(req->vcl, req->privs); HTTP_Clone(req->http, req->http0); if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; WS_Reset(req->ws, req->ws_req); }
VCL_BOOL vmod_workspace_overflowed(VRT_CTX, VCL_ENUM which) { struct ws *ws; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); ws = wsfind(ctx, which); WS_Assert(ws); return (WS_Overflowed(ws)); }
void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->topreq, REQ_MAGIC); assert(sp == req->sp); AZ(req->vcl0); req->director_hint = NULL; req->restarts = 0; AZ(req->privs->magic); if (req->vcl != NULL) VCL_Recache(wrk, &req->vcl); /* Charge and log byte counters */ req_AcctLogCharge(wrk->stats, req); if (req->vsl->wid) VSL_End(req->vsl); if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle) sp->t_idle = req->t_prev; else sp->t_idle = W_TIM_real(wrk); req->t_first = NAN; req->t_prev = NAN; req->t_req = NAN; req->req_body_status = REQ_BODY_INIT; req->hash_always_miss = 0; req->hash_ignore_busy = 0; req->esi_level = 0; req->is_hit = 0; if (WS_Overflowed(req->ws)) wrk->stats->ws_client_overflow++; WS_Reset(req->ws, 0); }
void V1L_Reserve(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, double t0) { struct v1l *v1l; unsigned u; void *res; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AZ(wrk->v1l); if (WS_Overflowed(ws)) return; res = WS_Snapshot(ws); v1l = WS_Alloc(ws, sizeof *v1l); if (v1l == NULL) return; INIT_OBJ(v1l, V1L_MAGIC); v1l->ws = ws; v1l->res = res; u = WS_Reserve(ws, 0); u = PRNDDN(u); u /= sizeof(struct iovec); if (u == 0) { WS_Release(ws, 0); WS_MarkOverflow(ws); return; } else if (u > IOV_MAX) u = IOV_MAX; v1l->iov = (void*)PRNDUP(ws->f); v1l->siov = u; v1l->ciov = u; v1l->werr = 0; v1l->liov = 0; v1l->niov = 0; v1l->wfd = fd; v1l->t0 = t0; v1l->vsl = vsl; wrk->v1l = v1l; }
static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { unsigned l; uint16_t nhttp; struct object *obj; struct objiter *oi; void *sp; ssize_t sl, al, tl, vl; struct storage *st; enum objiter_status ois; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); l = 0; if (bo->ims_obj->vary != NULL) { vl = VRY_Validate(bo->ims_obj->vary); l += vl; } else vl = 0; l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp); bo->stats = &wrk->stats; obj = STV_NewObject(bo, bo->storage_hint, l, nhttp); if (obj == NULL) { (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc); return (F_STP_DONE); } AZ(bo->fetch_obj); bo->fetch_obj = obj; obj->gziped = bo->ims_obj->gziped; obj->gzip_start = bo->ims_obj->gzip_start; obj->gzip_last = bo->ims_obj->gzip_last; obj->gzip_stop = bo->ims_obj->gzip_stop; /* XXX: ESI */ if (bo->ims_obj->vary != NULL) { obj->vary = (void *)WS_Copy(obj->http->ws, bo->ims_obj->vary, vl); assert(vl == VRY_Validate(obj->vary)); } obj->vxid = bo->vsl->wid; obj->http->logtag = HTTP_Obj; /* XXX: we should have our own HTTP_A_CONDFETCH */ http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS); http_CopyHome(obj->http); AZ(WS_Overflowed(bo->ws_o)); VBO_setstate(bo, BOS_FETCHING); HSH_Unbusy(&wrk->stats, obj->objcore); if (!(obj->objcore->flags & OC_F_PRIVATE)) { EXP_Insert(obj->objcore); AN(obj->objcore->ban); } st = NULL; al = 0; oi = ObjIterBegin(wrk, bo->ims_obj); do { ois = ObjIter(oi, &sp, &sl); while (sl > 0) { if (st == NULL) { st = VFP_GetStorage(bo, bo->ims_obj->len - al); XXXAN(st); } tl = sl; if (tl > st->space - st->len) tl = st->space - st->len; memcpy(st->ptr + st->len, sp, tl); al += tl; sp = (char *)sp + tl; sl -= tl; VBO_extend(bo, al); if (st->len == st->space) st = NULL; } } while (ois == OIS_DATA || ois == OIS_STREAM); ObjIterEnd(&oi); bo->stats = NULL; assert(al == bo->ims_obj->len); assert(obj->len == al); if (bo->state != BOS_FAILED) VBO_setstate(bo, BOS_FINISHED); HSH_Complete(obj->objcore); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct http *hp, *hp2; char *b; uint16_t nhttp; unsigned l; struct vsb *vary = NULL; int varyl = 0; struct object *obj; ssize_t est = -1; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* If we do gunzip, remove the C-E header */ if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; /* If we do gzip, add the C-E header */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->vbc != NULL) est = V1F_Setup_Fetch(bo); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) { RFC2616_Weaken_Etag(bo->beresp); VFP_Push(bo, vfp_gunzip_pull, 0); } if (bo->do_esi && bo->do_gzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi) { VFP_Push(bo, vfp_esi_pull, 0); } else if (bo->do_gzip) { VFP_Push(bo, vfp_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_testgunzip_pull, 0); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; l = 0; /* Create Vary instructions */ if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) { varyl = VRY_Create(bo, &vary); if (varyl > 0) { AN(vary); assert(varyl == VSB_len(vary)); l += varyl; } else if (varyl < 0) { /* * Vary parse error * Complain about it, and make this a pass. */ VSLb(bo->vsl, SLT_Error, "Illegal 'Vary' header from backend, " "making this a pass."); bo->uncacheable = 1; AZ(vary); } else /* No vary */ AZ(vary); } l += http_EstimateWS(bo->beresp, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); if (bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; if (bo->uncacheable || bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived) bo->storage_hint = TRANSIENT_STORAGE; AZ(bo->stats); bo->stats = &wrk->stats; AN(bo->fetch_objcore); obj = STV_NewObject(bo, bo->storage_hint, l, nhttp); if (obj == NULL) { /* * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ if (bo->exp.ttl > cache_param->shortlived) bo->exp.ttl = cache_param->shortlived; bo->exp.grace = 0.0; bo->exp.keep = 0.0; obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp); } if (obj == NULL) { bo->stats = NULL; (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc); return (F_STP_DONE); } CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); bo->storage_hint = NULL; AZ(bo->fetch_obj); bo->fetch_obj = obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (vary != NULL) { obj->vary = (void *)WS_Copy(obj->http->ws, VSB_data(vary), varyl); AN(obj->vary); (void)VRY_Validate(obj->vary); VSB_delete(vary); } obj->vxid = bo->vsl->wid; obj->response = bo->err_code; WS_Assert(bo->ws_o); /* Filter into object */ hp = bo->beresp; hp2 = obj->http; hp2->logtag = HTTP_Obj; http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS); http_CopyHome(hp2); if (http_GetHdr(hp, H_Last_Modified, &b)) obj->last_modified = VTIM_parse(b); else obj->last_modified = floor(bo->exp.t_origin); assert(WRW_IsReleased(wrk)); /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); if (bo->do_stream) HSH_Unbusy(&wrk->stats, obj->objcore); assert(bo->state == BOS_REQ_DONE); VBO_setstate(bo, BOS_FETCHING); switch (bo->htc.body_status) { case BS_NONE: break; case BS_ERROR: /* XXX: Why not earlier ? */ bo->should_close |= VFP_Error(bo, "error incompatible Transfer-Encoding"); break; default: if (bo->vbc == NULL) (void)VFP_Error(bo, "Backend connection gone"); else VFP_Fetch_Body(bo, est); } bo->stats = NULL; bo->t_body = VTIM_mono(); if (bo->vbc != NULL) { if (bo->should_close) VDI_CloseFd(&bo->vbc); else VDI_RecycleFd(&bo->vbc); AZ(bo->vbc); } http_Teardown(bo->bereq); http_Teardown(bo->beresp); VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)", bo->htc.body_status, body_status_2str(bo->htc.body_status)); if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; } else { assert(bo->state == BOS_FETCHING); VSLb(bo->vsl, SLT_Length, "%zd", obj->len); { /* Sanity check fetch methods accounting */ ssize_t uu; struct storage *st; uu = 0; VTAILQ_FOREACH(st, &obj->store, list) uu += st->len; if (bo->do_stream) /* Streaming might have started freeing stuff */ assert(uu <= obj->len); else assert(uu == obj->len); } } if (!bo->do_stream && bo->state != BOS_FAILED) HSH_Unbusy(&wrk->stats, obj->objcore); if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) { EXP_Insert(obj->objcore); AN(obj->objcore->ban); } HSH_Complete(obj->objcore); assert(bo->refcount >= 1); if (bo->state != BOS_FAILED) VBO_setstate(bo, BOS_FINISHED); VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt); return (F_STP_DONE); }
static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; struct objiter *oi; void *sp; ssize_t sl, al, tl; struct storage *st; enum objiter_status ois; char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); if (bo->ims_obj->changed_gzip) { /* * If we modified the gzip status of the IMS object, that * must control the C-E header, if any. */ http_Unset(bo->beresp, H_Content_Encoding); if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p)) http_PrintfHeader(bo->beresp, "Content-Encoding: %s", p); } AZ(vbf_beresp2obj(bo)); obj = bo->fetch_obj; bo->vfc->body = obj->body; if (bo->ims_obj->esidata != NULL) { sl = bo->ims_obj->esidata->len; obj->esidata = STV_alloc(bo->vfc, sl); if (obj->esidata == NULL || obj->esidata->space < sl) { VSLb(bo->vsl, SLT_Error, "No space for %zd bytes of ESI data", sl); return (F_STP_FAIL); } memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl); obj->esidata->len = sl; } obj->gziped = bo->ims_obj->gziped; obj->gzip_start = bo->ims_obj->gzip_start; obj->gzip_last = bo->ims_obj->gzip_last; obj->gzip_stop = bo->ims_obj->gzip_stop; AZ(WS_Overflowed(bo->ws_o)); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } st = NULL; al = 0; oi = ObjIterBegin(wrk, bo->ims_obj); do { ois = ObjIter(oi, &sp, &sl); while (sl > 0) { if (st == NULL) st = VFP_GetStorage(bo->vfc, bo->ims_obj->len - al); if (st == NULL) break; tl = sl; if (tl > st->space - st->len) tl = st->space - st->len; memcpy(st->ptr + st->len, sp, tl); al += tl; sp = (char *)sp + tl; sl -= tl; VBO_extend(bo, tl); if (st->len == st->space) st = NULL; } } while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM)); ObjIterEnd(&oi); if (bo->vfc->failed) return (F_STP_FAIL); if (!bo->do_stream) HSH_Unbusy(&wrk->stats, obj->objcore); assert(al == bo->ims_obj->len); assert(obj->len == al); EXP_Rearm(bo->ims_obj->objcore, bo->ims_obj->objcore->exp.t_origin, 0, 0, 0); /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && bo->doclose == SC_NULL) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; bo->vfc->body = obj->body; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; if (bo->htc.body_status != BS_NONE) V1F_Setup_Fetch(bo); /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo); bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { ObjFreeObj(bo->fetch_objcore, bo->stats); bo->fetch_obj = NULL; } return (F_STP_ERROR); } if (bo->vfc->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && bo->doclose == SC_NULL) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->ims_obj != NULL) EXP_Rearm(bo->ims_obj->objcore, bo->ims_obj->objcore->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; ssize_t est; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); est = V1F_Setup_Fetch(bo); if (est == 0) { /* * If the length is known to be zero, it's not gziped. * A similar issue exists for chunked encoding but we * don't handle that. See #1320. */ http_Unset(bo->beresp, H_Content_Encoding); bo->is_gzip = 0; bo->is_gunzip = 1; } /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); /* Fix Content-Encoding, as appropriate */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); else if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) { RFC2616_Weaken_Etag(bo->beresp); VFP_Push(bo, vfp_gunzip_pull, 0); } if (bo->do_esi && bo->do_gzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi) { VFP_Push(bo, vfp_esi_pull, 0); } else if (bo->do_gzip) { VFP_Push(bo, vfp_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_testgunzip_pull, 0); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc, &bo->acct); return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo, est); } if (bo->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { oc_freeobj(bo->fetch_objcore); bo->fetch_obj = NULL; bo->stats->n_object--; } return (F_STP_ERROR); } if (bo->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && !(bo->should_close)) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
void http_CollectHdr(struct http *hp, const char *hdr) { unsigned u, l, ml, f, x, d; char *b = NULL, *e = NULL; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (WS_Overflowed(hp->ws)) return; l = hdr[0]; assert(l == strlen(hdr + 1)); assert(hdr[l] == ':'); f = http_findhdr(hp, l - 1, hdr + 1); if (f == 0) return; for (d = u = f + 1; u < hp->nhd; u++) { Tcheck(hp->hd[u]); if (!http_IsHdr(&hp->hd[u], hdr)) { if (d != u) { hp->hd[d] = hp->hd[u]; hp->hdf[d] = hp->hdf[u]; } d++; continue; } if (b == NULL) { /* Found second header, start our collection */ ml = WS_Reserve(hp->ws, 0); b = hp->ws->f; e = b + ml; x = Tlen(hp->hd[f]); if (b + x >= e) { http_fail(hp); VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1); WS_Release(hp->ws, 0); return; } memcpy(b, hp->hd[f].b, x); b += x; } AN(b); AN(e); /* Append the Nth header we found */ if (b < e) *b++ = ','; x = Tlen(hp->hd[u]) - l; if (b + x >= e) { http_fail(hp); VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1); WS_Release(hp->ws, 0); return; } memcpy(b, hp->hd[u].b + *hdr, x); b += x; } if (b == NULL) return; hp->nhd = (uint16_t)d; AN(e); *b = '\0'; hp->hd[f].b = hp->ws->f; hp->hd[f].e = b; WS_ReleaseP(hp->ws, b + 1); }