static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(vbf_beresp2obj(bo)); if (ObjHasAttr(bo->wrk, bo->stale_oc, OA_ESIDATA)) AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_ESIDATA)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS)); if (bo->do_stream) { ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0)) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->stale_oc->flags & OC_F_FAILED) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); wrk->stats->fetch_failed++; return (F_STP_FAIL); } return (F_STP_FETCHEND); }
static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(vbf_beresp2obj(bo)); if (ObjHasAttr(bo->wrk, bo->stale_oc, OA_ESIDATA)) AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_ESIDATA)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS)); if (bo->do_stream) { ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0)) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->stale_oc->flags & OC_F_FAILED) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); return (F_STP_FAIL); } AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, bo->fetch_objcore->boc->len_so_far)); if (!bo->do_stream) HSH_Unbusy(wrk, bo->fetch_objcore); HSH_Kill(bo->stale_oc); /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo->wrk, bo); ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
static enum fetch_step vbf_stp_error(struct worker *wrk, struct busyobj *bo) { ssize_t l, ll, o; double now; uint8_t *ptr; struct vsb *synth_body; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); AN(bo->fetch_objcore->flags & OC_F_BUSY); assert(bo->director_state == DIR_S_NULL); wrk->stats->fetch_failed++; now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); if (bo->fetch_objcore->stobj->stevedore != NULL) ObjFreeObj(bo->wrk, bo->fetch_objcore); if (bo->storage == NULL) bo->storage = STV_next(); // XXX: reset all beresp flags ? HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); http_TimeHeader(bo->beresp, "Date: ", now); http_SetHeader(bo->beresp, "Server: Varnish"); bo->fetch_objcore->t_origin = now; if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) { /* * If there is a waitinglist, it means that there is no * grace-able object, so cache the error return for a * short time, so the waiting list can drain, rather than * each objcore on the waiting list sequentially attempt * to fetch from the backend. */ bo->fetch_objcore->ttl = 1; bo->fetch_objcore->grace = 5; bo->fetch_objcore->keep = 5; } else { bo->fetch_objcore->ttl = 0; bo->fetch_objcore->grace = 0; bo->fetch_objcore->keep = 0; } synth_body = VSB_new_auto(); AN(synth_body); VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body); AZ(VSB_finish(synth_body)); if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) { VSB_destroy(&synth_body); return (F_STP_FAIL); } if (wrk->handling == VCL_RET_RETRY) { VSB_destroy(&synth_body); if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing"); return (F_STP_FAIL); } assert(wrk->handling == VCL_RET_DELIVER); assert(bo->vfc->wrk == bo->wrk); assert(bo->vfc->oc == bo->fetch_objcore); assert(bo->vfc->resp == bo->beresp); assert(bo->vfc->req == bo->bereq); if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); VSB_destroy(&synth_body); return (F_STP_FAIL); } ll = VSB_len(synth_body); o = 0; while (ll > 0) { l = ll; if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) break; if (l > ll) l = ll; memcpy(ptr, VSB_data(synth_body) + o, l); VFP_Extend(bo->vfc, l); ll -= l; o += l; } AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o)); VSB_destroy(&synth_body); HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); if (vbf_figure_out_vfp(bo)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); bo->fetch_objcore->boc->len_so_far = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->htc->doclose = SC_RX_BODY; VFP_Close(bo->vfc); VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (bo->do_esi) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (!(bo->fetch_objcore->flags & OC_F_PASS) && http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE && VDI_GetBody(bo->wrk, bo) != 0) { (void)VFP_Error(bo->vfc, "GetBody failed - workspace_backend overflow?"); VFP_Close(bo->vfc); bo->htc->doclose = SC_OVERLOAD; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } assert(bo->fetch_objcore->boc->refcount >= 1); assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); if (bo->do_stream) { ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); return (F_STP_FETCHBODY); } AZ(bo->vfc->failed); return (F_STP_FETCHEND); }
static enum fetch_step vbf_stp_error(struct worker *wrk, struct busyobj *bo) { ssize_t l, ll, o; double now; uint8_t *ptr; struct vsb *synth_body; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(bo->director_state == DIR_S_NULL); if(bo->fetch_objcore->stobj->stevedore != NULL) ObjFreeObj(bo->wrk, bo->fetch_objcore); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); AN(bo->fetch_objcore->flags & OC_F_BUSY); synth_body = VSB_new_auto(); AN(synth_body); // XXX: reset all beresp flags ? HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); http_TimeHeader(bo->beresp, "Date: ", now); http_SetHeader(bo->beresp, "Server: Varnish"); EXP_Clr(&bo->fetch_objcore->exp); bo->fetch_objcore->exp.t_origin = bo->t_prev; VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body); AZ(VSB_finish(synth_body)); if (wrk->handling == VCL_RET_RETRY || wrk->handling == VCL_RET_ABANDON) { VSB_delete(synth_body); if (bo->director_state != DIR_S_NULL) { bo->htc->doclose = SC_RESP_CLOSE; VDI_Finish(bo->wrk, bo); } if (wrk->handling == VCL_RET_RETRY) { if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, delivering 503"); } return (F_STP_FAIL); } assert(wrk->handling == VCL_RET_DELIVER); bo->vfc->bo = bo; bo->vfc->wrk = bo->wrk; bo->vfc->oc = bo->fetch_objcore; bo->vfc->http = bo->beresp; bo->vfc->esi_req = bo->bereq; if (vbf_beresp2obj(bo)) { VSB_delete(synth_body); return (F_STP_FAIL); } ll = VSB_len(synth_body); o = 0; while (ll > 0) { l = ll; if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) break; memcpy(ptr, VSB_data(synth_body) + o, l); VBO_extend(bo, l); ll -= l; o += l; } VSB_delete(synth_body); HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_FINISHED); return (F_STP_DONE); }
static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { void *oi; void *sp; ssize_t sl, al, l; uint8_t *ptr; enum objiter_status ois; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(vbf_beresp2obj(bo)); if (ObjGetattr(bo->wrk, bo->stale_oc, OA_ESIDATA, NULL) != NULL) AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_ESIDATA)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS)); AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS)); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } al = 0; oi = ObjIterBegin(wrk, bo->stale_oc); do { ois = ObjIter(bo->stale_oc, oi, &sp, &sl); if (ois == OIS_ERROR) (void)VFP_Error(bo->vfc, "Template object failed"); while (sl > 0) { l = ObjGetLen(bo->wrk, bo->stale_oc) - al; assert(l > 0); if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) break; if (sl < l) l = sl; memcpy(ptr, sp, l); VBO_extend(bo, l); al += l; sp = (char *)sp + l; sl -= l; } } while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM)); ObjIterEnd(bo->stale_oc, &oi); if (bo->stale_oc->flags & OC_F_FAILED) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); return (F_STP_FAIL); } if (!bo->do_stream) HSH_Unbusy(wrk, bo->fetch_objcore); assert(ObjGetLen(bo->wrk, bo->fetch_objcore) == al); EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0); /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; if (bo->htc->content_length == 0) http_Unset(bo->beresp, H_Content_Encoding); if (bo->htc->body_status != BS_NONE) { bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(bo->is_gzip == 0 || bo->is_gunzip == 0); } /* We won't gunzip unless it is non-empty and gzip'ed */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gunzip && !bo->is_gzip)) bo->do_gunzip = 0; /* We wont gzip unless it is non-empty and ungziped */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gzip && !bo->is_gunzip)) bo->do_gzip = 0; /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->htc->content_length != 0) { if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc->body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (bo->do_esi) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) AZ(VDI_GetBody(bo->wrk, bo)); assert(bo->refcount >= 1); assert(bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); vbf_fetch_body_helper(bo); } if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); if (!bo->do_stream) { assert(bo->state < BOS_STREAM); // XXX: doclose = ? return (F_STP_ERROR); } else { return (F_STP_FAIL); } } if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(wrk, bo->fetch_objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->stale_oc != NULL) EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }
static enum fetch_step vbf_stp_error(struct worker *wrk, struct busyobj *bo) { struct storage *st; ssize_t l; double now; char time_str[VTIM_FORMAT_SIZE]; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); AN(bo->fetch_objcore->flags & OC_F_BUSY); AZ(bo->synth_body); bo->synth_body = VSB_new_auto(); AN(bo->synth_body); // XXX: reset all beresp flags ? HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); VTIM_format(now, time_str); http_PrintfHeader(bo->beresp, "Date: %s", time_str); http_SetHeader(bo->beresp, "Server: Varnish"); bo->fetch_objcore->exp.t_origin = bo->t_prev; bo->fetch_objcore->exp.ttl = 0; bo->fetch_objcore->exp.grace = 0; bo->fetch_objcore->exp.keep = 0; VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws); AZ(VSB_finish(bo->synth_body)); if (wrk->handling == VCL_RET_RETRY) { VSB_delete(bo->synth_body); bo->synth_body = NULL; if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); bo->synth_body = NULL; return (F_STP_FAIL); } assert(wrk->handling == VCL_RET_DELIVER); VFP_Setup(bo->vfc); bo->vfc->bo = bo; bo->vfc->http = bo->beresp; bo->vfc->vsl = bo->vsl; if (vbf_beresp2obj(bo)) return (F_STP_FAIL); bo->vfc->body = bo->fetch_obj->body; l = VSB_len(bo->synth_body); if (l > 0) { st = VFP_GetStorage(bo->vfc, l); if (st != NULL) { if (st->space < l) { VSLb(bo->vsl, SLT_Error, "No space for %zd bytes of synth body", l); } else { memcpy(st->ptr, VSB_data(bo->synth_body), l); VBO_extend(bo, l); } } } VSB_delete(bo->synth_body); bo->synth_body = NULL; HSH_Unbusy(&wrk->stats, bo->fetch_objcore); VBO_setstate(bo, BOS_FINISHED); return (F_STP_DONE); }
static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; struct objiter *oi; void *sp; ssize_t sl, al, tl; struct storage *st; enum objiter_status ois; char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); if (bo->ims_obj->changed_gzip) { /* * If we modified the gzip status of the IMS object, that * must control the C-E header, if any. */ http_Unset(bo->beresp, H_Content_Encoding); if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p)) http_PrintfHeader(bo->beresp, "Content-Encoding: %s", p); } AZ(vbf_beresp2obj(bo)); obj = bo->fetch_obj; bo->vfc->body = obj->body; if (bo->ims_obj->esidata != NULL) { sl = bo->ims_obj->esidata->len; obj->esidata = STV_alloc(bo->vfc, sl); if (obj->esidata == NULL || obj->esidata->space < sl) { VSLb(bo->vsl, SLT_Error, "No space for %zd bytes of ESI data", sl); return (F_STP_FAIL); } memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl); obj->esidata->len = sl; } obj->gziped = bo->ims_obj->gziped; obj->gzip_start = bo->ims_obj->gzip_start; obj->gzip_last = bo->ims_obj->gzip_last; obj->gzip_stop = bo->ims_obj->gzip_stop; AZ(WS_Overflowed(bo->ws_o)); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } st = NULL; al = 0; oi = ObjIterBegin(wrk, bo->ims_obj); do { ois = ObjIter(oi, &sp, &sl); while (sl > 0) { if (st == NULL) st = VFP_GetStorage(bo->vfc, bo->ims_obj->len - al); if (st == NULL) break; tl = sl; if (tl > st->space - st->len) tl = st->space - st->len; memcpy(st->ptr + st->len, sp, tl); al += tl; sp = (char *)sp + tl; sl -= tl; VBO_extend(bo, tl); if (st->len == st->space) st = NULL; } } while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM)); ObjIterEnd(&oi); if (bo->vfc->failed) return (F_STP_FAIL); if (!bo->do_stream) HSH_Unbusy(&wrk->stats, obj->objcore); assert(al == bo->ims_obj->len); assert(obj->len == al); EXP_Rearm(bo->ims_obj->objcore, bo->ims_obj->objcore->exp.t_origin, 0, 0, 0); /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && bo->doclose == SC_NULL) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->doclose = SC_RX_BODY; return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; bo->vfc->body = obj->body; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; if (bo->htc.body_status != BS_NONE) V1F_Setup_Fetch(bo); /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo); bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { ObjFreeObj(bo->fetch_objcore, bo->stats); bo->fetch_obj = NULL; } return (F_STP_ERROR); } if (bo->vfc->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && bo->doclose == SC_NULL) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->ims_obj != NULL) EXP_Rearm(bo->ims_obj->objcore, bo->ims_obj->objcore->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct object *obj; ssize_t est; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; AN(bo->vbc); est = V1F_Setup_Fetch(bo); if (est == 0) { /* * If the length is known to be zero, it's not gziped. * A similar issue exists for chunked encoding but we * don't handle that. See #1320. */ http_Unset(bo->beresp, H_Content_Encoding); bo->is_gzip = 0; bo->is_gunzip = 1; } /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); /* Fix Content-Encoding, as appropriate */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); else if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) { RFC2616_Weaken_Etag(bo->beresp); VFP_Push(bo, vfp_gunzip_pull, 0); } if (bo->do_esi && bo->do_gzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_esi_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->do_esi) { VFP_Push(bo, vfp_esi_pull, 0); } else if (bo->do_gzip) { VFP_Push(bo, vfp_gzip_pull, 0); RFC2616_Weaken_Etag(bo->beresp); } else if (bo->is_gzip && !bo->do_gunzip) { VFP_Push(bo, vfp_testgunzip_pull, 0); } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo, "Could not get storage"); VDI_CloseFd(&bo->vbc, &bo->acct); return (F_STP_ERROR); } assert(WRW_IsReleased(wrk)); obj = bo->fetch_obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (bo->do_gzip || bo->do_gunzip) obj->changed_gzip = 1; /* * Ready to fetch the body */ assert(bo->refcount >= 1); AZ(WS_Overflowed(bo->ws_o)); assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, obj->objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc.body_status, body_status_2str(bo->htc.body_status), bo->do_stream ? "stream" : "-"); if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); VFP_Fetch_Body(bo, est); } if (bo->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_obj != NULL) { oc_freeobj(bo->fetch_objcore); bo->fetch_obj = NULL; bo->stats->n_object--; } return (F_STP_ERROR); } if (bo->failed) return (F_STP_FAIL); if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(&wrk->stats, obj->objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ if (bo->vbc != NULL && !(bo->should_close)) { VDI_RecycleFd(&bo->vbc, &bo->acct); AZ(bo->vbc); } VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); return (F_STP_DONE); }
static enum fetch_step vbf_stp_error(struct worker *wrk, struct busyobj *bo) { ssize_t l, ll, o; double now; uint8_t *ptr; char time_str[VTIM_FORMAT_SIZE]; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(bo->director_state == DIR_S_NULL); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); AN(bo->fetch_objcore->flags & OC_F_BUSY); AZ(bo->synth_body); bo->synth_body = VSB_new_auto(); AN(bo->synth_body); // XXX: reset all beresp flags ? HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); VTIM_format(now, time_str); http_PrintfHeader(bo->beresp, "Date: %s", time_str); http_SetHeader(bo->beresp, "Server: Varnish"); bo->fetch_objcore->exp.t_origin = bo->t_prev; bo->fetch_objcore->exp.ttl = 0; bo->fetch_objcore->exp.grace = 0; bo->fetch_objcore->exp.keep = 0; VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws); AZ(VSB_finish(bo->synth_body)); if (wrk->handling == VCL_RET_RETRY) { VSB_delete(bo->synth_body); bo->synth_body = NULL; if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); return (F_STP_FAIL); } assert(wrk->handling == VCL_RET_DELIVER); VFP_Setup(bo->vfc); bo->vfc->bo = bo; bo->vfc->wrk = bo->wrk; bo->vfc->oc = bo->fetch_objcore; bo->vfc->http = bo->beresp; bo->vfc->esi_req = bo->bereq; if (vbf_beresp2obj(bo)) return (F_STP_FAIL); ll = VSB_len(bo->synth_body); o = 0; while (ll > 0) { l = ll; if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) break; memcpy(ptr, VSB_data(bo->synth_body) + o, l); VBO_extend(bo, l); ll -= l; o += l; } VSB_delete(bo->synth_body); bo->synth_body = NULL; HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_FINISHED); return (F_STP_DONE); }