int FetchBody(struct worker *wrk, struct object *obj) { int cls; struct storage *st; int mklen; ssize_t cl; struct http_conn *htc; struct busyobj *bo; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); bo = wrk->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(bo->fetch_obj); CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC); htc = &bo->htc; if (bo->vfp == NULL) bo->vfp = &vfp_nop; AssertObjCorePassOrBusy(obj->objcore); AZ(bo->vgz_rx); AZ(VTAILQ_FIRST(&obj->store)); bo->fetch_obj = obj; bo->fetch_failed = 0; /* XXX: pick up estimate from objdr ? */ cl = 0; switch (bo->body_status) { case BS_NONE: cls = 0; mklen = 0; break; case BS_ZERO: cls = 0; mklen = 1; break; case BS_LENGTH: cl = fetch_number(bo->h_content_length, 10); bo->vfp->begin(wrk, cl > 0 ? cl : 0); cls = fetch_straight(wrk, htc, cl); mklen = 1; if (bo->vfp->end(wrk)) cls = -1; break; case BS_CHUNKED: bo->vfp->begin(wrk, cl); cls = fetch_chunked(wrk, htc); mklen = 1; if (bo->vfp->end(wrk)) cls = -1; break; case BS_EOF: bo->vfp->begin(wrk, cl); cls = fetch_eof(wrk, htc); mklen = 1; if (bo->vfp->end(wrk)) cls = -1; break; case BS_ERROR: cls = 1; mklen = 0; break; default: cls = 0; mklen = 0; INCOMPL(); } AZ(bo->vgz_rx); /* * It is OK for ->end to just leave the last storage segment * sitting on wrk->storage, we will always call vfp_nop_end() * to get it trimmed or thrown out if empty. */ AZ(vfp_nop_end(wrk)); bo->fetch_obj = NULL; WSLB(wrk, SLT_Fetch_Body, "%u(%s) cls %d mklen %d", bo->body_status, body_status(bo->body_status), cls, mklen); if (bo->body_status == BS_ERROR) { VDI_CloseFd(wrk, &bo->vbc); return (__LINE__); } if (cls < 0) { wrk->stats.fetch_failed++; /* XXX: Wouldn't this store automatically be released ? */ while (!VTAILQ_EMPTY(&obj->store)) { st = VTAILQ_FIRST(&obj->store); VTAILQ_REMOVE(&obj->store, st, list); STV_free(st); } VDI_CloseFd(wrk, &bo->vbc); obj->len = 0; return (__LINE__); } AZ(bo->fetch_failed); if (cls == 0 && bo->should_close) cls = 1; WSLB(wrk, SLT_Length, "%zd", obj->len); { /* Sanity check fetch methods accounting */ ssize_t uu; uu = 0; VTAILQ_FOREACH(st, &obj->store, list) uu += st->len; if (bo->do_stream) /* Streaming might have started freeing stuff */ assert (uu <= obj->len); else assert(uu == obj->len); } if (mklen > 0) { http_Unset(obj->http, H_Content_Length); http_PrintfHeader(wrk, bo->vbc->vsl_id, obj->http, "Content-Length: %zd", obj->len); } if (cls) VDI_CloseFd(wrk, &bo->vbc); else VDI_RecycleFd(wrk, &bo->vbc); return (0); }
static int cnt_streambody(struct sess *sp) { int i; struct stream_ctx sctx; uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ? cache_param->gzip_stack_buffer : 1]; struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); memset(&sctx, 0, sizeof sctx); sctx.magic = STREAM_CTX_MAGIC; AZ(wrk->sctx); wrk->sctx = &sctx; if (wrk->res_mode & RES_GUNZIP) { sctx.vgz = VGZ_NewUngzip(wrk, "U S -"); sctx.obuf = obuf; sctx.obuf_len = sizeof (obuf); } RES_StreamStart(sp); AssertObjCorePassOrBusy(wrk->obj->objcore); i = FetchBody(wrk, wrk->obj); http_Setup(wrk->busyobj->bereq, NULL); http_Setup(wrk->busyobj->beresp, NULL); wrk->busyobj->vfp = NULL; AZ(wrk->busyobj->vbc); AN(sp->director); if (!i && wrk->obj->objcore != NULL) { EXP_Insert(wrk->obj); AN(wrk->obj->objcore); AN(wrk->obj->objcore->ban); HSH_Unbusy(wrk); } else { sp->doclose = "Stream error"; } wrk->acct_tmp.fetch++; sp->director = NULL; sp->restarts = 0; RES_StreamEnd(sp); if (wrk->res_mode & RES_GUNZIP) (void)VGZ_Destroy(&sctx.vgz, sp->vsl_id); wrk->sctx = NULL; assert(WRW_IsReleased(wrk)); assert(wrk->wrw.ciov == wrk->wrw.siov); (void)HSH_Deref(wrk, NULL, &wrk->obj); VBO_DerefBusyObj(wrk, &wrk->busyobj); http_Setup(wrk->resp, NULL); sp->step = STP_DONE; return (0); }
static int cnt_fetchbody(struct sess *sp) { int i; struct http *hp, *hp2; char *b; uint16_t nhttp; unsigned l; struct vsb *vary = NULL; int varyl = 0, pass; struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); assert(sp->handling == VCL_RET_HIT_FOR_PASS || sp->handling == VCL_RET_DELIVER); if (wrk->objcore == NULL) { /* This is a pass from vcl_recv */ pass = 1; /* VCL may have fiddled this, but that doesn't help */ wrk->busyobj->exp.ttl = -1.; } else if (sp->handling == VCL_RET_HIT_FOR_PASS) { /* pass from vcl_fetch{} -> hit-for-pass */ /* XXX: the bereq was not filtered pass... */ pass = 1; } else { /* regular object */ pass = 0; } /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0; wrk->busyobj->is_gzip = http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip"); wrk->busyobj->is_gunzip = !http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip) wrk->busyobj->do_gunzip = 0; /* If we do gunzip, remove the C-E header */ if (wrk->busyobj->do_gunzip) http_Unset(wrk->busyobj->beresp, H_Content_Encoding); /* We wont gzip unless it is ungziped */ if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip) wrk->busyobj->do_gzip = 0; /* If we do gzip, add the C-E header */ if (wrk->busyobj->do_gzip) http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp, "Content-Encoding: gzip"); /* But we can't do both at the same time */ assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0); /* ESI takes precedence and handles gzip/gunzip itself */ if (wrk->busyobj->do_esi) wrk->busyobj->vfp = &vfp_esi; else if (wrk->busyobj->do_gunzip) wrk->busyobj->vfp = &vfp_gunzip; else if (wrk->busyobj->do_gzip) wrk->busyobj->vfp = &vfp_gzip; else if (wrk->busyobj->is_gzip) wrk->busyobj->vfp = &vfp_testgzip; if (wrk->busyobj->do_esi || sp->esi_level > 0) wrk->busyobj->do_stream = 0; if (!sp->wantbody) wrk->busyobj->do_stream = 0; l = http_EstimateWS(wrk->busyobj->beresp, pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); /* Create Vary instructions */ if (wrk->objcore != NULL) { CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC); vary = VRY_Create(sp, wrk->busyobj->beresp); if (vary != NULL) { varyl = VSB_len(vary); assert(varyl > 0); l += varyl; } } /* * Space for producing a Content-Length: header including padding * A billion gigabytes is enough for anybody. */ l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *); if (wrk->busyobj->exp.ttl < cache_param->shortlived || wrk->objcore == NULL) wrk->storage_hint = TRANSIENT_STORAGE; wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp); if (wrk->obj == NULL) { /* * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp); if (wrk->busyobj->exp.ttl > cache_param->shortlived) wrk->busyobj->exp.ttl = cache_param->shortlived; wrk->busyobj->exp.grace = 0.0; wrk->busyobj->exp.keep = 0.0; } if (wrk->obj == NULL) { sp->err_code = 503; sp->step = STP_ERROR; VDI_CloseFd(wrk, &wrk->busyobj->vbc); VBO_DerefBusyObj(wrk, &wrk->busyobj); return (0); } CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC); wrk->storage_hint = NULL; if (wrk->busyobj->do_gzip || (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip)) wrk->obj->gziped = 1; if (vary != NULL) { wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl); AN(wrk->obj->vary); memcpy(wrk->obj->vary, VSB_data(vary), varyl); VRY_Validate(wrk->obj->vary); VSB_delete(vary); } wrk->obj->xid = sp->xid; wrk->obj->response = sp->err_code; WS_Assert(wrk->obj->ws_o); /* Filter into object */ hp = wrk->busyobj->beresp; hp2 = wrk->obj->http; hp2->logtag = HTTP_Obj; http_CopyResp(hp2, hp); http_FilterFields(wrk, sp->vsl_id, hp2, hp, pass ? HTTPH_R_PASS : HTTPH_A_INS); http_CopyHome(wrk, sp->vsl_id, hp2); if (http_GetHdr(hp, H_Last_Modified, &b)) wrk->obj->last_modified = VTIM_parse(b); else wrk->obj->last_modified = floor(wrk->busyobj->exp.entered); assert(WRW_IsReleased(wrk)); /* * If we can deliver a 304 reply, we don't bother streaming. * Notice that vcl_deliver{} could still nuke the headers * that allow the 304, in which case we return 200 non-stream. */ if (wrk->obj->response == 200 && sp->http->conds && RFC2616_Do_Cond(sp)) wrk->busyobj->do_stream = 0; AssertObjCorePassOrBusy(wrk->obj->objcore); if (wrk->busyobj->do_stream) { sp->step = STP_PREPRESP; return (0); } /* Use unmodified headers*/ i = FetchBody(wrk, wrk->obj); http_Setup(wrk->busyobj->bereq, NULL); http_Setup(wrk->busyobj->beresp, NULL); wrk->busyobj->vfp = NULL; assert(WRW_IsReleased(wrk)); AZ(wrk->busyobj->vbc); AN(sp->director); if (i) { HSH_Drop(wrk); VBO_DerefBusyObj(wrk, &wrk->busyobj); AZ(wrk->obj); sp->err_code = 503; sp->step = STP_ERROR; return (0); } if (wrk->obj->objcore != NULL) { EXP_Insert(wrk->obj); AN(wrk->obj->objcore); AN(wrk->obj->objcore->ban); HSH_Unbusy(wrk); } VBO_DerefBusyObj(wrk, &wrk->busyobj); wrk->acct_tmp.fetch++; sp->step = STP_PREPRESP; return (0); }
static int cnt_fetch(struct sess *sp) { int i, need_host_hdr; struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); AN(sp->director); AZ(wrk->busyobj->vbc); AZ(wrk->busyobj->should_close); AZ(wrk->storage_hint); http_Setup(wrk->busyobj->beresp, wrk->ws); need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL); i = FetchHdr(sp, need_host_hdr); /* * If we recycle a backend connection, there is a finite chance * that the backend closed it before we get a request to it. * Do a single retry in that case. */ if (i == 1) { VSC_C_main->backend_retry++; i = FetchHdr(sp, need_host_hdr); } if (i) { sp->handling = VCL_RET_ERROR; sp->err_code = 503; } else { /* * These two headers can be spread over multiple actual headers * and we rely on their content outside of VCL, so collect them * into one line here. */ http_CollectHdr(wrk->busyobj->beresp, H_Cache_Control); http_CollectHdr(wrk->busyobj->beresp, H_Vary); /* * Figure out how the fetch is supposed to happen, before the * headers are adultered by VCL * NB: Also sets other wrk variables */ wrk->busyobj->body_status = RFC2616_Body(sp); sp->err_code = http_GetStatus(wrk->busyobj->beresp); /* * What does RFC2616 think about TTL ? */ EXP_Clr(&wrk->busyobj->exp); wrk->busyobj->exp.entered = W_TIM_real(wrk); RFC2616_Ttl(sp); /* pass from vclrecv{} has negative TTL */ if (wrk->objcore == NULL) wrk->busyobj->exp.ttl = -1.; AZ(wrk->busyobj->do_esi); VCL_fetch_method(sp); switch (sp->handling) { case VCL_RET_HIT_FOR_PASS: if (wrk->objcore != NULL) wrk->objcore->flags |= OC_F_PASS; sp->step = STP_FETCHBODY; return (0); case VCL_RET_DELIVER: AssertObjCorePassOrBusy(wrk->objcore); sp->step = STP_FETCHBODY; return (0); default: break; } /* We are not going to fetch the body, Close the connection */ VDI_CloseFd(wrk, &wrk->busyobj->vbc); } /* Clean up partial fetch */ AZ(wrk->busyobj->vbc); if (wrk->objcore != NULL) { CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC); AZ(HSH_Deref(wrk, wrk->objcore, NULL)); wrk->objcore = NULL; } VBO_DerefBusyObj(wrk, &wrk->busyobj); sp->director = NULL; wrk->storage_hint = NULL; switch (sp->handling) { case VCL_RET_RESTART: sp->restarts++; sp->step = STP_RECV; return (0); case VCL_RET_ERROR: sp->step = STP_ERROR; return (0); default: WRONG("Illegal action in vcl_fetch{}"); } }
static int cnt_prepresp(struct sess *sp) { struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); if (wrk->busyobj != NULL) { CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); AN(wrk->busyobj->do_stream); AssertObjCorePassOrBusy(wrk->obj->objcore); } wrk->res_mode = 0; if (wrk->busyobj == NULL) wrk->res_mode |= RES_LEN; if (wrk->busyobj != NULL && (wrk->busyobj->h_content_length != NULL || !wrk->busyobj->do_stream) && !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip) wrk->res_mode |= RES_LEN; if (!sp->disable_esi && wrk->obj->esidata != NULL) { /* In ESI mode, we don't know the aggregate length */ wrk->res_mode &= ~RES_LEN; wrk->res_mode |= RES_ESI; } if (sp->esi_level > 0) { wrk->res_mode &= ~RES_LEN; wrk->res_mode |= RES_ESI_CHILD; } if (cache_param->http_gzip_support && wrk->obj->gziped && !RFC2616_Req_Gzip(sp)) { /* * We don't know what it uncompresses to * XXX: we could cache that */ wrk->res_mode &= ~RES_LEN; wrk->res_mode |= RES_GUNZIP; } if (!(wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) { if (wrk->obj->len == 0 && (wrk->busyobj == NULL || !wrk->busyobj->do_stream)) /* * If the object is empty, neither ESI nor GUNZIP * can make it any different size */ wrk->res_mode |= RES_LEN; else if (!sp->wantbody) { /* Nothing */ } else if (sp->http->protover >= 11) { wrk->res_mode |= RES_CHUNKED; } else { wrk->res_mode |= RES_EOF; sp->doclose = "EOF mode"; } } sp->t_resp = W_TIM_real(wrk); if (wrk->obj->objcore != NULL) { if ((sp->t_resp - wrk->obj->last_lru) > cache_param->lru_timeout && EXP_Touch(wrk->obj->objcore)) wrk->obj->last_lru = sp->t_resp; wrk->obj->last_use = sp->t_resp; /* XXX: locking ? */ } http_Setup(wrk->resp, wrk->ws); RES_BuildHttp(sp); VCL_deliver_method(sp); switch (sp->handling) { case VCL_RET_DELIVER: break; case VCL_RET_RESTART: if (sp->restarts >= cache_param->max_restarts) break; if (wrk->busyobj != NULL) { AN(wrk->busyobj->do_stream); VDI_CloseFd(wrk, &wrk->busyobj->vbc); HSH_Drop(wrk); VBO_DerefBusyObj(wrk, &wrk->busyobj); } else { (void)HSH_Deref(wrk, NULL, &wrk->obj); } AZ(wrk->obj); sp->restarts++; sp->director = NULL; http_Setup(wrk->resp, NULL); sp->step = STP_RECV; return (0); default: WRONG("Illegal action in vcl_deliver{}"); } if (wrk->busyobj != NULL && wrk->busyobj->do_stream) { AssertObjCorePassOrBusy(wrk->obj->objcore); sp->step = STP_STREAMBODY; } else { sp->step = STP_DELIVER; } return (0); }
int FetchBody(struct sess *sp) { int cls; struct storage *st; struct worker *w; int mklen; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); w = sp->wrk; CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC); if (w->vfp == NULL) w->vfp = &vfp_nop; AN(sp->director); AssertObjCorePassOrBusy(sp->obj->objcore); AZ(w->vgz_rx); AZ(VTAILQ_FIRST(&sp->obj->store)); switch (w->body_status) { case BS_NONE: cls = 0; mklen = 0; break; case BS_ZERO: cls = 0; mklen = 1; break; case BS_LENGTH: cls = fetch_straight(sp, w->htc, w->h_content_length); mklen = 1; XXXAZ(w->vfp->end(sp)); break; case BS_CHUNKED: cls = fetch_chunked(sp, w->htc); mklen = 1; XXXAZ(w->vfp->end(sp)); break; case BS_EOF: cls = fetch_eof(sp, w->htc); mklen = 1; XXXAZ(w->vfp->end(sp)); break; case BS_ERROR: cls = 1; mklen = 0; break; default: cls = 0; mklen = 0; INCOMPL(); } AZ(w->vgz_rx); /* * It is OK for ->end to just leave the last storage segment * sitting on w->storage, we will always call vfp_nop_end() * to get it trimmed or thrown out if empty. */ AZ(vfp_nop_end(sp)); WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u", w->body_status, body_status(w->body_status), cls, mklen); if (w->body_status == BS_ERROR) { VDI_CloseFd(sp); return (__LINE__); } if (cls < 0) { w->stats.fetch_failed++; /* XXX: Wouldn't this store automatically be released ? */ while (!VTAILQ_EMPTY(&sp->obj->store)) { st = VTAILQ_FIRST(&sp->obj->store); VTAILQ_REMOVE(&sp->obj->store, st, list); STV_free(st); } VDI_CloseFd(sp); sp->obj->len = 0; return (__LINE__); } if (cls == 0 && w->do_close) cls = 1; WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len); { /* Sanity check fetch methods accounting */ ssize_t uu; uu = 0; VTAILQ_FOREACH(st, &sp->obj->store, list) uu += st->len; if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS)) /* Streaming might have started freeing stuff */ assert (uu <= sp->obj->len); else assert(uu == sp->obj->len); } if (mklen > 0) { http_Unset(sp->obj->http, H_Content_Length); http_PrintfHeader(w, sp->vsl_id, sp->obj->http, "Content-Length: %jd", (intmax_t)sp->obj->len); } if (cls) VDI_CloseFd(sp); else VDI_RecycleFd(sp); return (0); }