void VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs, const char *p, ...) { struct http *hp; va_list ap; const char *b; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(hs); AN(hs->what); hp = vrt_selecthttp(ctx, hs->where); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); va_start(ap, p); if (p == vrt_magic_string_unset) { http_Unset(hp, hs->what); } else { b = VRT_String(hp->ws, hs->what + 1, p, ap); if (b == NULL) { VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1); } else { http_Unset(hp, hs->what); http_SetHeader(hp, b); } } va_end(ap); }
vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) { struct vef_priv *vef; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); if (http_GetStatus(vc->resp) == 206) { VSLb(vc->wrk->vsl, SLT_VCL_Error, "Attempted ESI on partial (206) response"); return (VFP_ERROR); } ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); vc->obj_flags |= OF_GZIPED | OF_CHGGZIP | OF_ESIPROC; vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E"); vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef); vef->ibuf_sz = cache_param->gzip_buffer; vef->ibuf = calloc(1L, vef->ibuf_sz); if (vef->ibuf == NULL) return (vfp_esi_end(vc, vef, VFP_ERROR)); vef->ibuf_i = vef->ibuf; vef->ibuf_o = vef->ibuf; vfe->priv1 = vef; RFC2616_Weaken_Etag(vc->resp); http_Unset(vc->resp, H_Content_Length); http_Unset(vc->resp, H_Content_Encoding); http_SetHeader(vc->resp, "Content-Encoding: gzip"); RFC2616_Vary_AE(vc->resp); return (VFP_OK); }
vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) { struct vef_priv *vef; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->esi_req, HTTP_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E"); vef->vep = VEP_Init(vc, vc->esi_req, vfp_vep_callback, vef); vef->ibuf_sz = cache_param->gzip_buffer; vef->ibuf = calloc(1L, vef->ibuf_sz); if (vef->ibuf == NULL) return (vfp_esi_end(vc, vef, VFP_ERROR)); vef->ibuf_i = vef->ibuf; vef->ibuf_o = vef->ibuf; vfe->priv1 = vef; RFC2616_Weaken_Etag(vc->http); http_Unset(vc->http, H_Content_Length); http_Unset(vc->http, H_Content_Encoding); http_SetHeader(vc->http, "Content-Encoding: gzip"); RFC2616_Vary_AE(vc->http); return (VFP_OK); }
static void cnt_vdp(struct req *req, struct busyobj *bo) { const char *r; uint16_t status; int wantbody; CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC); req->res_mode = 0; wantbody = 1; status = http_GetStatus(req->resp); if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) { wantbody = 0; } else if (status < 200 || status == 204) { req->resp_len = 0; http_Unset(req->resp, H_Content_Length); wantbody = 0; } else if (status == 304) { http_Unset(req->resp, H_Content_Length); wantbody = 0; } else if (bo != NULL) req->resp_len = http_GetContentLength(req->resp); else req->resp_len = ObjGetLen(req->wrk, req->objcore); /* * Determine ESI status first. Not dependent on wantbody, because * we want ESI to supress C-L in HEAD too. */ if (!req->disable_esi && req->resp_len != 0 && wantbody && ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) { req->res_mode |= RES_ESI; RFC2616_Weaken_Etag(req->resp); req->resp_len = -1; VDP_push(req, VDP_ESI, NULL, 0); } if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) { req->res_mode |= RES_GUNZIP; VDP_push(req, VDP_gunzip, NULL, 1); } /* * Range comes after the others and pushes on bottom because * it can (maybe) generate a correct C-L header. */ if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) { http_SetHeader(req->resp, "Accept-Ranges: bytes"); if (wantbody && http_GetHdr(req->http, H_Range, &r)) VRG_dorange(req, r); } req->transport->deliver(req, bo, wantbody); }
static enum req_fsm_nxt cnt_synth(struct worker *wrk, struct req *req) { char date[40]; struct http *h; double now; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); wrk->stats.s_synth++; now = W_TIM_real(wrk); VSLb_ts_req(req, "Process", now); if (req->err_code < 100 || req->err_code > 999) req->err_code = 501; HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod); h = req->resp; VTIM_format(now, date); http_PrintfHeader(h, "Date: %s", date); http_SetHeader(h, "Server: Varnish"); http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid)); http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason); AZ(req->synth_body); req->synth_body = VSB_new_auto(); AN(req->synth_body); VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws); http_Unset(h, H_Content_Length); AZ(VSB_finish(req->synth_body)); if (wrk->handling == VCL_RET_RESTART) { HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod); VSB_delete(req->synth_body); req->synth_body = NULL; req->req_step = R_STP_RESTART; return (REQ_FSM_MORE); } assert(wrk->handling == VCL_RET_DELIVER); if (http_HdrIs(req->resp, H_Connection, "close")) req->doclose = SC_RESP_CLOSE; V1D_Deliver_Synth(req); VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); VSB_delete(req->synth_body); req->synth_body = NULL; req->err_code = 0; req->err_reason = NULL; return (REQ_FSM_DONE); }
void VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...) { va_list ap; const char *p; struct vsb *vsb; (void)flags; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC); vsb = SMS_Makesynth(sp->wrk->obj); AN(vsb); VSB_cat(vsb, str); va_start(ap, str); p = va_arg(ap, const char *); while (p != vrt_magic_string_end) { if (p == NULL) p = "(null)"; VSB_cat(vsb, p); p = va_arg(ap, const char *); } va_end(ap); SMS_Finish(sp->wrk->obj); http_Unset(sp->wrk->obj->http, H_Content_Length); http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->obj->http, "Content-Length: %zd", sp->wrk->obj->len); }
static enum fetch_step vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); AN(bo->director); AZ(bo->vbc); AZ(bo->should_close); AZ(bo->storage_hint); HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(bo->bereq0, bo->req->http, bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH); if (!bo->do_pass) { // XXX: Forcing GET should happen in vcl_miss{} ? http_ForceGet(bo->bereq0); if (cache_param->http_gzip_support) { /* * We always ask the backend for gzip, even if the * client doesn't grok it. We will uncompress for * the minority of clients which don't. */ http_Unset(bo->bereq0, H_Accept_Encoding); http_SetHeader(bo->bereq0, "Accept-Encoding: gzip"); } } return (F_STP_STARTFETCH); }
uint16_t HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) { uint16_t retval; const char *p; const char *b = NULL, *e; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); retval = http1_splitline(hp, htc, HTTP1_Req, cache_param->http_req_hdr_len); if (retval != 0) return (retval); hp->protover = http1_proto_ver(hp); if (hp->protover == 0) return (400); if (http_CountHdr(hp, H_Host) > 1) return (400); if (http_CountHdr(hp, H_Content_Length) > 1) return (400); /* RFC2616, section 5.2, point 1 */ if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) b = hp->hd[HTTP_HDR_URL].b + 7; else if (FEATURE(FEATURE_HTTPS_SCHEME) && !strncasecmp(hp->hd[HTTP_HDR_URL].b, "https://", 8)) b = hp->hd[HTTP_HDR_URL].b + 8; if (b) { e = strchr(b, '/'); if (e) { http_Unset(hp, H_Host); http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b); hp->hd[HTTP_HDR_URL].b = e; } } htc->body_status = http1_body_status(hp, htc, 1); if (htc->body_status == BS_ERROR) return (400); p = http_GetMethod(hp); AN(p); if (htc->body_status == BS_EOF) { assert(hp->protover == 10); /* RFC1945 8.3 p32 and D.1.1 p58 */ if (!strcasecmp(p, "post") || !strcasecmp(p, "put")) return (400); htc->body_status = BS_NONE; } /* HEAD with a body is a hard error */ if (htc->body_status != BS_NONE && !strcasecmp(p, "head")) return (400); return (retval); }
void VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...) { va_list ap; const char *p; struct vsb *vsb; (void)flags; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC); vsb = SMS_Makesynth(ctx->req->obj); AN(vsb); va_start(ap, str); p = str; while (p != vrt_magic_string_end) { if (p == NULL) p = "(null)"; VSB_cat(vsb, p); p = va_arg(ap, const char *); } va_end(ap); SMS_Finish(ctx->req->obj); http_Unset(ctx->req->obj->http, H_Content_Length); }
static int vbf_figure_out_vfp(struct busyobj *bo) { /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* No body or no GZIP supprt -> done */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || !cache_param->http_gzip_support) { http_Unset(bo->beresp, H_Content_Encoding); bo->do_gzip = bo->do_gunzip = 0; bo->do_stream = 0; return (0); } bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* We wont gzip unless if it already is gzip'ed */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL) return (-1); if (bo->do_esi && (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))) return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0); if (bo->do_esi) return (VFP_Push(bo->vfc, &VFP_esi) == NULL ? -1 : 0); if (bo->do_gzip) return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0); if (bo->is_gzip && !bo->do_gunzip) return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0); return (0); }
V1D_Deliver(struct req *req, struct boc *boc, int sendbody) { int err = 0; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_ORNULL(boc, BOC_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); if (sendbody) { if (http_GetHdr(req->resp, H_Content_Length, NULL)) req->res_mode |= RES_LEN; else if (req->http->protover == 11) { req->res_mode |= RES_CHUNKED; http_SetHeader(req->resp, "Transfer-Encoding: chunked"); } else { req->res_mode |= RES_EOF; req->doclose = SC_TX_EOF; } } VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode); if (req->doclose) { if (!http_HdrIs(req->resp, H_Connection, "close")) { http_Unset(req->resp, H_Connection); http_SetHeader(req->resp, "Connection: close"); } } else if (!http_GetHdr(req->resp, H_Connection, NULL)) http_SetHeader(req->resp, "Connection: keep-alive"); if (sendbody && req->resp_len != 0) VDP_push(req, v1d_bytes, NULL, 1); AZ(req->wrk->v1l); V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev); if (WS_Overflowed(req->ws)) { v1d_error(req, "workspace_client overflow"); AZ(req->wrk->v1l); return; } req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp); if (DO_DEBUG(DBG_FLUSH_HEAD)) (void)V1L_Flush(req->wrk); if (sendbody && req->resp_len != 0) { if (req->res_mode & RES_CHUNKED) V1L_Chunked(req->wrk); err = VDP_DeliverObj(req); if (!err && (req->res_mode & RES_CHUNKED)) V1L_EndChunk(req->wrk); } if ((V1L_FlushRelease(req->wrk) || err) && req->sp->fd >= 0) SES_Close(req->sp, SC_REM_CLOSE); AZ(req->wrk->v1l); VDP_close(req); }
uint16_t HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) { uint16_t retval; const char *p; const char *b, *e; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); retval = http1_splitline(hp, htc, HTTP1_Req); if (retval != 0) return (retval); hp->protover = http1_proto_ver(hp); if (hp->protover == 0) return (400); if (http_CountHdr(hp, H_Host) > 1) return (400); if (http_CountHdr(hp, H_Content_Length) > 1) return (400); /* RFC2616, section 5.2, point 1 */ if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) { b = e = hp->hd[HTTP_HDR_URL].b + 7; while (*e != '/' && *e != '\0') e++; if (*e == '/') { http_Unset(hp, H_Host); http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b); hp->hd[HTTP_HDR_URL].b = e; } } htc->body_status = http1_body_status(hp, htc); if (htc->body_status == BS_ERROR) return (400); p = http_GetMethod(hp); AN(p); if (htc->body_status == BS_EOF) { assert(hp->protover == 10); /* RFC1945 8.3 p32 and D.1.1 p58 */ if (!strcasecmp(p, "post") || !strcasecmp(p, "put")) return (400); htc->body_status = BS_NONE; } /* HEAD with a body is a hard error */ if (htc->body_status != BS_NONE && !strcasecmp(p, "head")) return (400); return (retval); }
vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) { struct vgz *vg; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); if (http_HdrIs(vc->http, H_Content_Length, "0")) { http_Unset(vc->http, H_Content_Encoding); return (VFP_NULL); } if (vfe->vfp->priv2 == VFP_GZIP) { if (http_GetHdr(vc->http, H_Content_Encoding, NULL)) return (VFP_NULL); vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1); } else { if (!http_HdrIs(vc->http, H_Content_Encoding, "gzip")) return (VFP_NULL); vg = VGZ_NewUngzip(vc->wrk->vsl, vfe->vfp->priv1); } if (vg == NULL) return (VFP_ERROR); vfe->priv1 = vg; if (vgz_getmbuf(vg)) return (VFP_ERROR); VGZ_Ibuf(vg, vg->m_buf, 0); AZ(vg->m_len); if (vfe->vfp->priv2 == VFP_GUNZIP || vfe->vfp->priv2 == VFP_GZIP) { http_Unset(vc->http, H_Content_Encoding); http_Unset(vc->http, H_Content_Length); RFC2616_Weaken_Etag(vc->http); } if (vfe->vfp->priv2 == VFP_GZIP) http_SetHeader(vc->http, "Content-Encoding: gzip"); if (vfe->vfp->priv2 == VFP_GZIP || vfe->vfp->priv2 == VFP_TESTGUNZIP) RFC2616_Vary_AE(vc->http); return (VFP_OK); }
void http_ForceHeader(struct http *to, const char *hdr, const char *val) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); if (http_HdrIs(to, hdr, val)) return; http_Unset(to, hdr); http_PrintfHeader(to, "%s %s", hdr + 1, val); }
static enum body_status http1_body_status(struct http *hp, struct http_conn *htc) { ssize_t cl; const char *b; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); htc->content_length = -1; if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { http_Unset(hp, H_Content_Length); return (BS_CHUNKED); } if (http_GetHdr(hp, H_Transfer_Encoding, &b)) return (BS_ERROR); cl = http_GetContentLength(hp); if (cl == -2) return (BS_ERROR); if (cl >= 0) { htc->content_length = cl; return (cl == 0 ? BS_NONE : BS_LENGTH); } if (http_HdrIs(hp, H_Connection, "keep-alive")) { /* * Keep alive with neither TE=Chunked or C-Len is impossible. * We assume a zero length body. */ return (BS_NONE); } if (http_HdrIs(hp, H_Connection, "close")) { /* * In this case, it is safe to just read what comes. */ return (BS_EOF); } if (hp->protover < 11) { /* * With no Connection header, assume EOF. */ return (BS_EOF); } /* * Fall back to EOF transfer. */ return (BS_EOF); }
static int cnt_miss(struct worker *wrk, struct req *req) { struct busyobj *bo; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(req->obj); HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(req, HTTPH_R_FETCH); http_ForceGet(bo->bereq); if (cache_param->http_gzip_support) { /* * We always ask the backend for gzip, even if the * client doesn't grok it. We will uncompress for * the minority of clients which don't. */ http_Unset(bo->bereq, H_Accept_Encoding); http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); } VCL_miss_method(req); if (req->handling == VCL_RET_FETCH) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); req->req_step = R_STP_FETCH; return (0); } AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); req->objcore = NULL; http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); switch(req->handling) { case VCL_RET_ERROR: req->req_step = R_STP_ERROR; break; case VCL_RET_PASS: req->req_step = R_STP_PASS; break; case VCL_RET_RESTART: req->req_step = R_STP_RESTART; break; default: WRONG("Illegal action in vcl_miss{}"); } return (0); }
void RFC2616_Vary_AE(struct http *hp) { const char *vary; if (http_GetHdrToken(hp, H_Vary, "Accept-Encoding", NULL, NULL)) return; if (http_GetHdr(hp, H_Vary, &vary)) { http_Unset(hp, H_Vary); http_PrintfHeader(hp, "Vary: %s, Accept-Encoding", vary); } else { http_SetHeader(hp, "Vary: Accept-Encoding"); } }
void VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr, const char *p, ...) { struct http *hp; va_list ap; char *b; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); hp = vrt_selecthttp(sp, where); va_start(ap, p); if (p == NULL) { http_Unset(hp, hdr); } else { b = VRT_String(hp->ws, hdr + 1, p, ap); if (b == NULL) { WSP(sp, SLT_LostHeader, "%s", hdr + 1); } else { http_Unset(hp, hdr); http_SetHeader(sp->wrk, sp->vsl_id, hp, b); } } va_end(ap); }
void RFC2616_Weaken_Etag(struct http *hp) { const char *p; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (!http_GetHdr(hp, H_ETag, &p)) return; AN(p); if (p[0] == 'W' && p[1] == '/') return; http_Unset(hp, H_ETag); http_PrintfHeader(hp, "ETag: W/%s", p); }
static enum fetch_step vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) { char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); AN(bo->director); AZ(bo->vbc); AZ(bo->should_close); AZ(bo->storage_hint); HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(bo->bereq0, bo->req->http, bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH); if (!bo->do_pass) { // XXX: Forcing GET should happen in vcl_miss{} ? http_ForceGet(bo->bereq0); if (cache_param->http_gzip_support) { /* * We always ask the backend for gzip, even if the * client doesn't grok it. We will uncompress for * the minority of clients which don't. */ http_Unset(bo->bereq0, H_Accept_Encoding); http_SetHeader(bo->bereq0, "Accept-Encoding: gzip"); } } if (bo->ims_obj != NULL) { if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) { http_PrintfHeader(bo->bereq0, "If-Modified-Since: %s", p); } else if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) { http_PrintfHeader(bo->bereq0, "If-None-Match: %s", p); } else { WRONG("Shouldn't have bo->ims_obj"); } } return (F_STP_STARTFETCH); }
uint16_t HTTP1_DissectRequest(struct req *req) { struct http_conn *htc; struct http *hp; uint16_t retval; char *b, *e; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); htc = req->htc; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); hp = req->http; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); retval = htc_splitline(hp, htc, 1); if (retval != 0) { VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf); return (retval); } htc_proto_ver(hp); retval = htc_request_check_host_hdr(hp); if (retval != 0) { return (retval); } /* RFC2616, section 5.2, point 1 */ if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) { b = e = hp->hd[HTTP_HDR_URL].b + 7; while (*e != '/' && *e != '\0') e++; if (*e == '/') { http_Unset(hp, H_Host); http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b); hp->hd[HTTP_HDR_URL].b = e; } } return (retval); }
static void ved_include(struct req *preq, const char *src, const char *host, struct ecx *ecx) { struct worker *wrk; struct req *req; enum req_fsm_nxt s; struct transport xp; CHECK_OBJ_NOTNULL(preq, REQ_MAGIC); CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC); wrk = preq->wrk; if (preq->esi_level >= cache_param->max_esi_depth) return; req = Req_New(wrk, preq->sp); req->req_body_status = REQ_BODY_NONE; AZ(req->vsl->wid); req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); VSLb(req->vsl, SLT_Begin, "req %u esi", VXID(preq->vsl->wid)); VSLb(preq->vsl, SLT_Link, "req %u esi", VXID(req->vsl->wid)); req->esi_level = preq->esi_level + 1; if (preq->esi_level == 0) assert(preq->top == preq); else CHECK_OBJ_NOTNULL(preq->top, REQ_MAGIC); req->top = preq->top; HTTP_Copy(req->http0, preq->http0); req->http0->ws = req->ws; req->http0->vsl = req->vsl; req->http0->logtag = SLT_ReqMethod; req->http0->conds = 0; http_SetH(req->http0, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { http_Unset(req->http0, H_Host); http_SetHeader(req->http0, host); } http_ForceField(req->http0, HTTP_HDR_METHOD, "GET"); http_ForceField(req->http0, HTTP_HDR_PROTO, "HTTP/1.1"); /* Don't allow conditionalss, we can't use a 304 */ http_Unset(req->http0, H_If_Modified_Since); http_Unset(req->http0, H_If_None_Match); /* Don't allow Range */ http_Unset(req->http0, H_Range); /* Set Accept-Encoding according to what we want */ http_Unset(req->http0, H_Accept_Encoding); if (ecx->isgzip) http_ForceHeader(req->http0, H_Accept_Encoding, "gzip"); /* Client content already taken care of */ http_Unset(req->http0, H_Content_Length); /* Reset request to status before we started messing with it */ HTTP_Copy(req->http, req->http0); req->vcl = preq->vcl; preq->vcl = NULL; req->wrk = preq->wrk; /* * XXX: We should decide if we should cache the director * XXX: or not (for session/backend coupling). Until then * XXX: make sure we don't trip up the check in vcl_recv. */ req->req_step = R_STP_RECV; req->t_req = preq->t_req; assert(isnan(req->t_first)); assert(isnan(req->t_prev)); INIT_OBJ(&xp, TRANSPORT_MAGIC); xp.deliver = VED_Deliver; req->transport = &xp; req->transport_priv = ecx; THR_SetRequest(req); VSLb_ts_req(req, "Start", W_TIM_real(wrk)); req->ws_req = WS_Snapshot(req->ws); while (1) { req->wrk = wrk; s = CNT_Request(wrk, req); if (s == REQ_FSM_DONE) break; DSL(DBG_WAITINGLIST, req->vsl->wid, "loop waiting for ESI (%d)", (int)s); assert(s == REQ_FSM_DISEMBARK); AZ(req->wrk); (void)usleep(10000); } VRTPRIV_dynamic_kill(req->sp->privs, (uintptr_t)req); CNT_AcctLogCharge(wrk->stats, req); VSL_End(req->vsl); preq->vcl = req->vcl; req->vcl = NULL; req->wrk = NULL; THR_SetRequest(preq); Req_Release(req); }
static enum fetch_step vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) { int i; double now; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(bo->storage); bo->storage = bo->do_pass ? stv_transient : STV_next(); if (bo->retries > 0) http_Unset(bo->bereq, "\012X-Varnish:"); http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid)); VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); bo->uncacheable = bo->do_pass; if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) return (F_STP_FAIL); assert (wrk->handling == VCL_RET_FETCH); HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); AZ(bo->htc); VFP_Setup(bo->vfc, wrk); bo->vfc->oc = bo->fetch_objcore; bo->vfc->resp = bo->beresp; bo->vfc->req = bo->bereq; i = VDI_GetHdr(wrk, bo); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Beresp", now); if (i) { assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } http_VSL_log(bo->beresp); if (bo->htc->body_status == BS_ERROR) { bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } if (!http_GetHdr(bo->beresp, H_Date, NULL)) { /* * RFC 2616 14.18 Date: The Date general-header field * represents the date and time at which the message was * originated, having the same semantics as orig-date in * RFC 822. ... A received message that does not have a * Date header field MUST be assigned one by the recipient * if the message will be cached by that recipient or * gatewayed via a protocol which requires a Date. * * If we didn't get a Date header, we assign one here. */ http_TimeHeader(bo->beresp, "Date: ", now); } /* * These two headers can be spread over multiple actual headers * and we rely on their content outside of VCL, so collect them * into one line here. */ http_CollectHdr(bo->beresp, H_Cache_Control); http_CollectHdr(bo->beresp, H_Vary); if (bo->fetch_objcore->flags & OC_F_PRIVATE) { /* private objects have negative TTL */ bo->fetch_objcore->t_origin = now; bo->fetch_objcore->ttl = -1.; bo->fetch_objcore->grace = 0; bo->fetch_objcore->keep = 0; } else { /* What does RFC2616 think about TTL ? */ RFC2616_Ttl(bo, now, &bo->fetch_objcore->t_origin, &bo->fetch_objcore->ttl, &bo->fetch_objcore->grace, &bo->fetch_objcore->keep ); } AZ(bo->do_esi); AZ(bo->was_304); if (http_IsStatus(bo->beresp, 304)) { if (bo->stale_oc != NULL && ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) { if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) { /* * If we changed the gzip status of the object * the stored Content_Encoding controls we * must weaken any new ETag we get. */ http_Unset(bo->beresp, H_Content_Encoding); RFC2616_Weaken_Etag(bo->beresp); } http_Unset(bo->beresp, H_Content_Length); HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); assert(http_IsStatus(bo->beresp, 200)); bo->was_304 = 1; } else if (!bo->do_pass) { /* * Backend sent unallowed 304 */ VSLb(bo->vsl, SLT_Error, "304 response but not conditional fetch"); bo->htc->doclose = SC_RX_BAD; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } } VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) { bo->htc->doclose = SC_RESP_CLOSE; VDI_Finish(bo->wrk, bo); return (F_STP_FAIL); } if (wrk->handling == VCL_RET_RETRY) { if (bo->htc->body_status != BS_NONE) bo->htc->doclose = SC_RESP_CLOSE; if (bo->director_state != DIR_S_NULL) VDI_Finish(bo->wrk, bo); if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, delivering 503"); assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); if (bo->fetch_objcore->boc->state != BOS_REQ_DONE) { bo->req = NULL; ObjSetState(wrk, bo->fetch_objcore, BOS_REQ_DONE); } if (bo->do_esi) bo->do_stream = 0; if (wrk->handling == VCL_RET_PASS) { bo->fetch_objcore->flags |= OC_F_HFP; bo->uncacheable = 1; wrk->handling = VCL_RET_DELIVER; } if (bo->do_pass || bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; assert(wrk->handling == VCL_RET_DELIVER); return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH); }
VDP_gunzip(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { enum vgzret_e vr; ssize_t dl; const void *dp; struct worker *wrk; struct vgz *vg; const char *p; uint64_t u; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); if (act == VDP_INIT) { vg = VGZ_NewUngzip(req->vsl, "U D -"); AN(vg); if (vgz_getmbuf(vg)) { (void)VGZ_Destroy(&vg); return (-1); } req->res_mode |= RES_GUNZIP; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); *priv = vg; http_Unset(req->resp, H_Content_Encoding); req->resp_len = -1; if (req->objcore->boc != NULL) return (0); /* No idea about length (yet) */ p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); if (p == NULL || dl != 32) return (0); /* No OA_GZIPBITS yet */ u = vbe64dec(p + 24); /* * If the size is non-zero AND we are the top * VDP (ie: no ESI), we know what size the output will be. */ if (u != 0 && VTAILQ_FIRST(&req->vdp)->func == VDP_gunzip) req->resp_len = u; return (0); } CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC); AN(vg->m_buf); if (act == VDP_FINI) { /* NB: Gunzip'ing may or may not have completed successfully. */ AZ(len); (void)VGZ_Destroy(&vg); *priv = NULL; return (0); } if (len == 0) return (0); VGZ_Ibuf(vg, ptr, len); do { vr = VGZ_Gunzip(vg, &dp, &dl); vg->m_len += dl; if (vr < VGZ_OK) return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len)) return (req->vdp_retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } } while (!VGZ_IbufEmpty(vg)); assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END); return (0); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct http *hp, *hp2; char *b; uint16_t nhttp; unsigned l; struct vsb *vary = NULL; int varyl = 0; struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * * XXX: BS_NONE/cl==0 should avoid gzip/gunzip */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); /* It can't be both */ assert(bo->is_gzip == 0 || bo->is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ if (bo->do_gunzip && !bo->is_gzip) bo->do_gunzip = 0; /* If we do gunzip, remove the C-E header */ if (bo->do_gunzip) http_Unset(bo->beresp, H_Content_Encoding); /* We wont gzip unless it is ungziped */ if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; /* If we do gzip, add the C-E header */ if (bo->do_gzip) http_SetHeader(bo->beresp, "Content-Encoding: gzip"); /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); /* ESI takes precedence and handles gzip/gunzip itself */ if (bo->do_esi) bo->vfp = &vfp_esi; else if (bo->do_gunzip) bo->vfp = &vfp_gunzip; else if (bo->do_gzip) bo->vfp = &vfp_gzip; else if (bo->is_gzip) bo->vfp = &vfp_testgzip; if (bo->fetch_objcore->objhead == NULL) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc.body_status == BS_NONE) bo->do_stream = 0; l = http_EstimateWS(bo->beresp, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); /* Create Vary instructions */ if (bo->fetch_objcore->objhead != NULL) { varyl = VRY_Create(bo, &vary); if (varyl > 0) { AN(vary); assert(varyl == VSB_len(vary)); l += varyl; } else if (varyl < 0) { /* * Vary parse error * Complain about it, and make this a pass. */ VSLb(bo->vsl, SLT_Error, "Illegal 'Vary' header from backend, " "making this a pass."); bo->uncacheable = 1; AZ(vary); } else /* No vary */ AZ(vary); } if (bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1) bo->storage_hint = TRANSIENT_STORAGE; /* * Space for producing a Content-Length: header including padding * A billion gigabytes is enough for anybody. */ l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *); AZ(bo->stats); bo->stats = &wrk->stats; obj = STV_NewObject(bo, bo->storage_hint, l, nhttp); if (obj == NULL) { /* * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ if (bo->exp.ttl > cache_param->shortlived) bo->exp.ttl = cache_param->shortlived; bo->exp.grace = 0.0; bo->exp.keep = 0.0; obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp); } bo->stats = NULL; if (obj == NULL) { AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL)); bo->fetch_objcore = NULL; VDI_CloseFd(&bo->vbc); VBO_setstate(bo, BOS_FAILED); return (F_STP_ABANDON); } CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); bo->storage_hint = NULL; AZ(bo->fetch_obj); bo->fetch_obj = obj; if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) obj->gziped = 1; if (vary != NULL) { obj->vary = (void *)WS_Copy(obj->http->ws, VSB_data(vary), varyl); AN(obj->vary); VRY_Validate(obj->vary); VSB_delete(vary); } obj->vxid = bo->vsl->wid; obj->response = bo->err_code; WS_Assert(obj->ws_o); /* Filter into object */ hp = bo->beresp; hp2 = obj->http; hp2->logtag = HTTP_Obj; http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS); http_CopyHome(hp2); if (http_GetHdr(hp, H_Last_Modified, &b)) obj->last_modified = VTIM_parse(b); else obj->last_modified = floor(bo->exp.entered); assert(WRW_IsReleased(wrk)); /* * Ready to fetch the body */ assert(bo->refcount >= 1); if (obj->objcore->objhead != NULL) { EXP_Insert(obj); AN(obj->objcore->ban); AZ(obj->ws_o->overflow); HSH_Unbusy(&wrk->stats, obj->objcore); } if (bo->vfp == NULL) bo->vfp = &VFP_nop; assert(bo->state == BOS_INVALID); VBO_setstate(bo, BOS_FETCHING); V1F_fetch_body(wrk, bo); assert(bo->refcount >= 1); if (obj->objcore->objhead != NULL) HSH_Ref(obj->objcore); if (bo->state == BOS_FAILED) { /* handle early failures */ (void)HSH_Deref(&wrk->stats, NULL, &obj); return (F_STP_ABANDON); } VBO_setstate(bo, BOS_FINISHED); VBO_DerefBusyObj(wrk, &bo); // XXX ? return (F_STP_DONE); }
static enum fetch_step vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) { int i; double now; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(bo->storage_hint); if (bo->do_pass) AN(bo->req); else AZ(bo->req); if (bo->retries > 0) http_Unset(bo->bereq, "\012X-Varnish:"); http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid)); VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); bo->uncacheable = bo->do_pass; if (wrk->handling == VCL_RET_ABANDON) return (F_STP_FAIL); assert (wrk->handling == VCL_RET_FETCH); HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); assert(bo->state <= BOS_REQ_DONE); AZ(bo->htc); i = VDI_GetHdr(wrk, bo); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Beresp", now); if (i) { assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } http_VSL_log(bo->beresp); if (!http_GetHdr(bo->beresp, H_Date, NULL)) { /* * RFC 2616 14.18 Date: The Date general-header field * represents the date and time at which the message was * originated, having the same semantics as orig-date in * RFC 822. ... A received message that does not have a * Date header field MUST be assigned one by the recipient * if the message will be cached by that recipient or * gatewayed via a protocol which requires a Date. * * If we didn't get a Date header, we assign one here. */ http_TimeHeader(bo->beresp, "Date: ", now); } /* * These two headers can be spread over multiple actual headers * and we rely on their content outside of VCL, so collect them * into one line here. */ http_CollectHdr(bo->beresp, H_Cache_Control); http_CollectHdr(bo->beresp, H_Vary); /* * Figure out how the fetch is supposed to happen, before the * headers are adultered by VCL */ if (!strcasecmp(http_GetMethod(bo->bereq), "head")) { /* * A HEAD request can never have a body in the reply, * no matter what the headers might say. * [RFC2516 4.3 p33] */ wrk->stats->fetch_head++; bo->htc->body_status = BS_NONE; } else if (http_GetStatus(bo->beresp) <= 199) { /* * 1xx responses never have a body. * [RFC2616 4.3 p33] * ... but we should never see them. */ wrk->stats->fetch_1xx++; bo->htc->body_status = BS_ERROR; } else if (http_IsStatus(bo->beresp, 204)) { /* * 204 is "No Content", obviously don't expect a body. * [RFC7230 3.3.1 p28 and 3.3.2 p30] */ wrk->stats->fetch_204++; if (http_GetHdr(bo->beresp, H_Content_Length, NULL) || http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL)) bo->htc->body_status = BS_ERROR; else bo->htc->body_status = BS_NONE; } else if (http_IsStatus(bo->beresp, 304)) { /* * 304 is "Not Modified" it has no body. * [RFC2616 10.3.5 p63] */ wrk->stats->fetch_304++; bo->htc->body_status = BS_NONE; } else if (bo->htc->body_status == BS_CHUNKED) { wrk->stats->fetch_chunked++; } else if (bo->htc->body_status == BS_LENGTH) { assert(bo->htc->content_length > 0); wrk->stats->fetch_length++; } else if (bo->htc->body_status == BS_EOF) { wrk->stats->fetch_eof++; } else if (bo->htc->body_status == BS_ERROR) { wrk->stats->fetch_bad++; } else if (bo->htc->body_status == BS_NONE) { wrk->stats->fetch_none++; } else { WRONG("wrong bodystatus"); } if (bo->htc->body_status == BS_ERROR) { bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } /* * What does RFC2616 think about TTL ? */ EXP_Clr(&bo->fetch_objcore->exp); RFC2616_Ttl(bo, now); /* private objects have negative TTL */ if (bo->fetch_objcore->flags & OC_F_PRIVATE) bo->fetch_objcore->exp.ttl = -1.; AZ(bo->do_esi); AZ(bo->was_304); if (http_IsStatus(bo->beresp, 304)) { if (bo->stale_oc != NULL && ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) { if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) { /* * If we changed the gzip status of the object * the stored Content_Encoding controls we * must weaken any new ETag we get. */ http_Unset(bo->beresp, H_Content_Encoding); RFC2616_Weaken_Etag(bo->beresp); } http_Unset(bo->beresp, H_Content_Length); HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); assert(http_IsStatus(bo->beresp, 200)); bo->was_304 = 1; } else if (!bo->do_pass) { /* * Backend sent unallowed 304 */ VSLb(bo->vsl, SLT_Error, "304 response but not conditional fetch"); bo->htc->doclose = SC_RX_BAD; VDI_Finish(bo->wrk, bo); return (F_STP_FAIL); } } bo->vfc->bo = bo; bo->vfc->oc = bo->fetch_objcore; bo->vfc->wrk = bo->wrk; bo->vfc->http = bo->beresp; bo->vfc->esi_req = bo->bereq; VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); if (wrk->handling == VCL_RET_ABANDON) { bo->htc->doclose = SC_RESP_CLOSE; VDI_Finish(bo->wrk, bo); return (F_STP_FAIL); } if (wrk->handling == VCL_RET_RETRY) { if (bo->htc->body_status != BS_NONE) bo->htc->doclose = SC_RESP_CLOSE; if (bo->director_state != DIR_S_NULL) VDI_Finish(bo->wrk, bo); if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, delivering 503"); assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } assert(bo->state == BOS_REQ_DONE); if (bo->do_esi) bo->do_stream = 0; if (bo->do_pass || bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; assert(wrk->handling == VCL_RET_DELIVER); return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH); }
static enum req_fsm_nxt cnt_recv(struct worker *wrk, struct req *req) { unsigned recv_handling; struct SHA256Context sha256ctx; const char *xff; const char *ci, *cp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(req->vcl); AZ(req->objcore); AZ(req->err_code); AZ(isnan(req->t_first)); AZ(isnan(req->t_prev)); AZ(isnan(req->t_req)); ci = SES_Get_String_Attr(req->sp, SA_CLIENT_IP); cp = SES_Get_String_Attr(req->sp, SA_CLIENT_PORT); VSLb(req->vsl, SLT_ReqStart, "%s %s", ci, cp); http_VSL_log(req->http); if (req->restarts == 0) { /* * This really should be done earlier, but we want to capture * it in the VSL log. */ http_CollectHdr(req->http, H_X_Forwarded_For); if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { http_Unset(req->http, H_X_Forwarded_For); http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff, ci); } else { http_PrintfHeader(req->http, "X-Forwarded-For: %s", ci); } } /* By default we use the first backend */ AZ(req->director_hint); req->director_hint = VCL_DefaultDirector(req->vcl); AN(req->director_hint); req->d_ttl = -1; req->disable_esi = 0; req->hash_always_miss = 0; req->hash_ignore_busy = 0; req->client_identity = NULL; http_CollectHdr(req->http, H_Cache_Control); VFP_Setup(req->htc->vfc); req->htc->vfc->http = req->http; req->htc->vfc->wrk = wrk; if (req->transport->req_body != NULL) { req->transport->req_body(req); if (req->req_body_status == REQ_BODY_FAIL) { req->doclose = SC_OVERLOAD; return (REQ_FSM_DONE); } } VCL_recv_method(req->vcl, wrk, req, NULL, NULL); /* Attempts to cache req.body may fail */ if (req->req_body_status == REQ_BODY_FAIL) { req->doclose = SC_RX_BODY; return (REQ_FSM_DONE); } recv_handling = wrk->handling; /* We wash the A-E header here for the sake of VRY */ if (cache_param->http_gzip_support && (recv_handling != VCL_RET_PIPE) && (recv_handling != VCL_RET_PASS)) { if (RFC2616_Req_Gzip(req->http)) { http_ForceHeader(req->http, H_Accept_Encoding, "gzip"); } else { http_Unset(req->http, H_Accept_Encoding); } } SHA256_Init(&sha256ctx); VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx); assert(wrk->handling == VCL_RET_LOOKUP); SHA256_Final(req->digest, &sha256ctx); switch(recv_handling) { case VCL_RET_PURGE: req->req_step = R_STP_PURGE; return (REQ_FSM_MORE); case VCL_RET_HASH: req->req_step = R_STP_LOOKUP; return (REQ_FSM_MORE); case VCL_RET_PIPE: if (req->esi_level == 0) { req->req_step = R_STP_PIPE; return (REQ_FSM_MORE); } VSLb(req->vsl, SLT_VCL_Error, "vcl_recv{} returns pipe for ESI included object." " Doing pass."); req->req_step = R_STP_PASS; return (REQ_FSM_DONE); case VCL_RET_PASS: req->req_step = R_STP_PASS; return (REQ_FSM_MORE); case VCL_RET_SYNTH: req->req_step = R_STP_SYNTH; return (REQ_FSM_MORE); default: WRONG("Illegal return from vcl_recv{}"); } }
static enum req_fsm_nxt cnt_transmit(struct worker *wrk, struct req *req) { struct boc *boc; const char *r; uint16_t status; int sendbody; intmax_t clval; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC); /* Grab a ref to the bo if there is one */ boc = HSH_RefBoc(req->objcore); clval = http_GetContentLength(req->resp); if (boc != NULL) req->resp_len = clval; else req->resp_len = ObjGetLen(req->wrk, req->objcore); req->res_mode = 0; /* RFC 7230, 3.3.3 */ status = http_GetStatus(req->resp); if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) { if (req->objcore->flags & OC_F_PASS) sendbody = -1; else sendbody = 0; } else if (status < 200 || status == 204 || status == 304) { req->resp_len = -1; sendbody = 0; } else sendbody = 1; if (sendbody >= 0) { if (!req->disable_esi && req->resp_len != 0 && ObjHasAttr(wrk, req->objcore, OA_ESIDATA)) VDP_push(req, VDP_ESI, NULL, 0, "ESI"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) VDP_push(req, VDP_gunzip, NULL, 1, "GUZ"); if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) { http_SetHeader(req->resp, "Accept-Ranges: bytes"); if (sendbody && http_GetHdr(req->http, H_Range, &r)) VRG_dorange(req, r); } } if (sendbody < 0) { /* Don't touch pass+HEAD C-L */ sendbody = 0; } else if (clval >= 0 && clval == req->resp_len) { /* Reuse C-L header */ } else { http_Unset(req->resp, H_Content_Length); if (req->resp_len >= 0 && sendbody) http_PrintfHeader(req->resp, "Content-Length: %jd", req->resp_len); } req->transport->deliver(req, boc, sendbody); VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) { if (boc != NULL) { HSH_Abandon(req->objcore); ObjWaitState(req->objcore, BOS_FINISHED); } ObjSlim(wrk, req->objcore); } if (boc != NULL) HSH_DerefBoc(wrk, req->objcore); (void)HSH_DerefObjCore(wrk, &req->objcore); http_Teardown(req->resp); return (REQ_FSM_DONE); }
static enum req_fsm_nxt cnt_synth(struct worker *wrk, struct req *req) { struct http *h; double now; struct vsb *synth_body; ssize_t sz, szl; uint8_t *ptr; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); wrk->stats->s_synth++; now = W_TIM_real(wrk); VSLb_ts_req(req, "Process", now); if (req->err_code < 100 || req->err_code > 999) req->err_code = 501; HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod); h = req->resp; http_TimeHeader(h, "Date: ", now); http_SetHeader(h, "Server: Varnish"); http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid)); http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason); synth_body = VSB_new_auto(); AN(synth_body); VCL_synth_method(req->vcl, wrk, req, NULL, synth_body); AZ(VSB_finish(synth_body)); http_Unset(h, H_Content_Length); http_PrintfHeader(req->resp, "Content-Length: %zd", VSB_len(synth_body)); /* Discard any lingering request body before delivery */ (void)VRB_Ignore(req); if (wrk->handling == VCL_RET_RESTART) { HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod); VSB_destroy(&synth_body); req->req_step = R_STP_RESTART; return (REQ_FSM_MORE); } assert(wrk->handling == VCL_RET_DELIVER); req->objcore = HSH_Private(wrk); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); szl = -1; if (STV_NewObject(wrk, req->objcore, TRANSIENT_STORAGE, 1024)) { szl = VSB_len(synth_body); assert(szl >= 0); sz = szl; if (sz > 0 && ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) { memcpy(ptr, VSB_data(synth_body), szl); ObjExtend(wrk, req->objcore, szl); } else if (sz > 0) { szl = -1; } } if (szl >= 0) AZ(ObjSetU64(wrk, req->objcore, OA_LEN, szl)); HSH_DerefBoc(wrk, req->objcore); VSB_destroy(&synth_body); if (szl < 0) { VSLb(req->vsl, SLT_Error, "Could not get storage"); req->doclose = SC_OVERLOAD; VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); (void)HSH_DerefObjCore(wrk, &req->objcore); http_Teardown(req->resp); return (REQ_FSM_DONE); } req->req_step = R_STP_TRANSMIT; return (REQ_FSM_MORE); }
static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); /* * The VCL variables beresp.do_g[un]zip tells us how we want the * object processed before it is stored. * * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * */ /* We do nothing unless the param is set */ if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; if (bo->htc->content_length == 0) http_Unset(bo->beresp, H_Content_Encoding); if (bo->htc->body_status != BS_NONE) { bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(bo->is_gzip == 0 || bo->is_gunzip == 0); } /* We won't gunzip unless it is non-empty and gzip'ed */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gunzip && !bo->is_gzip)) bo->do_gunzip = 0; /* We wont gzip unless it is non-empty and ungziped */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0 || (bo->do_gzip && !bo->is_gunzip)) bo->do_gzip = 0; /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); if (bo->htc->content_length != 0) { if (bo->do_esi && bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); } else if (bo->do_esi) { (void)VFP_Push(bo->vfc, &vfp_esi, 1); } else if (bo->do_gzip) { (void)VFP_Push(bo->vfc, &vfp_gzip, 1); } else if (bo->is_gzip && !bo->do_gunzip) { (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); } } if (bo->fetch_objcore->flags & OC_F_PRIVATE) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ if (bo->htc->body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch pipeline failed to open"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->htc->doclose = SC_RX_BODY; VDI_Finish(bo->wrk, bo); return (F_STP_ERROR); } if (bo->do_esi) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) AZ(VDI_GetBody(bo->wrk, bo)); assert(bo->refcount >= 1); assert(bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); vbf_fetch_body_helper(bo); } if (bo->vfc->failed) { VDI_Finish(bo->wrk, bo); if (!bo->do_stream) { assert(bo->state < BOS_STREAM); // XXX: doclose = ? return (F_STP_ERROR); } else { return (F_STP_FAIL); } } if (bo->do_stream) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); HSH_Unbusy(wrk, bo->fetch_objcore); } /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->stale_oc != NULL) EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0); return (F_STP_DONE); }