void
VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs,
const char *p, ...)
{
struct http *hp;
va_list ap;
const char *b;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
AN(hs);
AN(hs->what);
hp = vrt_selecthttp(ctx, hs->where);
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
va_start(ap, p);
if (p == vrt_magic_string_unset) {
http_Unset(hp, hs->what);
} else {
b = VRT_String(hp->ws, hs->what + 1, p, ap);
if (b == NULL) {
VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1);
} else {
http_Unset(hp, hs->what);
http_SetHeader(hp, b);
}
}
va_end(ap);
}
vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
{
struct vef_priv *vef;
CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC);
CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
if (http_GetStatus(vc->resp) == 206) {
VSLb(vc->wrk->vsl, SLT_VCL_Error,
"Attempted ESI on partial (206) response");
return (VFP_ERROR);
}
ALLOC_OBJ(vef, VEF_MAGIC);
if (vef == NULL)
return (VFP_ERROR);
vc->obj_flags |= OF_GZIPED | OF_CHGGZIP | OF_ESIPROC;
vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E");
vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef);
vef->ibuf_sz = cache_param->gzip_buffer;
vef->ibuf = calloc(1L, vef->ibuf_sz);
if (vef->ibuf == NULL)
return (vfp_esi_end(vc, vef, VFP_ERROR));
vef->ibuf_i = vef->ibuf;
vef->ibuf_o = vef->ibuf;
vfe->priv1 = vef;
RFC2616_Weaken_Etag(vc->resp);
http_Unset(vc->resp, H_Content_Length);
http_Unset(vc->resp, H_Content_Encoding);
http_SetHeader(vc->resp, "Content-Encoding: gzip");
RFC2616_Vary_AE(vc->resp);
return (VFP_OK);
}
vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
{
struct vef_priv *vef;
CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
CHECK_OBJ_NOTNULL(vc->esi_req, HTTP_MAGIC);
CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
ALLOC_OBJ(vef, VEF_MAGIC);
if (vef == NULL)
return (VFP_ERROR);
vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E");
vef->vep = VEP_Init(vc, vc->esi_req, vfp_vep_callback, vef);
vef->ibuf_sz = cache_param->gzip_buffer;
vef->ibuf = calloc(1L, vef->ibuf_sz);
if (vef->ibuf == NULL)
return (vfp_esi_end(vc, vef, VFP_ERROR));
vef->ibuf_i = vef->ibuf;
vef->ibuf_o = vef->ibuf;
vfe->priv1 = vef;
RFC2616_Weaken_Etag(vc->http);
http_Unset(vc->http, H_Content_Length);
http_Unset(vc->http, H_Content_Encoding);
http_SetHeader(vc->http, "Content-Encoding: gzip");
RFC2616_Vary_AE(vc->http);
return (VFP_OK);
}
static void
cnt_vdp(struct req *req, struct busyobj *bo)
{
const char *r;
uint16_t status;
int wantbody;
CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
req->res_mode = 0;
wantbody = 1;
status = http_GetStatus(req->resp);
if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) {
wantbody = 0;
} else if (status < 200 || status == 204) {
req->resp_len = 0;
http_Unset(req->resp, H_Content_Length);
wantbody = 0;
} else if (status == 304) {
http_Unset(req->resp, H_Content_Length);
wantbody = 0;
} else if (bo != NULL)
req->resp_len = http_GetContentLength(req->resp);
else
req->resp_len = ObjGetLen(req->wrk, req->objcore);
/*
* Determine ESI status first. Not dependent on wantbody, because
* we want ESI to supress C-L in HEAD too.
*/
if (!req->disable_esi && req->resp_len != 0 && wantbody &&
ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) {
req->res_mode |= RES_ESI;
RFC2616_Weaken_Etag(req->resp);
req->resp_len = -1;
VDP_push(req, VDP_ESI, NULL, 0);
}
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http)) {
req->res_mode |= RES_GUNZIP;
VDP_push(req, VDP_gunzip, NULL, 1);
}
/*
* Range comes after the others and pushes on bottom because
* it can (maybe) generate a correct C-L header.
*/
if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) {
http_SetHeader(req->resp, "Accept-Ranges: bytes");
if (wantbody && http_GetHdr(req->http, H_Range, &r))
VRG_dorange(req, r);
}
req->transport->deliver(req, bo, wantbody);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
char date[40];
struct http *h;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats.s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
VTIM_format(now, date);
http_PrintfHeader(h, "Date: %s", date);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
AZ(req->synth_body);
req->synth_body = VSB_new_auto();
AN(req->synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws);
http_Unset(h, H_Content_Length);
AZ(VSB_finish(req->synth_body));
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
if (http_HdrIs(req->resp, H_Connection, "close"))
req->doclose = SC_RESP_CLOSE;
V1D_Deliver_Synth(req);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
VSB_delete(req->synth_body);
req->synth_body = NULL;
req->err_code = 0;
req->err_reason = NULL;
return (REQ_FSM_DONE);
}
void
VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
{
va_list ap;
const char *p;
struct vsb *vsb;
(void)flags;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
vsb = SMS_Makesynth(sp->wrk->obj);
AN(vsb);
VSB_cat(vsb, str);
va_start(ap, str);
p = va_arg(ap, const char *);
while (p != vrt_magic_string_end) {
if (p == NULL)
p = "(null)";
VSB_cat(vsb, p);
p = va_arg(ap, const char *);
}
va_end(ap);
SMS_Finish(sp->wrk->obj);
http_Unset(sp->wrk->obj->http, H_Content_Length);
http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->obj->http,
"Content-Length: %zd", sp->wrk->obj->len);
}
static enum fetch_step
vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
{
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
AN(bo->director);
AZ(bo->vbc);
AZ(bo->should_close);
AZ(bo->storage_hint);
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, HTTP_Bereq);
http_FilterReq(bo->bereq0, bo->req->http,
bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
if (!bo->do_pass) {
// XXX: Forcing GET should happen in vcl_miss{} ?
http_ForceGet(bo->bereq0);
if (cache_param->http_gzip_support) {
/*
* We always ask the backend for gzip, even if the
* client doesn't grok it. We will uncompress for
* the minority of clients which don't.
*/
http_Unset(bo->bereq0, H_Accept_Encoding);
http_SetHeader(bo->bereq0, "Accept-Encoding: gzip");
}
}
return (F_STP_STARTFETCH);
}
uint16_t
HTTP1_DissectRequest(struct http_conn *htc, struct http *hp)
{
uint16_t retval;
const char *p;
const char *b = NULL, *e;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
retval = http1_splitline(hp, htc,
HTTP1_Req, cache_param->http_req_hdr_len);
if (retval != 0)
return (retval);
hp->protover = http1_proto_ver(hp);
if (hp->protover == 0)
return (400);
if (http_CountHdr(hp, H_Host) > 1)
return (400);
if (http_CountHdr(hp, H_Content_Length) > 1)
return (400);
/* RFC2616, section 5.2, point 1 */
if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7))
b = hp->hd[HTTP_HDR_URL].b + 7;
else if (FEATURE(FEATURE_HTTPS_SCHEME) &&
!strncasecmp(hp->hd[HTTP_HDR_URL].b, "https://", 8))
b = hp->hd[HTTP_HDR_URL].b + 8;
if (b) {
e = strchr(b, '/');
if (e) {
http_Unset(hp, H_Host);
http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b);
hp->hd[HTTP_HDR_URL].b = e;
}
}
htc->body_status = http1_body_status(hp, htc, 1);
if (htc->body_status == BS_ERROR)
return (400);
p = http_GetMethod(hp);
AN(p);
if (htc->body_status == BS_EOF) {
assert(hp->protover == 10);
/* RFC1945 8.3 p32 and D.1.1 p58 */
if (!strcasecmp(p, "post") || !strcasecmp(p, "put"))
return (400);
htc->body_status = BS_NONE;
}
/* HEAD with a body is a hard error */
if (htc->body_status != BS_NONE && !strcasecmp(p, "head"))
return (400);
return (retval);
}
void
VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
{
va_list ap;
const char *p;
struct vsb *vsb;
(void)flags;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC);
vsb = SMS_Makesynth(ctx->req->obj);
AN(vsb);
va_start(ap, str);
p = str;
while (p != vrt_magic_string_end) {
if (p == NULL)
p = "(null)";
VSB_cat(vsb, p);
p = va_arg(ap, const char *);
}
va_end(ap);
SMS_Finish(ctx->req->obj);
http_Unset(ctx->req->obj->http, H_Content_Length);
}
static int
vbf_figure_out_vfp(struct busyobj *bo)
{
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* No body or no GZIP supprt -> done */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
!cache_param->http_gzip_support) {
http_Unset(bo->beresp, H_Content_Encoding);
bo->do_gzip = bo->do_gunzip = 0;
bo->do_stream = 0;
return (0);
}
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless if it already is gzip'ed */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL)
return (-1);
if (bo->do_esi && (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)))
return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0);
if (bo->do_esi)
return (VFP_Push(bo->vfc, &VFP_esi) == NULL ? -1 : 0);
if (bo->do_gzip)
return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0);
if (bo->is_gzip && !bo->do_gunzip)
return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0);
return (0);
}
V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
{
int err = 0;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
if (sendbody) {
if (http_GetHdr(req->resp, H_Content_Length, NULL))
req->res_mode |= RES_LEN;
else if (req->http->protover == 11) {
req->res_mode |= RES_CHUNKED;
http_SetHeader(req->resp, "Transfer-Encoding: chunked");
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
if (req->doclose) {
if (!http_HdrIs(req->resp, H_Connection, "close")) {
http_Unset(req->resp, H_Connection);
http_SetHeader(req->resp, "Connection: close");
}
} else if (!http_GetHdr(req->resp, H_Connection, NULL))
http_SetHeader(req->resp, "Connection: keep-alive");
if (sendbody && req->resp_len != 0)
VDP_push(req, v1d_bytes, NULL, 1);
AZ(req->wrk->v1l);
V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev);
if (WS_Overflowed(req->ws)) {
v1d_error(req, "workspace_client overflow");
AZ(req->wrk->v1l);
return;
}
req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
if (DO_DEBUG(DBG_FLUSH_HEAD))
(void)V1L_Flush(req->wrk);
if (sendbody && req->resp_len != 0) {
if (req->res_mode & RES_CHUNKED)
V1L_Chunked(req->wrk);
err = VDP_DeliverObj(req);
if (!err && (req->res_mode & RES_CHUNKED))
V1L_EndChunk(req->wrk);
}
if ((V1L_FlushRelease(req->wrk) || err) && req->sp->fd >= 0)
SES_Close(req->sp, SC_REM_CLOSE);
AZ(req->wrk->v1l);
VDP_close(req);
}
uint16_t
HTTP1_DissectRequest(struct http_conn *htc, struct http *hp)
{
uint16_t retval;
const char *p;
const char *b, *e;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
retval = http1_splitline(hp, htc, HTTP1_Req);
if (retval != 0)
return (retval);
hp->protover = http1_proto_ver(hp);
if (hp->protover == 0)
return (400);
if (http_CountHdr(hp, H_Host) > 1)
return (400);
if (http_CountHdr(hp, H_Content_Length) > 1)
return (400);
/* RFC2616, section 5.2, point 1 */
if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) {
b = e = hp->hd[HTTP_HDR_URL].b + 7;
while (*e != '/' && *e != '\0')
e++;
if (*e == '/') {
http_Unset(hp, H_Host);
http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b);
hp->hd[HTTP_HDR_URL].b = e;
}
}
htc->body_status = http1_body_status(hp, htc);
if (htc->body_status == BS_ERROR)
return (400);
p = http_GetMethod(hp);
AN(p);
if (htc->body_status == BS_EOF) {
assert(hp->protover == 10);
/* RFC1945 8.3 p32 and D.1.1 p58 */
if (!strcasecmp(p, "post") || !strcasecmp(p, "put"))
return (400);
htc->body_status = BS_NONE;
}
/* HEAD with a body is a hard error */
if (htc->body_status != BS_NONE && !strcasecmp(p, "head"))
return (400);
return (retval);
}
vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
{
struct vgz *vg;
CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
if (http_HdrIs(vc->http, H_Content_Length, "0")) {
http_Unset(vc->http, H_Content_Encoding);
return (VFP_NULL);
}
if (vfe->vfp->priv2 == VFP_GZIP) {
if (http_GetHdr(vc->http, H_Content_Encoding, NULL))
return (VFP_NULL);
vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
} else {
if (!http_HdrIs(vc->http, H_Content_Encoding, "gzip"))
return (VFP_NULL);
vg = VGZ_NewUngzip(vc->wrk->vsl, vfe->vfp->priv1);
}
if (vg == NULL)
return (VFP_ERROR);
vfe->priv1 = vg;
if (vgz_getmbuf(vg))
return (VFP_ERROR);
VGZ_Ibuf(vg, vg->m_buf, 0);
AZ(vg->m_len);
if (vfe->vfp->priv2 == VFP_GUNZIP || vfe->vfp->priv2 == VFP_GZIP) {
http_Unset(vc->http, H_Content_Encoding);
http_Unset(vc->http, H_Content_Length);
RFC2616_Weaken_Etag(vc->http);
}
if (vfe->vfp->priv2 == VFP_GZIP)
http_SetHeader(vc->http, "Content-Encoding: gzip");
if (vfe->vfp->priv2 == VFP_GZIP || vfe->vfp->priv2 == VFP_TESTGUNZIP)
RFC2616_Vary_AE(vc->http);
return (VFP_OK);
}
void
http_ForceHeader(struct http *to, const char *hdr, const char *val)
{
CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
if (http_HdrIs(to, hdr, val))
return;
http_Unset(to, hdr);
http_PrintfHeader(to, "%s %s", hdr + 1, val);
}
static enum body_status
http1_body_status(struct http *hp, struct http_conn *htc)
{
ssize_t cl;
const char *b;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
htc->content_length = -1;
if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
http_Unset(hp, H_Content_Length);
return (BS_CHUNKED);
}
if (http_GetHdr(hp, H_Transfer_Encoding, &b))
return (BS_ERROR);
cl = http_GetContentLength(hp);
if (cl == -2)
return (BS_ERROR);
if (cl >= 0) {
htc->content_length = cl;
return (cl == 0 ? BS_NONE : BS_LENGTH);
}
if (http_HdrIs(hp, H_Connection, "keep-alive")) {
/*
* Keep alive with neither TE=Chunked or C-Len is impossible.
* We assume a zero length body.
*/
return (BS_NONE);
}
if (http_HdrIs(hp, H_Connection, "close")) {
/*
* In this case, it is safe to just read what comes.
*/
return (BS_EOF);
}
if (hp->protover < 11) {
/*
* With no Connection header, assume EOF.
*/
return (BS_EOF);
}
/*
* Fall back to EOF transfer.
*/
return (BS_EOF);
}
static int
cnt_miss(struct worker *wrk, struct req *req)
{
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(req->obj);
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
http_FilterReq(req, HTTPH_R_FETCH);
http_ForceGet(bo->bereq);
if (cache_param->http_gzip_support) {
/*
* We always ask the backend for gzip, even if the
* client doesn't grok it. We will uncompress for
* the minority of clients which don't.
*/
http_Unset(bo->bereq, H_Accept_Encoding);
http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
}
VCL_miss_method(req);
if (req->handling == VCL_RET_FETCH) {
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
req->req_step = R_STP_FETCH;
return (0);
}
AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
req->objcore = NULL;
http_Teardown(bo->bereq);
VBO_DerefBusyObj(wrk, &req->busyobj);
switch(req->handling) {
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
break;
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
break;
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
break;
default:
WRONG("Illegal action in vcl_miss{}");
}
return (0);
}
void
RFC2616_Vary_AE(struct http *hp)
{
const char *vary;
if (http_GetHdrToken(hp, H_Vary, "Accept-Encoding", NULL, NULL))
return;
if (http_GetHdr(hp, H_Vary, &vary)) {
http_Unset(hp, H_Vary);
http_PrintfHeader(hp, "Vary: %s, Accept-Encoding", vary);
} else {
http_SetHeader(hp, "Vary: Accept-Encoding");
}
}
void
VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
const char *p, ...)
{
struct http *hp;
va_list ap;
char *b;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
hp = vrt_selecthttp(sp, where);
va_start(ap, p);
if (p == NULL) {
http_Unset(hp, hdr);
} else {
b = VRT_String(hp->ws, hdr + 1, p, ap);
if (b == NULL) {
WSP(sp, SLT_LostHeader, "%s", hdr + 1);
} else {
http_Unset(hp, hdr);
http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
}
}
va_end(ap);
}
void
RFC2616_Weaken_Etag(struct http *hp)
{
const char *p;
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
if (!http_GetHdr(hp, H_ETag, &p))
return;
AN(p);
if (p[0] == 'W' && p[1] == '/')
return;
http_Unset(hp, H_ETag);
http_PrintfHeader(hp, "ETag: W/%s", p);
}
static enum fetch_step
vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
{
char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
AN(bo->director);
AZ(bo->vbc);
AZ(bo->should_close);
AZ(bo->storage_hint);
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, HTTP_Bereq);
http_FilterReq(bo->bereq0, bo->req->http,
bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
if (!bo->do_pass) {
// XXX: Forcing GET should happen in vcl_miss{} ?
http_ForceGet(bo->bereq0);
if (cache_param->http_gzip_support) {
/*
* We always ask the backend for gzip, even if the
* client doesn't grok it. We will uncompress for
* the minority of clients which don't.
*/
http_Unset(bo->bereq0, H_Accept_Encoding);
http_SetHeader(bo->bereq0, "Accept-Encoding: gzip");
}
}
if (bo->ims_obj != NULL) {
if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
http_PrintfHeader(bo->bereq0,
"If-Modified-Since: %s", p);
} else if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
http_PrintfHeader(bo->bereq0,
"If-None-Match: %s", p);
} else {
WRONG("Shouldn't have bo->ims_obj");
}
}
return (F_STP_STARTFETCH);
}
uint16_t
HTTP1_DissectRequest(struct req *req)
{
struct http_conn *htc;
struct http *hp;
uint16_t retval;
char *b, *e;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
htc = req->htc;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
hp = req->http;
CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
retval = htc_splitline(hp, htc, 1);
if (retval != 0) {
VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf);
return (retval);
}
htc_proto_ver(hp);
retval = htc_request_check_host_hdr(hp);
if (retval != 0) {
return (retval);
}
/* RFC2616, section 5.2, point 1 */
if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) {
b = e = hp->hd[HTTP_HDR_URL].b + 7;
while (*e != '/' && *e != '\0')
e++;
if (*e == '/') {
http_Unset(hp, H_Host);
http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b);
hp->hd[HTTP_HDR_URL].b = e;
}
}
return (retval);
}
static void
ved_include(struct req *preq, const char *src, const char *host,
struct ecx *ecx)
{
struct worker *wrk;
struct req *req;
enum req_fsm_nxt s;
struct transport xp;
CHECK_OBJ_NOTNULL(preq, REQ_MAGIC);
CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC);
wrk = preq->wrk;
if (preq->esi_level >= cache_param->max_esi_depth)
return;
req = Req_New(wrk, preq->sp);
req->req_body_status = REQ_BODY_NONE;
AZ(req->vsl->wid);
req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER);
VSLb(req->vsl, SLT_Begin, "req %u esi", VXID(preq->vsl->wid));
VSLb(preq->vsl, SLT_Link, "req %u esi", VXID(req->vsl->wid));
req->esi_level = preq->esi_level + 1;
if (preq->esi_level == 0)
assert(preq->top == preq);
else
CHECK_OBJ_NOTNULL(preq->top, REQ_MAGIC);
req->top = preq->top;
HTTP_Copy(req->http0, preq->http0);
req->http0->ws = req->ws;
req->http0->vsl = req->vsl;
req->http0->logtag = SLT_ReqMethod;
req->http0->conds = 0;
http_SetH(req->http0, HTTP_HDR_URL, src);
if (host != NULL && *host != '\0') {
http_Unset(req->http0, H_Host);
http_SetHeader(req->http0, host);
}
http_ForceField(req->http0, HTTP_HDR_METHOD, "GET");
http_ForceField(req->http0, HTTP_HDR_PROTO, "HTTP/1.1");
/* Don't allow conditionalss, we can't use a 304 */
http_Unset(req->http0, H_If_Modified_Since);
http_Unset(req->http0, H_If_None_Match);
/* Don't allow Range */
http_Unset(req->http0, H_Range);
/* Set Accept-Encoding according to what we want */
http_Unset(req->http0, H_Accept_Encoding);
if (ecx->isgzip)
http_ForceHeader(req->http0, H_Accept_Encoding, "gzip");
/* Client content already taken care of */
http_Unset(req->http0, H_Content_Length);
/* Reset request to status before we started messing with it */
HTTP_Copy(req->http, req->http0);
req->vcl = preq->vcl;
preq->vcl = NULL;
req->wrk = preq->wrk;
/*
* XXX: We should decide if we should cache the director
* XXX: or not (for session/backend coupling). Until then
* XXX: make sure we don't trip up the check in vcl_recv.
*/
req->req_step = R_STP_RECV;
req->t_req = preq->t_req;
assert(isnan(req->t_first));
assert(isnan(req->t_prev));
INIT_OBJ(&xp, TRANSPORT_MAGIC);
xp.deliver = VED_Deliver;
req->transport = &xp;
req->transport_priv = ecx;
THR_SetRequest(req);
VSLb_ts_req(req, "Start", W_TIM_real(wrk));
req->ws_req = WS_Snapshot(req->ws);
while (1) {
req->wrk = wrk;
s = CNT_Request(wrk, req);
if (s == REQ_FSM_DONE)
break;
DSL(DBG_WAITINGLIST, req->vsl->wid,
"loop waiting for ESI (%d)", (int)s);
assert(s == REQ_FSM_DISEMBARK);
AZ(req->wrk);
(void)usleep(10000);
}
VRTPRIV_dynamic_kill(req->sp->privs, (uintptr_t)req);
CNT_AcctLogCharge(wrk->stats, req);
VSL_End(req->vsl);
preq->vcl = req->vcl;
req->vcl = NULL;
req->wrk = NULL;
THR_SetRequest(preq);
Req_Release(req);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage);
bo->storage = bo->do_pass ? stv_transient : STV_next();
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
AZ(bo->htc);
VFP_Setup(bo->vfc, wrk);
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->resp = bo->beresp;
bo->vfc->req = bo->bereq;
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
if (bo->fetch_objcore->flags & OC_F_PRIVATE) {
/* private objects have negative TTL */
bo->fetch_objcore->t_origin = now;
bo->fetch_objcore->ttl = -1.;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
} else {
/* What does RFC2616 think about TTL ? */
RFC2616_Ttl(bo, now,
&bo->fetch_objcore->t_origin,
&bo->fetch_objcore->ttl,
&bo->fetch_objcore->grace,
&bo->fetch_objcore->keep
);
}
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
}
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
if (bo->fetch_objcore->boc->state != BOS_REQ_DONE) {
bo->req = NULL;
ObjSetState(wrk, bo->fetch_objcore, BOS_REQ_DONE);
}
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_PASS) {
bo->fetch_objcore->flags |= OC_F_HFP;
bo->uncacheable = 1;
wrk->handling = VCL_RET_DELIVER;
}
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
VDP_gunzip(struct req *req, enum vdp_action act, void **priv,
const void *ptr, ssize_t len)
{
enum vgzret_e vr;
ssize_t dl;
const void *dp;
struct worker *wrk;
struct vgz *vg;
const char *p;
uint64_t u;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk = req->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
if (act == VDP_INIT) {
vg = VGZ_NewUngzip(req->vsl, "U D -");
AN(vg);
if (vgz_getmbuf(vg)) {
(void)VGZ_Destroy(&vg);
return (-1);
}
req->res_mode |= RES_GUNZIP;
VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
*priv = vg;
http_Unset(req->resp, H_Content_Encoding);
req->resp_len = -1;
if (req->objcore->boc != NULL)
return (0); /* No idea about length (yet) */
p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
if (p == NULL || dl != 32)
return (0); /* No OA_GZIPBITS yet */
u = vbe64dec(p + 24);
/*
* If the size is non-zero AND we are the top
* VDP (ie: no ESI), we know what size the output will be.
*/
if (u != 0 && VTAILQ_FIRST(&req->vdp)->func == VDP_gunzip)
req->resp_len = u;
return (0);
}
CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
AN(vg->m_buf);
if (act == VDP_FINI) {
/* NB: Gunzip'ing may or may not have completed successfully. */
AZ(len);
(void)VGZ_Destroy(&vg);
*priv = NULL;
return (0);
}
if (len == 0)
return (0);
VGZ_Ibuf(vg, ptr, len);
do {
vr = VGZ_Gunzip(vg, &dp, &dl);
vg->m_len += dl;
if (vr < VGZ_OK)
return (-1);
if (vg->m_len == vg->m_sz || vr != VGZ_OK) {
if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len))
return (req->vdp_retval);
vg->m_len = 0;
VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
}
} while (!VGZ_IbufEmpty(vg));
assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END);
return (0);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
AZ(bo->htc);
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
*/
if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
wrk->stats->fetch_head++;
bo->htc->body_status = BS_NONE;
} else if (http_GetStatus(bo->beresp) <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
* ... but we should never see them.
*/
wrk->stats->fetch_1xx++;
bo->htc->body_status = BS_ERROR;
} else if (http_IsStatus(bo->beresp, 204)) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC7230 3.3.1 p28 and 3.3.2 p30]
*/
wrk->stats->fetch_204++;
if (http_GetHdr(bo->beresp, H_Content_Length, NULL) ||
http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
bo->htc->body_status = BS_ERROR;
else
bo->htc->body_status = BS_NONE;
} else if (http_IsStatus(bo->beresp, 304)) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
wrk->stats->fetch_304++;
bo->htc->body_status = BS_NONE;
} else if (bo->htc->body_status == BS_CHUNKED) {
wrk->stats->fetch_chunked++;
} else if (bo->htc->body_status == BS_LENGTH) {
assert(bo->htc->content_length > 0);
wrk->stats->fetch_length++;
} else if (bo->htc->body_status == BS_EOF) {
wrk->stats->fetch_eof++;
} else if (bo->htc->body_status == BS_ERROR) {
wrk->stats->fetch_bad++;
} else if (bo->htc->body_status == BS_NONE) {
wrk->stats->fetch_none++;
} else {
WRONG("wrong bodystatus");
}
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
}
bo->vfc->bo = bo;
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->wrk = bo->wrk;
bo->vfc->http = bo->beresp;
bo->vfc->esi_req = bo->bereq;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
const char *xff;
const char *ci, *cp;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
AN(req->vcl);
AZ(req->objcore);
AZ(req->err_code);
AZ(isnan(req->t_first));
AZ(isnan(req->t_prev));
AZ(isnan(req->t_req));
ci = SES_Get_String_Attr(req->sp, SA_CLIENT_IP);
cp = SES_Get_String_Attr(req->sp, SA_CLIENT_PORT);
VSLb(req->vsl, SLT_ReqStart, "%s %s", ci, cp);
http_VSL_log(req->http);
if (req->restarts == 0) {
/*
* This really should be done earlier, but we want to capture
* it in the VSL log.
*/
http_CollectHdr(req->http, H_X_Forwarded_For);
if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
http_Unset(req->http, H_X_Forwarded_For);
http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s",
xff, ci);
} else {
http_PrintfHeader(req->http, "X-Forwarded-For: %s", ci);
}
}
/* By default we use the first backend */
AZ(req->director_hint);
req->director_hint = VCL_DefaultDirector(req->vcl);
AN(req->director_hint);
req->d_ttl = -1;
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
http_CollectHdr(req->http, H_Cache_Control);
VFP_Setup(req->htc->vfc);
req->htc->vfc->http = req->http;
req->htc->vfc->wrk = wrk;
if (req->transport->req_body != NULL) {
req->transport->req_body(req);
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_OVERLOAD;
return (REQ_FSM_DONE);
}
}
VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
/* Attempts to cache req.body may fail */
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_RX_BODY;
return (REQ_FSM_DONE);
}
recv_handling = wrk->handling;
/* We wash the A-E header here for the sake of VRY */
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_ForceHeader(req->http, H_Accept_Encoding, "gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
SHA256_Init(&sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, &sha256ctx);
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
static enum req_fsm_nxt
cnt_transmit(struct worker *wrk, struct req *req)
{
struct boc *boc;
const char *r;
uint16_t status;
int sendbody;
intmax_t clval;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
/* Grab a ref to the bo if there is one */
boc = HSH_RefBoc(req->objcore);
clval = http_GetContentLength(req->resp);
if (boc != NULL)
req->resp_len = clval;
else
req->resp_len = ObjGetLen(req->wrk, req->objcore);
req->res_mode = 0;
/* RFC 7230, 3.3.3 */
status = http_GetStatus(req->resp);
if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) {
if (req->objcore->flags & OC_F_PASS)
sendbody = -1;
else
sendbody = 0;
} else if (status < 200 || status == 204 || status == 304) {
req->resp_len = -1;
sendbody = 0;
} else
sendbody = 1;
if (sendbody >= 0) {
if (!req->disable_esi && req->resp_len != 0 &&
ObjHasAttr(wrk, req->objcore, OA_ESIDATA))
VDP_push(req, VDP_ESI, NULL, 0, "ESI");
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http))
VDP_push(req, VDP_gunzip, NULL, 1, "GUZ");
if (cache_param->http_range_support &&
http_IsStatus(req->resp, 200)) {
http_SetHeader(req->resp, "Accept-Ranges: bytes");
if (sendbody && http_GetHdr(req->http, H_Range, &r))
VRG_dorange(req, r);
}
}
if (sendbody < 0) {
/* Don't touch pass+HEAD C-L */
sendbody = 0;
} else if (clval >= 0 && clval == req->resp_len) {
/* Reuse C-L header */
} else {
http_Unset(req->resp, H_Content_Length);
if (req->resp_len >= 0 && sendbody)
http_PrintfHeader(req->resp,
"Content-Length: %jd", req->resp_len);
}
req->transport->deliver(req, boc, sendbody);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) {
if (boc != NULL) {
HSH_Abandon(req->objcore);
ObjWaitState(req->objcore, BOS_FINISHED);
}
ObjSlim(wrk, req->objcore);
}
if (boc != NULL)
HSH_DerefBoc(wrk, req->objcore);
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
static enum req_fsm_nxt
cnt_synth(struct worker *wrk, struct req *req)
{
struct http *h;
double now;
struct vsb *synth_body;
ssize_t sz, szl;
uint8_t *ptr;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
wrk->stats->s_synth++;
now = W_TIM_real(wrk);
VSLb_ts_req(req, "Process", now);
if (req->err_code < 100 || req->err_code > 999)
req->err_code = 501;
HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
h = req->resp;
http_TimeHeader(h, "Date: ", now);
http_SetHeader(h, "Server: Varnish");
http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
synth_body = VSB_new_auto();
AN(synth_body);
VCL_synth_method(req->vcl, wrk, req, NULL, synth_body);
AZ(VSB_finish(synth_body));
http_Unset(h, H_Content_Length);
http_PrintfHeader(req->resp, "Content-Length: %zd",
VSB_len(synth_body));
/* Discard any lingering request body before delivery */
(void)VRB_Ignore(req);
if (wrk->handling == VCL_RET_RESTART) {
HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
VSB_destroy(&synth_body);
req->req_step = R_STP_RESTART;
return (REQ_FSM_MORE);
}
assert(wrk->handling == VCL_RET_DELIVER);
req->objcore = HSH_Private(wrk);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
szl = -1;
if (STV_NewObject(wrk, req->objcore, TRANSIENT_STORAGE, 1024)) {
szl = VSB_len(synth_body);
assert(szl >= 0);
sz = szl;
if (sz > 0 &&
ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) {
memcpy(ptr, VSB_data(synth_body), szl);
ObjExtend(wrk, req->objcore, szl);
} else if (sz > 0) {
szl = -1;
}
}
if (szl >= 0)
AZ(ObjSetU64(wrk, req->objcore, OA_LEN, szl));
HSH_DerefBoc(wrk, req->objcore);
VSB_destroy(&synth_body);
if (szl < 0) {
VSLb(req->vsl, SLT_Error, "Could not get storage");
req->doclose = SC_OVERLOAD;
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
req->req_step = R_STP_TRANSMIT;
return (REQ_FSM_MORE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
const char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
if (bo->htc->content_length == 0)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->htc->body_status != BS_NONE) {
bo->is_gzip =
http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip =
!http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
}
/* We won't gunzip unless it is non-empty and gzip'ed */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gunzip && !bo->is_gzip))
bo->do_gunzip = 0;
/* We wont gzip unless it is non-empty and ungziped */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gzip && !bo->is_gunzip))
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);
if (bo->htc->content_length != 0) {
if (bo->do_esi && bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
(void)VFP_Push(bo->vfc, &vfp_esi, 1);
} else if (bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
}
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc->body_status == BS_NONE)
bo->do_stream = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (bo->do_esi)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);
if (bo->do_gzip || bo->do_gunzip)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);
if (http_IsStatus(bo->beresp, 200) && (
http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
http_GetHdr(bo->beresp, H_ETag, &p)))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);
if (bo->htc->body_status != BS_NONE)
AZ(VDI_GetBody(bo->wrk, bo));
assert(bo->refcount >= 1);
assert(bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(wrk, bo->fetch_objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc->body_status, body_status_2str(bo->htc->body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc->body_status != BS_NONE) {
assert(bo->htc->body_status != BS_ERROR);
vbf_fetch_body_helper(bo);
}
if (bo->vfc->failed) {
VDI_Finish(bo->wrk, bo);
if (!bo->do_stream) {
assert(bo->state < BOS_STREAM);
// XXX: doclose = ?
return (F_STP_ERROR);
} else {
return (F_STP_FAIL);
}
}
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(wrk, bo->fetch_objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->stale_oc != NULL)
EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);
return (F_STP_DONE);
}
