vmod_collect(struct sess *sp, enum gethdr_e e, const char *h)
{
(void)e;
(void)sp;
(void)h;
if (e == HDR_REQ)
http_CollectHdr(sp->req->http, h);
else if (e == HDR_BERESP && sp->wrk->busyobj != NULL)
http_CollectHdr(sp->wrk->busyobj->beresp, h);
}
vmod_collect(const struct vrt_ctx *ctx, VCL_HEADER hdr)
{
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
if (hdr->where == HDR_REQ)
http_CollectHdr(ctx->http_req, hdr->what);
else if (hdr->where == HDR_BEREQ)
http_CollectHdr(ctx->http_bereq, hdr->what);
else if (hdr->where == HDR_BERESP)
http_CollectHdr(ctx->http_beresp, hdr->what);
else if (hdr->where == HDR_RESP)
http_CollectHdr(ctx->http_resp, hdr->what);
}
vmod_collect(VRT_CTX, VCL_HEADER hdr)
{
struct http *hp;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
hp = VRT_selecthttp(ctx, hdr->where);
http_CollectHdr(hp, hdr->what);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage);
bo->storage = bo->do_pass ? stv_transient : STV_next();
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
AZ(bo->htc);
VFP_Setup(bo->vfc, wrk);
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->resp = bo->beresp;
bo->vfc->req = bo->bereq;
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
if (bo->fetch_objcore->flags & OC_F_PRIVATE) {
/* private objects have negative TTL */
bo->fetch_objcore->t_origin = now;
bo->fetch_objcore->ttl = -1.;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
} else {
/* What does RFC2616 think about TTL ? */
RFC2616_Ttl(bo, now,
&bo->fetch_objcore->t_origin,
&bo->fetch_objcore->ttl,
&bo->fetch_objcore->grace,
&bo->fetch_objcore->keep
);
}
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
}
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
if (bo->fetch_objcore->boc->state != BOS_REQ_DONE) {
bo->req = NULL;
ObjSetState(wrk, bo->fetch_objcore, BOS_REQ_DONE);
}
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_PASS) {
bo->fetch_objcore->flags |= OC_F_HFP;
bo->uncacheable = 1;
wrk->handling = VCL_RET_DELIVER;
}
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum fetch_step
vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
{
int i;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
xxxassert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
if (!bo->do_pass)
vbf_release_req(bo); /* XXX: retry ?? */
assert(bo->state == BOS_INVALID);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
if (bo->do_pass)
vbf_release_req(bo); /* XXX : retry ?? */
AZ(bo->req);
if (i) {
AZ(bo->vbc);
bo->err_code = 503;
http_SetH(bo->beresp, HTTP_HDR_PROTO, "HTTP/1.1");
http_SetResp(bo->beresp,
"HTTP/1.1", 503, "Backend fetch failed");
http_SetHeader(bo->beresp, "Content-Length: 0");
http_SetHeader(bo->beresp, "Connection: close");
} else {
AN(bo->vbc);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
bo->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
bo->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(bo);
/* pass from vclrecv{} has negative TTL */
if (bo->fetch_objcore->objhead == NULL)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_DELIVER)
return (F_STP_FETCH);
if (wrk->handling == VCL_RET_RETRY) {
assert(bo->state == BOS_INVALID);
bo->retries++;
if (bo->retries <= cache_param->max_retries) {
VDI_CloseFd(&bo->vbc);
return (F_STP_STARTFETCH);
}
// XXX: wrk->handling = VCL_RET_SYNTH;
}
return (F_STP_NOTYET);
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
const char *xff;
const char *ci, *cp;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
AN(req->vcl);
AZ(req->objcore);
AZ(req->err_code);
AZ(isnan(req->t_first));
AZ(isnan(req->t_prev));
AZ(isnan(req->t_req));
ci = SES_Get_String_Attr(req->sp, SA_CLIENT_IP);
cp = SES_Get_String_Attr(req->sp, SA_CLIENT_PORT);
VSLb(req->vsl, SLT_ReqStart, "%s %s", ci, cp);
http_VSL_log(req->http);
if (req->restarts == 0) {
/*
* This really should be done earlier, but we want to capture
* it in the VSL log.
*/
http_CollectHdr(req->http, H_X_Forwarded_For);
if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
http_Unset(req->http, H_X_Forwarded_For);
http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s",
xff, ci);
} else {
http_PrintfHeader(req->http, "X-Forwarded-For: %s", ci);
}
}
/* By default we use the first backend */
AZ(req->director_hint);
req->director_hint = VCL_DefaultDirector(req->vcl);
AN(req->director_hint);
req->d_ttl = -1;
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
http_CollectHdr(req->http, H_Cache_Control);
VFP_Setup(req->htc->vfc);
req->htc->vfc->http = req->http;
req->htc->vfc->wrk = wrk;
if (req->transport->req_body != NULL) {
req->transport->req_body(req);
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_OVERLOAD;
return (REQ_FSM_DONE);
}
}
VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
/* Attempts to cache req.body may fail */
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_RX_BODY;
return (REQ_FSM_DONE);
}
recv_handling = wrk->handling;
/* We wash the A-E header here for the sake of VRY */
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_ForceHeader(req->http, H_Accept_Encoding, "gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
SHA256_Init(&sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, &sha256ctx);
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
AZ(bo->htc);
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
*/
if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
wrk->stats->fetch_head++;
bo->htc->body_status = BS_NONE;
} else if (http_GetStatus(bo->beresp) <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
* ... but we should never see them.
*/
wrk->stats->fetch_1xx++;
bo->htc->body_status = BS_ERROR;
} else if (http_IsStatus(bo->beresp, 204)) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC7230 3.3.1 p28 and 3.3.2 p30]
*/
wrk->stats->fetch_204++;
if (http_GetHdr(bo->beresp, H_Content_Length, NULL) ||
http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
bo->htc->body_status = BS_ERROR;
else
bo->htc->body_status = BS_NONE;
} else if (http_IsStatus(bo->beresp, 304)) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
wrk->stats->fetch_304++;
bo->htc->body_status = BS_NONE;
} else if (bo->htc->body_status == BS_CHUNKED) {
wrk->stats->fetch_chunked++;
} else if (bo->htc->body_status == BS_LENGTH) {
assert(bo->htc->content_length > 0);
wrk->stats->fetch_length++;
} else if (bo->htc->body_status == BS_EOF) {
wrk->stats->fetch_eof++;
} else if (bo->htc->body_status == BS_ERROR) {
wrk->stats->fetch_bad++;
} else if (bo->htc->body_status == BS_NONE) {
wrk->stats->fetch_none++;
} else {
WRONG("wrong bodystatus");
}
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
}
bo->vfc->bo = bo;
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->wrk = bo->wrk;
bo->vfc->http = bo->beresp;
bo->vfc->esi_req = bo->bereq;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum fetch_step
vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
{
int i, do_ims;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
xxxassert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
if (!bo->do_pass && bo->req != NULL)
vbf_release_req(bo); /* XXX: retry ?? */
assert(bo->state <= BOS_REQ_DONE);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
if (bo->do_pass && bo->req != NULL)
vbf_release_req(bo); /* XXX : retry ?? */
AZ(bo->req);
if (i) {
AZ(bo->vbc);
make_it_503(bo);
} else {
AN(bo->vbc);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
if (i && bo->htc.body_status == BS_LENGTH)
bo->htc.body_status = BS_NONE;
bo->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
RFC2616_Ttl(bo);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
bo->beresp->status = 200;
http_PrintfHeader(bo->beresp, "Content-Length: %jd",
(intmax_t)bo->ims_obj->len);
do_ims = 1;
} else
do_ims = 0;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass)
bo->fetch_objcore->flags |= OC_F_PASS;
if (wrk->handling == VCL_RET_DELIVER)
return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
if (wrk->handling == VCL_RET_RETRY) {
assert(bo->state == BOS_REQ_DONE);
bo->retries++;
if (bo->retries <= cache_param->max_retries) {
VDI_CloseFd(&bo->vbc);
return (F_STP_STARTFETCH);
}
// XXX: wrk->handling = VCL_RET_SYNTH;
}
INCOMPL();
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i, do_ims;
double now;
char time_str[VTIM_FORMAT_SIZE];
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->vbc);
assert(bo->doclose == SC_NULL);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
HTTP_Copy(bo->bereq, bo->bereq0);
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
AZ(bo->vbc);
return (F_STP_ERROR);
}
AN(bo->vbc);
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
VTIM_format(now, time_str);
http_PrintfHeader(bo->beresp, "Date: %s", time_str);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
if (bo->htc.body_status == BS_ERROR) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
http_Merge(bo->ims_obj->http, bo->beresp,
bo->ims_obj->changed_gzip);
assert(bo->beresp->status == 200);
do_ims = 1;
} else
do_ims = 0;
VFP_Setup(bo->vfc);
bo->vfc->bo = bo;
bo->vfc->http = bo->beresp;
bo->vfc->vsl = bo->vsl;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
if (wrk->handling == VCL_RET_RETRY) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
bo->doclose = SC_NULL;
bo->retries++;
if (bo->retries <= cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
}
static int
cnt_fetch(struct worker *wrk, struct req *req)
{
int i, need_host_hdr;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AN(req->director);
AZ(bo->vbc);
AZ(bo->should_close);
AZ(req->storage_hint);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
req->acct_req.fetch++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
}
if (i) {
req->handling = VCL_RET_ERROR;
req->err_code = 503;
} else {
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->body_status = RFC2616_Body(bo, &wrk->stats);
req->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
bo->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(bo);
/* pass from vclrecv{} has negative TTL */
if (req->objcore->objhead == NULL)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->do_pass);
VCL_fetch_method(req);
if (bo->do_pass)
req->objcore->flags |= OC_F_PASS;
switch (req->handling) {
case VCL_RET_DELIVER:
req->req_step = R_STP_FETCHBODY;
return (0);
default:
break;
}
/* We are not going to fetch the body, Close the connection */
VDI_CloseFd(&bo->vbc);
}
/* Clean up partial fetch */
AZ(bo->vbc);
if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
req->objcore = NULL;
}
assert(bo->refcount == 2);
VBO_DerefBusyObj(wrk, &bo);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->director = NULL;
req->storage_hint = NULL;
switch (req->handling) {
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
return (0);
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
}
static int
cnt_recv(const struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
AZ(req->obj);
AZ(req->busyobj);
VSLb(req->vsl, SLT_ReqStart, "%s %s", req->sp->addr, req->sp->port);
if (req->err_code) {
req->req_step = R_STP_ERROR;
return (0);
}
/* By default we use the first backend */
AZ(req->director);
req->director = req->vcl->director[0];
AN(req->director);
EXP_Clr(&req->exp);
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
http_CollectHdr(req->http, H_Cache_Control);
VCL_recv_method(req);
recv_handling = req->handling;
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_Unset(req->http, H_Accept_Encoding);
http_SetHeader(req->http, "Accept-Encoding: gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */
SHA256_Init(req->sha256ctx);
VCL_hash_method(req);
assert(req->handling == VCL_RET_HASH);
SHA256_Final(req->digest, req->sha256ctx);
req->sha256ctx = NULL;
if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
req->wantbody = 0;
else
req->wantbody = 1;
switch(recv_handling) {
case VCL_RET_LOOKUP:
req->req_step = R_STP_LOOKUP;
return (0);
case VCL_RET_PIPE:
if (req->esi_level > 0) {
/* XXX: VSL something */
INCOMPL();
return (1);
}
req->req_step = R_STP_PIPE;
return (0);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (0);
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_recv{}");
}
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
char *xff;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
AZ(req->objcore);
AZ(req->obj);
AZ(req->objcore);
AZ(isnan(req->t_first));
AZ(isnan(req->t_prev));
AZ(isnan(req->t_req));
VSLb(req->vsl, SLT_ReqStart, "%s %s",
req->sp->client_addr_str, req->sp->client_port_str);
http_VSL_log(req->http);
if (req->err_code) {
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
}
/* By default we use the first backend */
AZ(req->director_hint);
req->director_hint = req->vcl->director[0];
AN(req->director_hint);
req->d_ttl = -1;
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
if (req->restarts == 0) {
if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
http_Unset(req->http, H_X_Forwarded_For);
http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff,
req->sp->client_addr_str);
} else {
http_PrintfHeader(req->http, "X-Forwarded-For: %s",
req->sp->client_addr_str);
}
}
http_CollectHdr(req->http, H_Cache_Control);
VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws);
/* Attempts to cache req.body may fail */
if (req->req_body_status == REQ_BODY_FAIL) {
return (REQ_FSM_DONE);
}
recv_handling = wrk->handling;
/* We wash the A-E header here for the sake of VRY */
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_ForceHeader(req->http, H_Accept_Encoding, "gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */
SHA256_Init(req->sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, req->http->ws);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, req->sha256ctx);
req->sha256ctx = NULL;
if (!strcmp(req->http->hd[HTTP_HDR_METHOD].b, "HEAD"))
req->wantbody = 0;
else
req->wantbody = 1;
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
AZ(req->objcore);
AZ(req->obj);
AZ(req->objcore);
VSLb(req->vsl, SLT_ReqStart, "%s %s", req->sp->addr, req->sp->port);
if (req->err_code) {
req->req_step = R_STP_ERROR;
return (REQ_FSM_MORE);
}
/* By default we use the first backend */
AZ(req->director);
req->director = req->vcl->director[0];
AN(req->director);
EXP_Clr(&req->exp);
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
http_CollectHdr(req->http, H_Cache_Control);
VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws);
recv_handling = wrk->handling;
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_Unset(req->http, H_Accept_Encoding);
http_SetHeader(req->http, "Accept-Encoding: gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */
SHA256_Init(req->sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, req->http->ws);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, req->sha256ctx);
req->sha256ctx = NULL;
if (!strcmp(req->http->hd[HTTP_HDR_METHOD].b, "HEAD"))
req->wantbody = 0;
else
req->wantbody = 1;
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
static int
cnt_fetch(struct sess *sp)
{
int i, need_host_hdr;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
AN(sp->director);
AZ(wrk->busyobj->vbc);
AZ(wrk->busyobj->should_close);
AZ(wrk->storage_hint);
http_Setup(wrk->busyobj->beresp, wrk->ws);
need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL);
i = FetchHdr(sp, need_host_hdr);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = FetchHdr(sp, need_host_hdr);
}
if (i) {
sp->handling = VCL_RET_ERROR;
sp->err_code = 503;
} else {
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(wrk->busyobj->beresp, H_Cache_Control);
http_CollectHdr(wrk->busyobj->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
wrk->busyobj->body_status = RFC2616_Body(sp);
sp->err_code = http_GetStatus(wrk->busyobj->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&wrk->busyobj->exp);
wrk->busyobj->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(sp);
/* pass from vclrecv{} has negative TTL */
if (wrk->objcore == NULL)
wrk->busyobj->exp.ttl = -1.;
AZ(wrk->busyobj->do_esi);
VCL_fetch_method(sp);
switch (sp->handling) {
case VCL_RET_HIT_FOR_PASS:
if (wrk->objcore != NULL)
wrk->objcore->flags |= OC_F_PASS;
sp->step = STP_FETCHBODY;
return (0);
case VCL_RET_DELIVER:
AssertObjCorePassOrBusy(wrk->objcore);
sp->step = STP_FETCHBODY;
return (0);
default:
break;
}
/* We are not going to fetch the body, Close the connection */
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
}
/* Clean up partial fetch */
AZ(wrk->busyobj->vbc);
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
AZ(HSH_Deref(wrk, wrk->objcore, NULL));
wrk->objcore = NULL;
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
sp->director = NULL;
wrk->storage_hint = NULL;
switch (sp->handling) {
case VCL_RET_RESTART:
sp->restarts++;
sp->step = STP_RECV;
return (0);
case VCL_RET_ERROR:
sp->step = STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
}
static int
cnt_recv(struct sess *sp)
{
struct worker *wrk;
unsigned recv_handling;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
AZ(wrk->obj);
AZ(wrk->busyobj);
assert(wrk->wrw.ciov == wrk->wrw.siov);
/* By default we use the first backend */
AZ(sp->director);
sp->director = sp->vcl->director[0];
AN(sp->director);
sp->disable_esi = 0;
sp->hash_always_miss = 0;
sp->hash_ignore_busy = 0;
sp->client_identity = NULL;
http_CollectHdr(sp->http, H_Cache_Control);
VCL_recv_method(sp);
recv_handling = sp->handling;
if (sp->restarts >= cache_param->max_restarts) {
if (sp->err_code == 0)
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(sp)) {
http_Unset(sp->http, H_Accept_Encoding);
http_SetHeader(wrk, sp->vsl_id, sp->http,
"Accept-Encoding: gzip");
} else {
http_Unset(sp->http, H_Accept_Encoding);
}
}
SHA256_Init(wrk->sha256ctx);
VCL_hash_method(sp);
assert(sp->handling == VCL_RET_HASH);
SHA256_Final(sp->digest, wrk->sha256ctx);
if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
sp->wantbody = 0;
else
sp->wantbody = 1;
sp->sendbody = 0;
switch(recv_handling) {
case VCL_RET_LOOKUP:
/* XXX: discard req body, if any */
sp->step = STP_LOOKUP;
return (0);
case VCL_RET_PIPE:
if (sp->esi_level > 0) {
/* XXX: VSL something */
INCOMPL();
/* sp->step = STP_DONE; */
return (1);
}
sp->step = STP_PIPE;
return (0);
case VCL_RET_PASS:
sp->step = STP_PASS;
return (0);
case VCL_RET_ERROR:
/* XXX: discard req body, if any */
sp->step = STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_recv{}");
}
}
