vmod_body(VRT_CTX, struct vmod_priv *priv, VCL_BYTES maxsize)
{
struct vsb *vsb;
const char *p;
ssize_t size;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
if (priv->priv)
return (VSB_data(priv->priv));
if (!http_GetHdr(ctx->req->http, H_Content_Type, &p))
return (NULL);
if (strncasecmp(p, FORM_URLENCODED, sizeof(FORM_URLENCODED) - 1)) {
VSLb(ctx->vsl, SLT_Error,
"vsf.body: Unsupported form encoding (%s)", p);
return (NULL);
}
size = VRT_CacheReqBody(ctx, maxsize);
if (size <= 0)
return (NULL);
vsb = VSB_new(NULL, NULL, size + 1, 0);
if (!vsb) {
VSLb(ctx->vsl, SLT_Error, "vsf.body: Out of memory");
return (NULL);
}
if (VRB_Iterate(ctx->req, vsf_iter_req_body, vsb) == -1) {
VSLb(ctx->vsl, SLT_Error,
"vsf.body: Problem fetching the body");
VSB_delete(vsb);
return (NULL);
}
AZ(VSB_finish(vsb));
priv->free = (vmod_priv_free_f *)VSB_delete;
priv->priv = vsb;
return (VSB_data(vsb));
}
int
http_GetHdrData(const struct http *hp, const char *hdr,
const char *field, char **ptr)
{
char *h, *e;
unsigned fl;
if (ptr != NULL)
*ptr = NULL;
if (!http_GetHdr(hp, hdr, &h))
return (0);
AN(h);
e = strchr(h, '\0');
fl = strlen(field);
while (h + fl <= e) {
/* Skip leading whitespace and commas */
if (vct_islws(*h) || *h == ',') {
h++;
continue;
}
/* Check for substrings before memcmp() */
if ((h + fl == e || vct_issepctl(h[fl])) &&
!memcmp(h, field, fl)) {
if (ptr != NULL) {
h += fl;
while (vct_islws(*h))
h++;
*ptr = h;
}
return (1);
}
/* Skip until end of header or comma */
while (*h && *h != ',')
h++;
}
return (0);
}
int
VRY_Match(struct sess *sp, const uint8_t *vary)
{
uint8_t *vsp = sp->vary_b;
char *h, *e;
unsigned lh, ln;
int i, retval = 1, oflo = 0;
AN(vsp);
while (vary[2]) {
i = vry_cmp(&vary, &vsp);
if (i == 1) {
/* Build a new entry */
i = http_GetHdr(sp->http, (const char*)(vary+2), &h);
if (i) {
/* Trim trailing space */
e = strchr(h, '\0');
while (e > h && vct_issp(e[-1]))
e--;
lh = e - h;
assert(lh < 0xffff);
} else {
e = h = NULL;
lh = 0xffff;
}
/* Length of the entire new vary entry */
ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
if (vsp + ln >= sp->vary_e) {
vsp = sp->vary_b;
oflo = 1;
}
/*
* We MUST have space for one entry and the end marker
* after it, which prevents old junk from confusing us
*/
assert(vsp + ln + 2 < sp->vary_e);
vbe16enc(vsp, (uint16_t)lh);
memcpy(vsp + 2, vary + 2, vary[2] + 2);
if (h != NULL && e != NULL) {
memcpy(vsp + 2 + vsp[2] + 2, h, e - h);
vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0';
} else
vsp[2 + vary[2] + 2 + 2] = '\0';
i = vry_cmp(&vary, &vsp);
assert(i != 1); /* hdr must be the same now */
}
if (i != 0)
retval = 0;
vsp += vry_len(vsp);
vary += vry_len(vary);
}
if (vsp + 3 > sp->vary_e)
oflo = 1;
if (oflo) {
/* XXX: Should log this */
vsp = sp->vary_b;
}
vsp[0] = 0xff;
vsp[1] = 0xff;
vsp[2] = 0;
if (oflo)
sp->vary_l = NULL;
else
sp->vary_l = vsp + 3;
return (retval);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
const char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
if (vbf_figure_out_vfp(bo)) {
(bo)->htc->doclose = SC_OVERLOAD;
VDI_Finish((bo)->wrk, bo);
return (F_STP_ERROR);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
bo->fetch_objcore->boc->len_so_far = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->htc->doclose = SC_RX_BODY;
VFP_Close(bo->vfc);
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (bo->do_esi)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);
if (bo->do_gzip || bo->do_gunzip)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);
if (!(bo->fetch_objcore->flags & OC_F_PASS) &&
http_IsStatus(bo->beresp, 200) && (
http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
http_GetHdr(bo->beresp, H_ETag, &p)))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);
if (bo->htc->body_status != BS_NONE &&
VDI_GetBody(bo->wrk, bo) != 0) {
(void)VFP_Error(bo->vfc,
"GetBody failed - workspace_backend overflow?");
VFP_Close(bo->vfc);
bo->htc->doclose = SC_OVERLOAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->refcount >= 1);
assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
if (bo->do_stream) {
ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
HSH_Unbusy(wrk, bo->fetch_objcore);
ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc->body_status, body_status_2str(bo->htc->body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc->body_status != BS_NONE) {
assert(bo->htc->body_status != BS_ERROR);
return (F_STP_FETCHBODY);
}
AZ(bo->vfc->failed);
return (F_STP_FETCHEND);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage);
bo->storage = bo->do_pass ? stv_transient : STV_next();
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
AZ(bo->htc);
VFP_Setup(bo->vfc, wrk);
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->resp = bo->beresp;
bo->vfc->req = bo->bereq;
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
if (bo->fetch_objcore->flags & OC_F_PRIVATE) {
/* private objects have negative TTL */
bo->fetch_objcore->t_origin = now;
bo->fetch_objcore->ttl = -1.;
bo->fetch_objcore->grace = 0;
bo->fetch_objcore->keep = 0;
} else {
/* What does RFC2616 think about TTL ? */
RFC2616_Ttl(bo, now,
&bo->fetch_objcore->t_origin,
&bo->fetch_objcore->ttl,
&bo->fetch_objcore->grace,
&bo->fetch_objcore->keep
);
}
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
}
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
if (bo->fetch_objcore->boc->state != BOS_REQ_DONE) {
bo->req = NULL;
ObjSetState(wrk, bo->fetch_objcore, BOS_REQ_DONE);
}
if (bo->do_esi)
bo->do_stream = 0;
if (wrk->handling == VCL_RET_PASS) {
bo->fetch_objcore->flags |= OC_F_HFP;
bo->uncacheable = 1;
wrk->handling = VCL_RET_DELIVER;
}
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
enum body_status
RFC2616_Body(struct busyobj *bo, struct dstat *stats)
{
struct http *hp;
char *b;
hp = bo->beresp;
if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
bo->should_close = 1;
else if (http_HdrIs(hp, H_Connection, "close"))
bo->should_close = 1;
else
bo->should_close = 0;
if (!strcasecmp(http_GetReq(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
stats->fetch_head++;
return (BS_NONE);
}
if (hp->status <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
*/
stats->fetch_1xx++;
return (BS_NONE);
}
if (hp->status == 204) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC2616 10.2.5 p60]
*/
stats->fetch_204++;
return (BS_NONE);
}
if (hp->status == 304) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
stats->fetch_304++;
return (BS_NONE);
}
if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
stats->fetch_chunked++;
return (BS_CHUNKED);
}
if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
stats->fetch_bad++;
return (BS_ERROR);
}
if (http_GetHdr(hp, H_Content_Length, &bo->h_content_length)) {
stats->fetch_length++;
return (BS_LENGTH);
}
if (http_HdrIs(hp, H_Connection, "keep-alive")) {
/*
* Keep alive with neither TE=Chunked or C-Len is impossible.
* We assume a zero length body.
*/
stats->fetch_zero++;
return (BS_ZERO);
}
if (http_HdrIs(hp, H_Connection, "close")) {
/*
* In this case, it is safe to just read what comes.
*/
stats->fetch_close++;
return (BS_EOF);
}
if (hp->protover < 11) {
/*
* With no Connection header, assume EOF.
*/
stats->fetch_oldhttp++;
return (BS_EOF);
}
/*
* Fall back to EOF transfer.
*/
stats->fetch_eof++;
return (BS_EOF);
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
const char *xff;
const char *ci, *cp;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
AN(req->vcl);
AZ(req->objcore);
AZ(req->err_code);
AZ(isnan(req->t_first));
AZ(isnan(req->t_prev));
AZ(isnan(req->t_req));
ci = SES_Get_String_Attr(req->sp, SA_CLIENT_IP);
cp = SES_Get_String_Attr(req->sp, SA_CLIENT_PORT);
VSLb(req->vsl, SLT_ReqStart, "%s %s", ci, cp);
http_VSL_log(req->http);
if (req->restarts == 0) {
/*
* This really should be done earlier, but we want to capture
* it in the VSL log.
*/
http_CollectHdr(req->http, H_X_Forwarded_For);
if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
http_Unset(req->http, H_X_Forwarded_For);
http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s",
xff, ci);
} else {
http_PrintfHeader(req->http, "X-Forwarded-For: %s", ci);
}
}
/* By default we use the first backend */
AZ(req->director_hint);
req->director_hint = VCL_DefaultDirector(req->vcl);
AN(req->director_hint);
req->d_ttl = -1;
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
http_CollectHdr(req->http, H_Cache_Control);
VFP_Setup(req->htc->vfc);
req->htc->vfc->http = req->http;
req->htc->vfc->wrk = wrk;
if (req->transport->req_body != NULL) {
req->transport->req_body(req);
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_OVERLOAD;
return (REQ_FSM_DONE);
}
}
VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
/* Attempts to cache req.body may fail */
if (req->req_body_status == REQ_BODY_FAIL) {
req->doclose = SC_RX_BODY;
return (REQ_FSM_DONE);
}
recv_handling = wrk->handling;
/* We wash the A-E header here for the sake of VRY */
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_ForceHeader(req->http, H_Accept_Encoding, "gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
SHA256_Init(&sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, &sha256ctx);
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
const char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
if (bo->htc->content_length == 0)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->htc->body_status != BS_NONE) {
bo->is_gzip =
http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip =
!http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
}
/* We won't gunzip unless it is non-empty and gzip'ed */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gunzip && !bo->is_gzip))
bo->do_gunzip = 0;
/* We wont gzip unless it is non-empty and ungziped */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
(bo->do_gzip && !bo->is_gunzip))
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);
if (bo->htc->content_length != 0) {
if (bo->do_esi && bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
(void)VFP_Push(bo->vfc, &vfp_esi, 1);
} else if (bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
}
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc->body_status == BS_NONE)
bo->do_stream = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
return (F_STP_ERROR);
}
if (bo->do_esi)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);
if (bo->do_gzip || bo->do_gunzip)
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);
if (http_IsStatus(bo->beresp, 200) && (
http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
http_GetHdr(bo->beresp, H_ETag, &p)))
ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);
if (bo->htc->body_status != BS_NONE)
AZ(VDI_GetBody(bo->wrk, bo));
assert(bo->refcount >= 1);
assert(bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(wrk, bo->fetch_objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc->body_status, body_status_2str(bo->htc->body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc->body_status != BS_NONE) {
assert(bo->htc->body_status != BS_ERROR);
vbf_fetch_body_helper(bo);
}
if (bo->vfc->failed) {
VDI_Finish(bo->wrk, bo);
if (!bo->do_stream) {
assert(bo->state < BOS_STREAM);
// XXX: doclose = ?
return (F_STP_ERROR);
} else {
return (F_STP_FAIL);
}
}
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(wrk, bo->fetch_objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
VDI_Finish(bo->wrk, bo);
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->stale_oc != NULL)
EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);
return (F_STP_DONE);
}
int
VRY_Create(struct busyobj *bo, struct vsb **psb)
{
char *v, *p, *q, *h, *e;
struct vsb *sb, *sbh;
unsigned l;
int error = 0;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->bereq, HTTP_MAGIC);
CHECK_OBJ_NOTNULL(bo->beresp, HTTP_MAGIC);
AN(psb);
AZ(*psb);
/* No Vary: header, no worries */
if (!http_GetHdr(bo->beresp, H_Vary, &v))
return (0);
/* For vary matching string */
sb = VSB_new_auto();
AN(sb);
/* For header matching strings */
sbh = VSB_new_auto();
AN(sbh);
for (p = v; *p; p++) {
/* Find next header-name */
if (vct_issp(*p))
continue;
for (q = p; *q && !vct_issp(*q) && *q != ','; q++)
continue;
if (q - p > INT8_MAX) {
VSLb(bo->vsl, SLT_Error,
"Vary header name length exceeded");
error = 1;
break;
}
/* Build a header-matching string out of it */
VSB_clear(sbh);
VSB_printf(sbh, "%c%.*s:%c",
(char)(1 + (q - p)), (int)(q - p), p, 0);
AZ(VSB_finish(sbh));
if (http_GetHdr(bo->bereq, VSB_data(sbh), &h)) {
AZ(vct_issp(*h));
/* Trim trailing space */
e = strchr(h, '\0');
while (e > h && vct_issp(e[-1]))
e--;
/* Encode two byte length and contents */
l = e - h;
if (l > 0xffff - 1) {
VSLb(bo->vsl, SLT_Error,
"Vary header maximum length exceeded");
error = 1;
break;
}
} else {
e = h;
l = 0xffff;
}
VSB_printf(sb, "%c%c", (int)(l >> 8), (int)(l & 0xff));
/* Append to vary matching string */
VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh));
if (e != h)
VSB_bcat(sb, h, e - h);
while (vct_issp(*q))
q++;
if (*q == '\0')
break;
if (*q != ',') {
VSLb(bo->vsl, SLT_Error, "Malformed Vary header");
error = 1;
break;
}
p = q;
}
if (error) {
VSB_delete(sbh);
VSB_delete(sb);
return (-1);
}
/* Terminate vary matching string */
VSB_printf(sb, "%c%c%c", 0xff, 0xff, 0);
VSB_delete(sbh);
AZ(VSB_finish(sb));
*psb = sb;
return (VSB_len(sb));
}
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(req->handling == VCL_RET_DELIVER);
if (req->objcore->objhead == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
bo->exp.ttl = -1.;
} else if (bo->do_pass) {
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->do_esi || req->esi_level > 0)
bo->do_stream = 0;
if (!req->wantbody)
bo->do_stream = 0;
/* No reason to try streaming a non-existing body */
if (bo->body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (req->objcore->objhead != NULL) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
vary = VRY_Create(req, bo->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (bo->exp.ttl < cache_param->shortlived ||
req->objcore == NULL)
req->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
nhttp);
if (req->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
l, nhttp);
}
bo->stats = NULL;
if (req->obj == NULL) {
req->err_code = 503;
req->req_step = R_STP_ERROR;
VDI_CloseFd(&bo->vbc);
VBO_DerefBusyObj(wrk, &req->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
req->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = req->obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
req->obj->gziped = 1;
if (vary != NULL) {
req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
VSB_data(vary), varyl);
AN(req->obj->vary);
VRY_Validate(req->obj->vary);
VSB_delete(vary);
}
req->obj->vxid = bo->vsl->wid;
req->obj->response = req->err_code;
WS_Assert(req->obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = req->obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
req->obj->last_modified = VTIM_parse(b);
else
req->obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (req->obj->response == 200 &&
req->http->conds &&
RFC2616_Do_Cond(req))
bo->do_stream = 0;
/*
* Ready to fetch the body
*/
bo->fetch_task.func = FetchBody;
bo->fetch_task.priv = bo;
assert(bo->refcount == 2); /* one for each thread */
if (req->obj->objcore->objhead != NULL) {
EXP_Insert(req->obj);
AN(req->obj->objcore->ban);
AZ(req->obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, req->obj->objcore);
}
if (!bo->do_stream ||
Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
FetchBody(wrk, bo);
if (req->obj->objcore->objhead != NULL)
HSH_Ref(req->obj->objcore);
if (bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &req->busyobj);
} else if (bo->state == BOS_FAILED) {
/* handle early failures */
HSH_Deref(&wrk->stats, NULL, &req->obj);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->err_code = 503;
req->req_step = R_STP_ERROR;
return (0);
}
assert(WRW_IsReleased(wrk));
req->req_step = R_STP_PREPRESP;
return (0);
}
static int
cnt_fetch(struct worker *wrk, struct req *req)
{
int i, need_host_hdr;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AN(req->director);
AZ(bo->vbc);
AZ(bo->should_close);
AZ(req->storage_hint);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
req->acct_req.fetch++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSC_C_main->backend_retry++;
i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
}
if (i) {
req->handling = VCL_RET_ERROR;
req->err_code = 503;
} else {
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->body_status = RFC2616_Body(bo, &wrk->stats);
req->err_code = http_GetStatus(bo->beresp);
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->exp);
bo->exp.entered = W_TIM_real(wrk);
RFC2616_Ttl(bo);
/* pass from vclrecv{} has negative TTL */
if (req->objcore->objhead == NULL)
bo->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->do_pass);
VCL_fetch_method(req);
if (bo->do_pass)
req->objcore->flags |= OC_F_PASS;
switch (req->handling) {
case VCL_RET_DELIVER:
req->req_step = R_STP_FETCHBODY;
return (0);
default:
break;
}
/* We are not going to fetch the body, Close the connection */
VDI_CloseFd(&bo->vbc);
}
/* Clean up partial fetch */
AZ(bo->vbc);
if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
req->objcore = NULL;
}
assert(bo->refcount == 2);
VBO_DerefBusyObj(wrk, &bo);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->director = NULL;
req->storage_hint = NULL;
switch (req->handling) {
case VCL_RET_RESTART:
req->req_step = R_STP_RESTART;
return (0);
case VCL_RET_ERROR:
req->req_step = R_STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
}
static enum req_fsm_nxt
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
struct SHA256Context sha256ctx;
char *xff;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
AZ(req->objcore);
AZ(req->obj);
AZ(req->objcore);
AZ(isnan(req->t_first));
AZ(isnan(req->t_prev));
AZ(isnan(req->t_req));
VSLb(req->vsl, SLT_ReqStart, "%s %s",
req->sp->client_addr_str, req->sp->client_port_str);
http_VSL_log(req->http);
if (req->err_code) {
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
}
/* By default we use the first backend */
AZ(req->director_hint);
req->director_hint = req->vcl->director[0];
AN(req->director_hint);
req->d_ttl = -1;
req->disable_esi = 0;
req->hash_always_miss = 0;
req->hash_ignore_busy = 0;
req->client_identity = NULL;
if (req->restarts == 0) {
if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) {
http_Unset(req->http, H_X_Forwarded_For);
http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff,
req->sp->client_addr_str);
} else {
http_PrintfHeader(req->http, "X-Forwarded-For: %s",
req->sp->client_addr_str);
}
}
http_CollectHdr(req->http, H_Cache_Control);
VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws);
/* Attempts to cache req.body may fail */
if (req->req_body_status == REQ_BODY_FAIL) {
return (REQ_FSM_DONE);
}
recv_handling = wrk->handling;
/* We wash the A-E header here for the sake of VRY */
if (cache_param->http_gzip_support &&
(recv_handling != VCL_RET_PIPE) &&
(recv_handling != VCL_RET_PASS)) {
if (RFC2616_Req_Gzip(req->http)) {
http_ForceHeader(req->http, H_Accept_Encoding, "gzip");
} else {
http_Unset(req->http, H_Accept_Encoding);
}
}
req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */
SHA256_Init(req->sha256ctx);
VCL_hash_method(req->vcl, wrk, req, NULL, req->http->ws);
assert(wrk->handling == VCL_RET_LOOKUP);
SHA256_Final(req->digest, req->sha256ctx);
req->sha256ctx = NULL;
if (!strcmp(req->http->hd[HTTP_HDR_METHOD].b, "HEAD"))
req->wantbody = 0;
else
req->wantbody = 1;
switch(recv_handling) {
case VCL_RET_PURGE:
req->req_step = R_STP_PURGE;
return (REQ_FSM_MORE);
case VCL_RET_HASH:
req->req_step = R_STP_LOOKUP;
return (REQ_FSM_MORE);
case VCL_RET_PIPE:
if (req->esi_level == 0) {
req->req_step = R_STP_PIPE;
return (REQ_FSM_MORE);
}
VSLb(req->vsl, SLT_VCL_Error,
"vcl_recv{} returns pipe for ESI included object."
" Doing pass.");
req->req_step = R_STP_PASS;
return (REQ_FSM_DONE);
case VCL_RET_PASS:
req->req_step = R_STP_PASS;
return (REQ_FSM_MORE);
case VCL_RET_SYNTH:
req->req_step = R_STP_SYNTH;
return (REQ_FSM_MORE);
default:
WRONG("Illegal return from vcl_recv{}");
}
}
void
V1D_Deliver(struct req *req, struct busyobj *bo)
{
const char *r;
enum objiter_status ois;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
req->res_mode = 0;
/*
* Determine ESI status first. Not dependent on wantbody, because
* we want ESI to supress C-L in HEAD too.
*/
if (!req->disable_esi &&
ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL)
req->res_mode |= RES_ESI;
/*
* ESI-childen don't care about headers -> early escape
*/
if (req->esi_level > 0) {
ESI_DeliverChild(req, bo);
return;
}
if (req->res_mode & RES_ESI) {
RFC2616_Weaken_Etag(req->resp);
} else if (http_IsStatus(req->resp, 304)) {
http_Unset(req->resp, H_Content_Length);
req->wantbody = 0;
} else if (bo == NULL &&
!http_GetHdr(req->resp, H_Content_Length, NULL)) {
http_PrintfHeader(req->resp,
"Content-Length: %ju", (uintmax_t)ObjGetLen(
req->wrk, req->objcore));
}
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http)) {
/*
* We don't know what it uncompresses to
* XXX: we could cache that, but would still deliver
* XXX: with multiple writes because of the gunzip buffer
*/
req->res_mode |= RES_GUNZIP;
VDP_push(req, VDP_gunzip, NULL, 0);
}
if (req->res_mode & RES_ESI) {
/* Gunzip could have added back a C-L */
http_Unset(req->resp, H_Content_Length);
}
/*
* Range comes after the others and pushes on bottom because it
* can generate a correct C-L header.
*/
if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) {
http_SetHeader(req->resp, "Accept-Ranges: bytes");
if (req->wantbody && http_GetHdr(req->http, H_Range, &r))
VRG_dorange(req, bo, r);
}
if (http_GetHdr(req->resp, H_Content_Length, NULL))
req->res_mode |= RES_LEN;
if (req->wantbody && !(req->res_mode & RES_LEN)) {
if (req->http->protover >= 11) {
req->res_mode |= RES_CHUNKED;
http_SetHeader(req->resp, "Transfer-Encoding: chunked");
} else {
req->res_mode |= RES_EOF;
req->doclose = SC_TX_EOF;
}
}
VSLb(req->vsl, SLT_Debug, "RES_MODE %x", req->res_mode);
if (req->doclose) {
if (!http_HdrIs(req->resp, H_Connection, "close")) {
http_Unset(req->resp, H_Connection);
http_SetHeader(req->resp, "Connection: close");
}
} else if (!http_GetHdr(req->resp, H_Connection, NULL))
http_SetHeader(req->resp, "Connection: keep-alive");
VDP_push(req, v1d_bytes, NULL, 1);
V1L_Reserve(req->wrk, req->ws, &req->sp->fd, req->vsl, req->t_prev);
req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
if (DO_DEBUG(DBG_FLUSH_HEAD))
(void)V1L_Flush(req->wrk);
ois = OIS_DONE;
if (req->wantbody) {
if (req->res_mode & RES_CHUNKED)
V1L_Chunked(req->wrk);
ois = VDP_DeliverObj(req);
(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
if (ois == OIS_DONE && (req->res_mode & RES_CHUNKED))
V1L_EndChunk(req->wrk);
}
if ((V1L_FlushRelease(req->wrk) || ois != OIS_DONE) && req->sp->fd >= 0)
SES_Close(req->sp, SC_REM_CLOSE);
VDP_close(req);
}
int
V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
{
struct vbc *vc;
struct http *hp;
enum htc_status_e hs;
int retry = -1;
int i, first;
struct http_conn *htc;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_ORNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
htc = &bo->htc;
AN(bo->director);
hp = bo->bereq;
bo->vbc = VDI_GetFd(NULL, bo);
if (bo->vbc == NULL) {
VSLb(bo->vsl, SLT_FetchError, "no backend connection");
return (-1);
}
vc = bo->vbc;
if (vc->recycled)
retry = 1;
/*
* Now that we know our backend, we can set a default Host:
* header if one is necessary. This cannot be done in the VCL
* because the backend may be chosen by a director.
*/
if (!http_GetHdr(bo->bereq, H_Host, NULL))
VDI_AddHostHeader(bo->bereq, vc);
(void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */
WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_fetch);
bo->t_send = VTIM_mono();
(void)HTTP1_Write(wrk, hp, 0); /* XXX: stats ? */
/* Deal with any message-body the request might (still) have */
i = 0;
if (req != NULL) {
i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
if (req->req_body_status == REQ_BODY_DONE)
retry = -1;
if (req->req_body_status == REQ_BODY_FAIL) {
VSLb(bo->vsl, SLT_FetchError,
"req.body read error: %d (%s)",
errno, strerror(errno));
req->doclose = SC_RX_BODY;
retry = -1;
}
}
if (WRW_FlushRelease(wrk) || i != 0) {
VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
errno, strerror(errno));
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (retry);
}
VSC_C_main->backend_req++;
bo->t_sent = VTIM_mono();
/* Receive response */
HTTP1_Init(htc, bo->ws, vc->fd, vc->vsl,
cache_param->http_resp_size,
cache_param->http_resp_hdr_len);
VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
first = 1;
do {
hs = HTTP1_Rx(htc);
if (hs == HTTP1_OVERFLOW) {
VSLb(bo->vsl, SLT_FetchError,
"http %sread error: overflow",
first ? "first " : "");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (-1);
}
if (hs == HTTP1_ERROR_EOF) {
VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
first ? "first " : "");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (retry);
}
if (first) {
retry = -1;
first = 0;
VTCP_set_read_timeout(vc->fd,
vc->between_bytes_timeout);
}
} while (hs != HTTP1_COMPLETE);
bo->t_hdr = VTIM_mono();
hp = bo->beresp;
if (HTTP1_DissectResponse(hp, htc)) {
VSLb(bo->vsl, SLT_FetchError, "http format error");
VDI_CloseFd(&bo->vbc);
/* XXX: other cleanup ? */
return (-1);
}
return (0);
}
struct vsb *
VRY_Create(const struct sess *sp, const struct http *hp)
{
char *v, *p, *q, *h, *e;
struct vsb *sb, *sbh;
unsigned l;
/* No Vary: header, no worries */
if (!http_GetHdr(hp, H_Vary, &v))
return (NULL);
/* For vary matching string */
sb = VSB_new_auto();
AN(sb);
/* For header matching strings */
sbh = VSB_new_auto();
AN(sbh);
if (*v == ':') {
WSP(sp, SLT_Error, "Vary header had extra ':', fix backend");
v++;
}
for (p = v; *p; p++) {
/* Find next header-name */
if (vct_issp(*p))
continue;
for (q = p; *q && !vct_issp(*q) && *q != ','; q++)
continue;
/* Build a header-matching string out of it */
VSB_clear(sbh);
VSB_printf(sbh, "%c%.*s:%c",
(char)(1 + (q - p)), (int)(q - p), p, 0);
AZ(VSB_finish(sbh));
if (http_GetHdr(sp->http, VSB_data(sbh), &h)) {
AZ(vct_issp(*h));
/* Trim trailing space */
e = strchr(h, '\0');
while (e > h && vct_issp(e[-1]))
e--;
/* Encode two byte length and contents */
l = e - h;
assert(!(l & ~0xffff));
} else {
e = h;
l = 0xffff;
}
VSB_printf(sb, "%c%c", (int)(l >> 8), (int)(l & 0xff));
/* Append to vary matching string */
VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh));
if (e != h)
VSB_bcat(sb, h, e - h);
while (vct_issp(*q))
q++;
if (*q == '\0')
break;
xxxassert(*q == ',');
p = q;
}
/* Terminate vary matching string */
VSB_printf(sb, "%c%c%c", 0xff, 0xff, 0);
VSB_delete(sbh);
AZ(VSB_finish(sb));
return(sb);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
int
VRY_Match(struct req *req, const uint8_t *vary)
{
uint8_t *vsp = req->vary_b;
char *h, *e;
unsigned lh, ln;
int i, oflo = 0;
AN(vsp);
while (vary[2]) {
if (vsp + 2 >= req->vary_e) {
/*
* Too little workspace to find out
*/
oflo = 1;
break;
}
i = vry_cmp(vary, vsp);
if (i == 1) {
/*
* Different header, build a new entry,
* then compare again with that new entry.
*/
ln = 2 + vary[2] + 2;
i = http_GetHdr(req->http, (const char*)(vary+2), &h);
if (i) {
/* Trim trailing space */
e = strchr(h, '\0');
while (e > h && vct_issp(e[-1]))
e--;
lh = e - h;
assert(lh < 0xffff);
ln += lh;
} else {
e = h = NULL;
lh = 0xffff;
}
if (vsp + ln + 2 >= req->vary_e) {
/*
* Not enough space to build new entry
* and put terminator behind it.
*/
oflo = 1;
break;
}
vbe16enc(vsp, (uint16_t)lh);
memcpy(vsp + 2, vary + 2, vary[2] + 2);
if (h != NULL)
memcpy(vsp + 2 + vsp[2] + 2, h, lh);
vsp[ln + 0] = 0xff;
vsp[ln + 1] = 0xff;
vsp[ln + 2] = 0;
(void)VRY_Validate(vsp);
req->vary_l = vsp + ln + 3;
i = vry_cmp(vary, vsp);
assert(i == 0 || i == 2);
}
if (i == 0) {
/* Same header, same contents */
vsp += VRY_Len(vsp);
vary += VRY_Len(vary);
} else if (i == 2) {
/* Same header, different contents, cannot match */
return (0);
}
}
if (oflo) {
vsp = req->vary_b;
req->vary_l = NULL;
if (vsp + 2 < req->vary_e) {
vsp[0] = 0xff;
vsp[1] = 0xff;
vsp[2] = 0;
}
return (0);
} else {
return (1);
}
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
ssize_t est;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
AN(bo->vbc);
est = V1F_Setup_Fetch(bo);
if (est == 0) {
/*
* If the length is known to be zero, it's not gziped.
* A similar issue exists for chunked encoding but we
* don't handle that. See #1320.
*/
http_Unset(bo->beresp, H_Content_Encoding);
bo->is_gzip = 0;
bo->is_gunzip = 1;
}
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* Fix Content-Encoding, as appropriate */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
else if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc, &bo->acct);
return (F_STP_ERROR);
}
assert(WRW_IsReleased(wrk));
obj = bo->fetch_obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (bo->do_gzip || bo->do_gunzip)
obj->changed_gzip = 1;
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
assert (bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc.body_status, body_status_2str(bo->htc.body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc.body_status != BS_NONE) {
assert(bo->htc.body_status != BS_ERROR);
VFP_Fetch_Body(bo, est);
}
if (bo->failed && !bo->do_stream) {
assert(bo->state < BOS_STREAM);
if (bo->fetch_obj != NULL) {
oc_freeobj(bo->fetch_objcore);
bo->fetch_obj = NULL;
bo->stats->n_object--;
}
return (F_STP_ERROR);
}
if (bo->failed)
return (F_STP_FAIL);
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && !(bo->should_close)) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i;
double now;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
if (bo->retries > 0)
http_Unset(bo->bereq, "\012X-Varnish:");
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
AZ(bo->htc);
i = VDI_GetHdr(wrk, bo);
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
http_TimeHeader(bo->beresp, "Date: ", now);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
*/
if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
/*
* A HEAD request can never have a body in the reply,
* no matter what the headers might say.
* [RFC2516 4.3 p33]
*/
wrk->stats->fetch_head++;
bo->htc->body_status = BS_NONE;
} else if (http_GetStatus(bo->beresp) <= 199) {
/*
* 1xx responses never have a body.
* [RFC2616 4.3 p33]
* ... but we should never see them.
*/
wrk->stats->fetch_1xx++;
bo->htc->body_status = BS_ERROR;
} else if (http_IsStatus(bo->beresp, 204)) {
/*
* 204 is "No Content", obviously don't expect a body.
* [RFC7230 3.3.1 p28 and 3.3.2 p30]
*/
wrk->stats->fetch_204++;
if (http_GetHdr(bo->beresp, H_Content_Length, NULL) ||
http_GetHdr(bo->beresp, H_Transfer_Encoding, NULL))
bo->htc->body_status = BS_ERROR;
else
bo->htc->body_status = BS_NONE;
} else if (http_IsStatus(bo->beresp, 304)) {
/*
* 304 is "Not Modified" it has no body.
* [RFC2616 10.3.5 p63]
*/
wrk->stats->fetch_304++;
bo->htc->body_status = BS_NONE;
} else if (bo->htc->body_status == BS_CHUNKED) {
wrk->stats->fetch_chunked++;
} else if (bo->htc->body_status == BS_LENGTH) {
assert(bo->htc->content_length > 0);
wrk->stats->fetch_length++;
} else if (bo->htc->body_status == BS_EOF) {
wrk->stats->fetch_eof++;
} else if (bo->htc->body_status == BS_ERROR) {
wrk->stats->fetch_bad++;
} else if (bo->htc->body_status == BS_NONE) {
wrk->stats->fetch_none++;
} else {
WRONG("wrong bodystatus");
}
if (bo->htc->body_status == BS_ERROR) {
bo->htc->doclose = SC_RX_BODY;
VDI_Finish(bo->wrk, bo);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
AZ(bo->was_304);
if (http_IsStatus(bo->beresp, 304)) {
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
/*
* If we changed the gzip status of the object
* the stored Content_Encoding controls we
* must weaken any new ETag we get.
*/
http_Unset(bo->beresp, H_Content_Encoding);
RFC2616_Weaken_Etag(bo->beresp);
}
http_Unset(bo->beresp, H_Content_Length);
HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
assert(http_IsStatus(bo->beresp, 200));
bo->was_304 = 1;
} else if (!bo->do_pass) {
/*
* Backend sent unallowed 304
*/
VSLb(bo->vsl, SLT_Error,
"304 response but not conditional fetch");
bo->htc->doclose = SC_RX_BAD;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
}
bo->vfc->bo = bo;
bo->vfc->oc = bo->fetch_objcore;
bo->vfc->wrk = bo->wrk;
bo->vfc->http = bo->beresp;
bo->vfc->esi_req = bo->bereq;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
if (wrk->handling == VCL_RET_ABANDON) {
bo->htc->doclose = SC_RESP_CLOSE;
VDI_Finish(bo->wrk, bo);
return (F_STP_FAIL);
}
if (wrk->handling == VCL_RET_RETRY) {
if (bo->htc->body_status != BS_NONE)
bo->htc->doclose = SC_RESP_CLOSE;
if (bo->director_state != DIR_S_NULL)
VDI_Finish(bo->wrk, bo);
if (bo->retries++ < cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
assert(bo->director_state == DIR_S_NULL);
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum fetch_step
vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
{
int i, do_ims;
double now;
char time_str[VTIM_FORMAT_SIZE];
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->vbc);
assert(bo->doclose == SC_NULL);
AZ(bo->storage_hint);
if (bo->do_pass)
AN(bo->req);
else
AZ(bo->req);
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
HTTP_Copy(bo->bereq, bo->bereq0);
http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
bo->uncacheable = bo->do_pass;
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
assert (wrk->handling == VCL_RET_FETCH);
HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
assert(bo->state <= BOS_REQ_DONE);
i = V1F_fetch_hdr(wrk, bo, bo->req);
/*
* If we recycle a backend connection, there is a finite chance
* that the backend closed it before we get a request to it.
* Do a single retry in that case.
*/
if (i == 1) {
VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
VSC_C_main->backend_retry++;
i = V1F_fetch_hdr(wrk, bo, bo->req);
}
now = W_TIM_real(wrk);
VSLb_ts_busyobj(bo, "Beresp", now);
if (i) {
AZ(bo->vbc);
return (F_STP_ERROR);
}
AN(bo->vbc);
http_VSL_log(bo->beresp);
if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
/*
* RFC 2616 14.18 Date: The Date general-header field
* represents the date and time at which the message was
* originated, having the same semantics as orig-date in
* RFC 822. ... A received message that does not have a
* Date header field MUST be assigned one by the recipient
* if the message will be cached by that recipient or
* gatewayed via a protocol which requires a Date.
*
* If we didn't get a Date header, we assign one here.
*/
VTIM_format(now, time_str);
http_PrintfHeader(bo->beresp, "Date: %s", time_str);
}
/*
* These two headers can be spread over multiple actual headers
* and we rely on their content outside of VCL, so collect them
* into one line here.
*/
http_CollectHdr(bo->beresp, H_Cache_Control);
http_CollectHdr(bo->beresp, H_Vary);
/*
* Figure out how the fetch is supposed to happen, before the
* headers are adultered by VCL
* NB: Also sets other wrk variables
*/
bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
if (bo->htc.body_status == BS_ERROR) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
return (F_STP_ERROR);
}
/*
* What does RFC2616 think about TTL ?
*/
EXP_Clr(&bo->fetch_objcore->exp);
RFC2616_Ttl(bo, now);
/* private objects have negative TTL */
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
bo->fetch_objcore->exp.ttl = -1.;
AZ(bo->do_esi);
if (bo->ims_obj != NULL && bo->beresp->status == 304) {
http_Merge(bo->ims_obj->http, bo->beresp,
bo->ims_obj->changed_gzip);
assert(bo->beresp->status == 200);
do_ims = 1;
} else
do_ims = 0;
VFP_Setup(bo->vfc);
bo->vfc->bo = bo;
bo->vfc->http = bo->beresp;
bo->vfc->vsl = bo->vsl;
VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
if (wrk->handling == VCL_RET_ABANDON)
return (F_STP_FAIL);
if (wrk->handling == VCL_RET_RETRY) {
AN (bo->vbc);
VDI_CloseFd(&bo->vbc, &bo->acct);
bo->doclose = SC_NULL;
bo->retries++;
if (bo->retries <= cache_param->max_retries)
return (F_STP_RETRY);
VSLb(bo->vsl, SLT_VCL_Error,
"Too many retries, delivering 503");
return (F_STP_ERROR);
}
assert(bo->state == BOS_REQ_DONE);
if (bo->do_esi)
bo->do_stream = 0;
if (bo->do_pass || bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
assert(wrk->handling == VCL_RET_DELIVER);
return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
}
static enum req_fsm_nxt
cnt_transmit(struct worker *wrk, struct req *req)
{
struct boc *boc;
const char *r;
uint16_t status;
int sendbody;
intmax_t clval;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
/* Grab a ref to the bo if there is one */
boc = HSH_RefBoc(req->objcore);
clval = http_GetContentLength(req->resp);
if (boc != NULL)
req->resp_len = clval;
else
req->resp_len = ObjGetLen(req->wrk, req->objcore);
req->res_mode = 0;
/* RFC 7230, 3.3.3 */
status = http_GetStatus(req->resp);
if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) {
if (req->objcore->flags & OC_F_PASS)
sendbody = -1;
else
sendbody = 0;
} else if (status < 200 || status == 204 || status == 304) {
req->resp_len = -1;
sendbody = 0;
} else
sendbody = 1;
if (sendbody >= 0) {
if (!req->disable_esi && req->resp_len != 0 &&
ObjHasAttr(wrk, req->objcore, OA_ESIDATA))
VDP_push(req, VDP_ESI, NULL, 0, "ESI");
if (cache_param->http_gzip_support &&
ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
!RFC2616_Req_Gzip(req->http))
VDP_push(req, VDP_gunzip, NULL, 1, "GUZ");
if (cache_param->http_range_support &&
http_IsStatus(req->resp, 200)) {
http_SetHeader(req->resp, "Accept-Ranges: bytes");
if (sendbody && http_GetHdr(req->http, H_Range, &r))
VRG_dorange(req, r);
}
}
if (sendbody < 0) {
/* Don't touch pass+HEAD C-L */
sendbody = 0;
} else if (clval >= 0 && clval == req->resp_len) {
/* Reuse C-L header */
} else {
http_Unset(req->resp, H_Content_Length);
if (req->resp_len >= 0 && sendbody)
http_PrintfHeader(req->resp,
"Content-Length: %jd", req->resp_len);
}
req->transport->deliver(req, boc, sendbody);
VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) {
if (boc != NULL) {
HSH_Abandon(req->objcore);
ObjWaitState(req->objcore, BOS_FINISHED);
}
ObjSlim(wrk, req->objcore);
}
if (boc != NULL)
HSH_DerefBoc(wrk, req->objcore);
(void)HSH_DerefObjCore(wrk, &req->objcore);
http_Teardown(req->resp);
return (REQ_FSM_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
AN(bo->vbc);
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);
if (bo->do_esi && bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
} else if (bo->do_esi) {
(void)VFP_Push(bo->vfc, &vfp_esi, 1);
} else if (bo->do_gzip) {
(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
} else if (bo->is_gzip && !bo->do_gunzip) {
(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
if (VFP_Open(bo->vfc)) {
(void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open");
bo->doclose = SC_RX_BODY;
return (F_STP_ERROR);
}
if (vbf_beresp2obj(bo)) {
(void)VFP_Error(bo->vfc, "Could not get storage");
bo->doclose = SC_RX_BODY;
return (F_STP_ERROR);
}
assert(WRW_IsReleased(wrk));
obj = bo->fetch_obj;
bo->vfc->body = obj->body;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (bo->do_gzip || bo->do_gunzip)
obj->changed_gzip = 1;
if (bo->htc.body_status != BS_NONE)
V1F_Setup_Fetch(bo);
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
assert (bo->state == BOS_REQ_DONE);
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
bo->htc.body_status, body_status_2str(bo->htc.body_status),
bo->do_stream ? "stream" : "-");
if (bo->htc.body_status != BS_NONE) {
assert(bo->htc.body_status != BS_ERROR);
VFP_Fetch_Body(bo);
bo->acct.beresp_bodybytes = bo->vfc->bodybytes;
}
if (bo->vfc->failed && !bo->do_stream) {
assert(bo->state < BOS_STREAM);
if (bo->fetch_obj != NULL) {
ObjFreeObj(bo->fetch_objcore, bo->stats);
bo->fetch_obj = NULL;
}
return (F_STP_ERROR);
}
if (bo->vfc->failed)
return (F_STP_FAIL);
if (bo->do_stream)
assert(bo->state == BOS_STREAM);
else {
assert(bo->state == BOS_REQ_DONE);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && bo->doclose == SC_NULL) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
if (bo->ims_obj != NULL)
EXP_Rearm(bo->ims_obj->objcore,
bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
struct object *obj;
struct objiter *oi;
void *sp;
ssize_t sl, al, tl;
struct storage *st;
enum objiter_status ois;
char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
if (bo->ims_obj->changed_gzip) {
/*
* If we modified the gzip status of the IMS object, that
* must control the C-E header, if any.
*/
http_Unset(bo->beresp, H_Content_Encoding);
if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p))
http_PrintfHeader(bo->beresp,
"Content-Encoding: %s", p);
}
AZ(vbf_beresp2obj(bo));
obj = bo->fetch_obj;
bo->vfc->body = obj->body;
if (bo->ims_obj->esidata != NULL) {
sl = bo->ims_obj->esidata->len;
obj->esidata = STV_alloc(bo->vfc, sl);
if (obj->esidata == NULL || obj->esidata->space < sl) {
VSLb(bo->vsl, SLT_Error,
"No space for %zd bytes of ESI data", sl);
return (F_STP_FAIL);
}
memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
obj->esidata->len = sl;
}
obj->gziped = bo->ims_obj->gziped;
obj->gzip_start = bo->ims_obj->gzip_start;
obj->gzip_last = bo->ims_obj->gzip_last;
obj->gzip_stop = bo->ims_obj->gzip_stop;
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream) {
HSH_Unbusy(&wrk->stats, obj->objcore);
VBO_setstate(bo, BOS_STREAM);
}
st = NULL;
al = 0;
oi = ObjIterBegin(wrk, bo->ims_obj);
do {
ois = ObjIter(oi, &sp, &sl);
while (sl > 0) {
if (st == NULL)
st = VFP_GetStorage(bo->vfc,
bo->ims_obj->len - al);
if (st == NULL)
break;
tl = sl;
if (tl > st->space - st->len)
tl = st->space - st->len;
memcpy(st->ptr + st->len, sp, tl);
al += tl;
sp = (char *)sp + tl;
sl -= tl;
VBO_extend(bo, tl);
if (st->len == st->space)
st = NULL;
}
} while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM));
ObjIterEnd(&oi);
if (bo->vfc->failed)
return (F_STP_FAIL);
if (!bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(al == bo->ims_obj->len);
assert(obj->len == al);
EXP_Rearm(bo->ims_obj->objcore,
bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
/* Recycle the backend connection before setting BOS_FINISHED to
give predictable backend reuse behavior for varnishtest */
if (bo->vbc != NULL && bo->doclose == SC_NULL) {
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
VBO_setstate(bo, BOS_FINISHED);
VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
return (F_STP_DONE);
}
void
RFC2616_Ttl(struct busyobj *bo)
{
unsigned max_age, age;
double h_date, h_expires;
char *p;
const struct http *hp;
struct exp *expp;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
expp = &bo->exp;
hp = bo->beresp;
assert(bo->t_fetch != 0.0 && !isnan(bo->t_fetch));
expp->t_origin = bo->t_fetch;
expp->ttl = cache_param->default_ttl;
expp->grace = cache_param->default_grace;
expp->keep = cache_param->default_keep;
max_age = age = 0;
h_expires = 0;
h_date = 0;
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges yet, so 206 is out.
*/
if (http_GetHdr(hp, H_Age, &p)) {
/*
* We deliberately run with partial results, rather than
* reject the Age: header outright. This will be future
* compatible with fractional seconds.
*/
age = strtoul(p, NULL, 10);
expp->t_origin -= age;
}
if (http_GetHdr(hp, H_Expires, &p))
h_expires = VTIM_parse(p);
if (http_GetHdr(hp, H_Date, &p))
h_date = VTIM_parse(p);
switch (http_GetStatus(hp)) {
default:
expp->ttl = -1.;
break;
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 302: /* Moved Temporarily */
case 307: /* Temporary Redirect */
case 410: /* Gone */
case 404: /* Not Found */
/*
* First find any relative specification from the backend
* These take precedence according to RFC2616, 13.2.4
*/
if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
p != NULL) {
if (*p == '-')
max_age = 0;
else
max_age = strtoul(p, NULL, 0);
if (age > max_age)
expp->ttl = 0;
else
expp->ttl = max_age - age;
break;
}
/* No expire header, fall back to default */
if (h_expires == 0)
break;
/* If backend told us it is expired already, don't cache. */
if (h_expires < h_date) {
expp->ttl = 0;
break;
}
if (h_date == 0 ||
fabs(h_date - bo->t_fetch) < cache_param->clock_skew) {
/*
* If we have no Date: header or if it is
* sufficiently close to our clock we will
* trust Expires: relative to our own clock.
*/
if (h_expires < bo->t_fetch)
expp->ttl = 0;
else
expp->ttl = h_expires - bo->t_fetch;
break;
} else {
/*
* But even if the clocks are out of whack we can still
* derive a relative time from the two headers.
* (the negative ttl case is caught above)
*/
expp->ttl = (int)(h_expires - h_date);
}
}
/* calculated TTL, Our time, Date, Expires, max-age, age */
VSLb(bo->vsl, SLT_TTL,
"RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
expp->ttl, -1., -1., bo->t_fetch,
expp->t_origin, h_date, h_expires, max_age);
}
void
VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
struct object *oldobj, enum vbf_fetch_mode_e mode)
{
struct busyobj *bo;
const char *how;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
CHECK_OBJ_ORNULL(oldobj, OBJECT_MAGIC);
switch(mode) {
case VBF_PASS: how = "pass"; break;
case VBF_NORMAL: how = "fetch"; break;
case VBF_BACKGROUND: how = "bgfetch"; break;
default: WRONG("Wrong fetch mode");
}
bo = VBO_GetBusyObj(wrk, req);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
VSLb(bo->vsl, SLT_Begin, "bereq %u %s ", VXID(req->vsl->wid), how);
VSLb(req->vsl, SLT_Link, "bereq %u %s ", VXID(bo->vsl->wid), how);
THR_SetBusyobj(bo);
bo->refcount = 2;
oc->busyobj = bo;
CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
if (mode == VBF_PASS)
bo->do_pass = 1;
bo->vary = req->vary_b;
req->vary_b = NULL;
if (mode != VBF_BACKGROUND)
HSH_Ref(oc);
bo->fetch_objcore = oc;
AZ(bo->ims_obj);
if (oldobj != NULL) {
if (http_GetHdr(oldobj->http, H_Last_Modified, NULL) ||
http_GetHdr(oldobj->http, H_ETag, NULL)) {
assert(oldobj->objcore->refcnt > 0);
HSH_Ref(oldobj->objcore);
bo->ims_obj = oldobj;
}
}
AZ(bo->req);
bo->req = req;
bo->fetch_task.priv = bo;
bo->fetch_task.func = vbf_fetch_thread;
if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
vbf_fetch_thread(wrk, bo);
if (mode == VBF_BACKGROUND) {
VBO_waitstate(bo, BOS_REQ_DONE);
} else {
VBO_waitstate(bo, BOS_STREAM);
if (bo->state == BOS_FAILED) {
AN((oc->flags & OC_F_FAILED));
} else {
AZ(bo->fetch_objcore->flags & OC_F_BUSY);
}
}
VSLb_ts_req(req, "Fetch", W_TIM_real(wrk));
THR_SetBusyobj(NULL);
VBO_DerefBusyObj(wrk, &bo);
}
static int
vbf_figure_out_vfp(struct busyobj *bo)
{
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* On partial responses (206 on pass), we fail if do_esi is
* requested because it could leak partial esi-directives, and
* ignore gzipery, because it makes no sense.
*
*/
if (http_GetStatus(bo->beresp) == 206) {
if (bo->do_esi) {
VSLb(bo->vsl, SLT_VCL_Error,
"beresp.do_esi on partial response");
return (-1);
}
bo->do_gzip = bo->do_gunzip = 0;
return (0);
}
/* No body or no GZIP support -> done */
if (bo->htc->body_status == BS_NONE ||
bo->htc->content_length == 0 ||
!cache_param->http_gzip_support) {
http_Unset(bo->beresp, H_Content_Encoding);
bo->do_gzip = bo->do_gunzip = 0;
bo->do_stream = 0;
return (0);
}
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* We wont gzip unless if it already is gzip'ed */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL)
return (-1);
if (bo->do_esi && (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)))
return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0);
if (bo->do_esi)
return (VFP_Push(bo->vfc, &VFP_esi) == NULL ? -1 : 0);
if (bo->do_gzip)
return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0);
if (bo->is_gzip && !bo->do_gunzip)
return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0);
return (0);
}
static int
vbf_beresp2obj(struct busyobj *bo)
{
unsigned l;
char *b;
struct vsb *vary = NULL;
int varyl = 0;
uint16_t nhttp;
struct object *obj;
struct http *hp, *hp2;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
obj = vbf_allocobj(bo, l, nhttp);
if (obj == NULL)
return (-1);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = SLT_ObjMethod;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->fetch_objcore->exp.t_origin);
/* Disassociate the obj from the bo's workspace */
hp2->ws = NULL;
return (0);
}
static int
vbf_beresp2obj(struct busyobj *bo)
{
unsigned l, l2;
const char *b;
uint8_t *bp;
struct vsb *vary = NULL;
int varyl = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += PRNDUP((intptr_t)varyl);
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l2 = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS);
l += l2;
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (!vbf_allocobj(bo, l)) {
if (vary != NULL)
VSB_destroy(&vary);
AZ(vary);
return (-1);
}
if (vary != NULL) {
AN(ObjSetAttr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl,
VSB_data(vary)));
VSB_destroy(&vary);
}
AZ(ObjSetU32(bo->wrk, bo->fetch_objcore, OA_VXID, VXID(bo->vsl->wid)));
/* for HTTP_Encode() VSLH call */
bo->beresp->logtag = SLT_ObjMethod;
/* Filter into object */
bp = ObjSetAttr(bo->wrk, bo->fetch_objcore, OA_HEADERS, l2, NULL);
AN(bp);
HTTP_Encode(bo->beresp, bp, l2,
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS);
if (http_GetHdr(bo->beresp, H_Last_Modified, &b))
AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED,
VTIM_parse(b)));
else
AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED,
floor(bo->fetch_objcore->t_origin)));
return (0);
}
static int
cnt_fetch(struct sess *sp)
{
int i, transient;
struct http *hp, *hp2;
char *b;
unsigned handling;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
AN(sp->director);
AZ(sp->vbe);
/* sp->wrk->http[0] is (still) bereq */
sp->wrk->beresp = &sp->wrk->http[1];
http_Setup(sp->wrk->beresp, sp->wrk->ws);
i = FetchHdr(sp);
/*
* Save a copy before it might get mangled in VCL. When it comes to
* dealing with the body, we want to see the unadultered headers.
*/
sp->wrk->beresp1 = &sp->wrk->http[2];
*sp->wrk->beresp1 = *sp->wrk->beresp;
if (i) {
if (sp->objhead) {
CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
HSH_DerefObjCore(sp);
}
AZ(sp->obj);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
sp->err_code = http_GetStatus(sp->wrk->beresp);
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges yet, so 206 is out.
*/
switch (sp->err_code) {
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 302: /* Moved Temporarily */
case 410: /* Gone */
case 404: /* Not Found */
sp->wrk->cacheable = 1;
break;
default:
sp->wrk->cacheable = 0;
break;
}
sp->wrk->entered = TIM_real();
sp->wrk->age = 0;
sp->wrk->ttl = RFC2616_Ttl(sp);
if (sp->wrk->ttl == 0.)
sp->wrk->cacheable = 0;
sp->wrk->do_esi = 0;
sp->wrk->grace = NAN;
VCL_fetch_method(sp);
/*
* When we fetch the body, we may hit the LRU cleanup and that
* will overwrite sp->handling, so we have to save our plans
* here.
*/
handling = sp->handling;
if (sp->objhead == NULL)
transient = 1;
else if (sp->handling == VCL_RET_DELIVER)
transient = 0;
else
transient = 1;
/*
* XXX: If we have a Length: header, we should allocate the body
* XXX: also.
*/
sp->obj = HSH_NewObject(sp, transient);
if (sp->objhead != NULL) {
CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
sp->objcore->obj = sp->obj;
sp->obj->objcore = sp->objcore;
sp->obj->objhead = sp->objhead;
sp->objhead = NULL; /* refcnt follows pointer. */
sp->objcore = NULL; /* refcnt follows pointer. */
}
BAN_NewObj(sp->obj);
sp->obj->xid = sp->xid;
sp->obj->response = sp->err_code;
sp->obj->cacheable = sp->wrk->cacheable;
sp->obj->ttl = sp->wrk->ttl;
sp->obj->grace = sp->wrk->grace;
if (sp->obj->ttl == 0. && sp->obj->grace == 0.)
sp->obj->cacheable = 0;
sp->obj->age = sp->wrk->age;
sp->obj->entered = sp->wrk->entered;
WS_Assert(sp->obj->ws_o);
/* Filter into object */
hp = sp->wrk->beresp;
hp2 = sp->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(sp->wrk, sp->fd, hp2, hp, HTTPH_A_INS);
http_CopyHome(sp->wrk, sp->fd, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
sp->obj->last_modified = TIM_parse(b);
i = FetchBody(sp);
AZ(sp->wrk->wfd);
AZ(sp->vbe);
AN(sp->director);
if (i) {
HSH_Drop(sp);
AZ(sp->obj);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (!transient)
HSH_Object(sp);
if (sp->wrk->do_esi)
ESI_Parse(sp);
switch (handling) {
case VCL_RET_RESTART:
HSH_Drop(sp);
sp->director = NULL;
sp->restarts++;
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_RECV;
return (0);
case VCL_RET_PASS:
if (sp->obj->objcore != NULL)
sp->obj->objcore->flags |= OC_F_PASS;
if (sp->obj->ttl - sp->t_req < params->default_ttl)
sp->obj->ttl = sp->t_req + params->default_ttl;
break;
case VCL_RET_DELIVER:
break;
case VCL_RET_ERROR:
HSH_Drop(sp);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
sp->obj->cacheable = 1;
if (sp->obj->objhead != NULL) {
VRY_Create(sp);
EXP_Insert(sp->obj);
AN(sp->obj->ban);
HSH_Unbusy(sp);
}
sp->acct_req.fetch++;
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_DELIVER;
return (0);
}
