static enum fetch_step
vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
{
const char *q;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
assert(bo->fetch_objcore->boc->state == BOS_INVALID);
AZ(bo->storage);
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
http_FilterReq(bo->bereq0, bo->req->http,
bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
if (!bo->do_pass) {
http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
if (cache_param->http_gzip_support)
http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
} else {
AZ(bo->stale_oc);
if (bo->bereq0->protover > 11)
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
}
http_CopyHome(bo->bereq0);
if (bo->stale_oc != NULL &&
ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) &&
(bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) {
AZ(bo->stale_oc->flags & OC_F_PASS);
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified);
if (q != NULL)
http_PrintfHeader(bo->bereq0,
"If-Modified-Since: %s", q);
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_ETag);
if (q != NULL)
http_PrintfHeader(bo->bereq0,
"If-None-Match: %s", q);
}
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
bo->ws_bo = WS_Snapshot(bo->ws);
HTTP_Copy(bo->bereq, bo->bereq0);
if (bo->req->req_body_status == REQ_BODY_NONE) {
bo->req = NULL;
ObjSetState(bo->wrk, bo->fetch_objcore, BOS_REQ_DONE);
}
return (F_STP_STARTFETCH);
}
static enum fetch_step
vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
{
const char *q;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
assert(bo->state == BOS_INVALID);
AZ(bo->storage_hint);
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
http_FilterReq(bo->bereq0, bo->req->http,
bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
if (!bo->do_pass) {
http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
if (cache_param->http_gzip_support)
http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
AN(bo->req);
bo->req = NULL;
http_CopyHome(bo->bereq0);
} else
AZ(bo->stale_oc);
if (bo->stale_oc != NULL) {
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified);
if (q != NULL)
http_PrintfHeader(bo->bereq0,
"If-Modified-Since: %s", q);
q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_ETag);
if (q != NULL)
http_PrintfHeader(bo->bereq0,
"If-None-Match: %s", q);
}
HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
bo->ws_bo = WS_Snapshot(bo->ws);
HTTP_Copy(bo->bereq, bo->bereq0);
VBO_setstate(bo, BOS_REQ_DONE);
return (F_STP_STARTFETCH);
}
static enum fetch_step
vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
{
char *p;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
assert(bo->state == BOS_INVALID);
AZ(bo->vbc);
assert(bo->doclose == SC_NULL);
AZ(bo->storage_hint);
HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
http_FilterReq(bo->bereq0, bo->req->http,
bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
if (!bo->do_pass) {
http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
if (cache_param->http_gzip_support)
http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
AN(bo->req);
bo->req = NULL;
http_CopyHome(bo->bereq0);
}
if (bo->ims_obj != NULL && bo->ims_obj->http->status == 200) {
if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
http_PrintfHeader(bo->bereq0,
"If-Modified-Since: %s", p);
}
if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
http_PrintfHeader(bo->bereq0,
"If-None-Match: %s", p);
}
}
VBO_setstate(bo, BOS_REQ_DONE);
return (F_STP_STARTFETCH);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
unsigned l;
uint16_t nhttp;
struct object *obj;
struct objiter *oi;
void *sp;
ssize_t sl, al, tl, vl;
struct storage *st;
enum objiter_status ois;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
l = 0;
if (bo->ims_obj->vary != NULL) {
vl = VRY_Validate(bo->ims_obj->vary);
l += vl;
} else
vl = 0;
l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
obj->gziped = bo->ims_obj->gziped;
obj->gzip_start = bo->ims_obj->gzip_start;
obj->gzip_last = bo->ims_obj->gzip_last;
obj->gzip_stop = bo->ims_obj->gzip_stop;
/* XXX: ESI */
if (bo->ims_obj->vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
bo->ims_obj->vary, vl);
assert(vl == VRY_Validate(obj->vary));
}
obj->vxid = bo->vsl->wid;
obj->http->logtag = HTTP_Obj;
/* XXX: we should have our own HTTP_A_CONDFETCH */
http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
http_CopyHome(obj->http);
AZ(WS_Overflowed(bo->ws_o));
VBO_setstate(bo, BOS_FETCHING);
HSH_Unbusy(&wrk->stats, obj->objcore);
if (!(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
st = NULL;
al = 0;
oi = ObjIterBegin(wrk, bo->ims_obj);
do {
ois = ObjIter(oi, &sp, &sl);
while (sl > 0) {
if (st == NULL) {
st = VFP_GetStorage(bo, bo->ims_obj->len - al);
XXXAN(st);
}
tl = sl;
if (tl > st->space - st->len)
tl = st->space - st->len;
memcpy(st->ptr + st->len, sp, tl);
al += tl;
sp = (char *)sp + tl;
sl -= tl;
VBO_extend(bo, al);
if (st->len == st->space)
st = NULL;
}
} while (ois == OIS_DATA || ois == OIS_STREAM);
ObjIterEnd(&oi);
bo->stats = NULL;
assert(al == bo->ims_obj->len);
assert(obj->len == al);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
HSH_Complete(obj->objcore);
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
static int
vbf_beresp2obj(struct busyobj *bo)
{
unsigned l;
char *b;
struct vsb *vary = NULL;
int varyl = 0;
uint16_t nhttp;
struct object *obj;
struct http *hp, *hp2;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
obj = vbf_allocobj(bo, l, nhttp);
if (obj == NULL)
return (-1);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = SLT_ObjMethod;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->fetch_objcore->exp.t_origin);
/* Disassociate the obj from the bo's workspace */
hp2->ws = NULL;
return (0);
}
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(req->handling == VCL_RET_DELIVER);
if (req->objcore->objhead == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
bo->exp.ttl = -1.;
} else if (bo->do_pass) {
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->do_esi || req->esi_level > 0)
bo->do_stream = 0;
if (!req->wantbody)
bo->do_stream = 0;
/* No reason to try streaming a non-existing body */
if (bo->body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (req->objcore->objhead != NULL) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
vary = VRY_Create(req, bo->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (bo->exp.ttl < cache_param->shortlived ||
req->objcore == NULL)
req->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
nhttp);
if (req->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
l, nhttp);
}
bo->stats = NULL;
if (req->obj == NULL) {
req->err_code = 503;
req->req_step = R_STP_ERROR;
VDI_CloseFd(&bo->vbc);
VBO_DerefBusyObj(wrk, &req->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
req->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = req->obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
req->obj->gziped = 1;
if (vary != NULL) {
req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
VSB_data(vary), varyl);
AN(req->obj->vary);
VRY_Validate(req->obj->vary);
VSB_delete(vary);
}
req->obj->vxid = bo->vsl->wid;
req->obj->response = req->err_code;
WS_Assert(req->obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = req->obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
req->obj->last_modified = VTIM_parse(b);
else
req->obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (req->obj->response == 200 &&
req->http->conds &&
RFC2616_Do_Cond(req))
bo->do_stream = 0;
/*
* Ready to fetch the body
*/
bo->fetch_task.func = FetchBody;
bo->fetch_task.priv = bo;
assert(bo->refcount == 2); /* one for each thread */
if (req->obj->objcore->objhead != NULL) {
EXP_Insert(req->obj);
AN(req->obj->objcore->ban);
AZ(req->obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, req->obj->objcore);
}
if (!bo->do_stream ||
Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
FetchBody(wrk, bo);
if (req->obj->objcore->objhead != NULL)
HSH_Ref(req->obj->objcore);
if (bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &req->busyobj);
} else if (bo->state == BOS_FAILED) {
/* handle early failures */
HSH_Deref(&wrk->stats, NULL, &req->obj);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->err_code = 503;
req->req_step = R_STP_ERROR;
return (0);
}
assert(WRW_IsReleased(wrk));
req->req_step = R_STP_PREPRESP;
return (0);
}
static int
cnt_fetch(struct sess *sp)
{
int i, transient;
struct http *hp, *hp2;
char *b;
unsigned handling;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
AN(sp->director);
AZ(sp->vbe);
/* sp->wrk->http[0] is (still) bereq */
sp->wrk->beresp = &sp->wrk->http[1];
http_Setup(sp->wrk->beresp, sp->wrk->ws);
i = FetchHdr(sp);
/*
* Save a copy before it might get mangled in VCL. When it comes to
* dealing with the body, we want to see the unadultered headers.
*/
sp->wrk->beresp1 = &sp->wrk->http[2];
*sp->wrk->beresp1 = *sp->wrk->beresp;
if (i) {
if (sp->objhead) {
CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
HSH_DerefObjCore(sp);
}
AZ(sp->obj);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
sp->err_code = http_GetStatus(sp->wrk->beresp);
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges yet, so 206 is out.
*/
switch (sp->err_code) {
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 302: /* Moved Temporarily */
case 410: /* Gone */
case 404: /* Not Found */
sp->wrk->cacheable = 1;
break;
default:
sp->wrk->cacheable = 0;
break;
}
sp->wrk->entered = TIM_real();
sp->wrk->age = 0;
sp->wrk->ttl = RFC2616_Ttl(sp);
if (sp->wrk->ttl == 0.)
sp->wrk->cacheable = 0;
sp->wrk->do_esi = 0;
sp->wrk->grace = NAN;
VCL_fetch_method(sp);
/*
* When we fetch the body, we may hit the LRU cleanup and that
* will overwrite sp->handling, so we have to save our plans
* here.
*/
handling = sp->handling;
if (sp->objhead == NULL)
transient = 1;
else if (sp->handling == VCL_RET_DELIVER)
transient = 0;
else
transient = 1;
/*
* XXX: If we have a Length: header, we should allocate the body
* XXX: also.
*/
sp->obj = HSH_NewObject(sp, transient);
if (sp->objhead != NULL) {
CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
sp->objcore->obj = sp->obj;
sp->obj->objcore = sp->objcore;
sp->obj->objhead = sp->objhead;
sp->objhead = NULL; /* refcnt follows pointer. */
sp->objcore = NULL; /* refcnt follows pointer. */
}
BAN_NewObj(sp->obj);
sp->obj->xid = sp->xid;
sp->obj->response = sp->err_code;
sp->obj->cacheable = sp->wrk->cacheable;
sp->obj->ttl = sp->wrk->ttl;
sp->obj->grace = sp->wrk->grace;
if (sp->obj->ttl == 0. && sp->obj->grace == 0.)
sp->obj->cacheable = 0;
sp->obj->age = sp->wrk->age;
sp->obj->entered = sp->wrk->entered;
WS_Assert(sp->obj->ws_o);
/* Filter into object */
hp = sp->wrk->beresp;
hp2 = sp->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(sp->wrk, sp->fd, hp2, hp, HTTPH_A_INS);
http_CopyHome(sp->wrk, sp->fd, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
sp->obj->last_modified = TIM_parse(b);
i = FetchBody(sp);
AZ(sp->wrk->wfd);
AZ(sp->vbe);
AN(sp->director);
if (i) {
HSH_Drop(sp);
AZ(sp->obj);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (!transient)
HSH_Object(sp);
if (sp->wrk->do_esi)
ESI_Parse(sp);
switch (handling) {
case VCL_RET_RESTART:
HSH_Drop(sp);
sp->director = NULL;
sp->restarts++;
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_RECV;
return (0);
case VCL_RET_PASS:
if (sp->obj->objcore != NULL)
sp->obj->objcore->flags |= OC_F_PASS;
if (sp->obj->ttl - sp->t_req < params->default_ttl)
sp->obj->ttl = sp->t_req + params->default_ttl;
break;
case VCL_RET_DELIVER:
break;
case VCL_RET_ERROR:
HSH_Drop(sp);
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_ERROR;
return (0);
default:
WRONG("Illegal action in vcl_fetch{}");
}
sp->obj->cacheable = 1;
if (sp->obj->objhead != NULL) {
VRY_Create(sp);
EXP_Insert(sp->obj);
AN(sp->obj->ban);
HSH_Unbusy(sp);
}
sp->acct_req.fetch++;
sp->wrk->bereq = NULL;
sp->wrk->beresp = NULL;
sp->wrk->beresp1 = NULL;
sp->step = STP_DELIVER;
return (0);
}
static int
cnt_fetchbody(struct sess *sp)
{
int i;
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
sp->handling == VCL_RET_DELIVER);
if (wrk->objcore == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
wrk->busyobj->exp.ttl = -1.;
} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
/* pass from vcl_fetch{} -> hit-for-pass */
/* XXX: the bereq was not filtered pass... */
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
wrk->busyobj->is_gzip =
http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
wrk->busyobj->is_gunzip =
!http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
wrk->busyobj->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (wrk->busyobj->do_gunzip)
http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
wrk->busyobj->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (wrk->busyobj->do_gzip)
http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
"Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (wrk->busyobj->do_esi)
wrk->busyobj->vfp = &vfp_esi;
else if (wrk->busyobj->do_gunzip)
wrk->busyobj->vfp = &vfp_gunzip;
else if (wrk->busyobj->do_gzip)
wrk->busyobj->vfp = &vfp_gzip;
else if (wrk->busyobj->is_gzip)
wrk->busyobj->vfp = &vfp_testgzip;
if (wrk->busyobj->do_esi || sp->esi_level > 0)
wrk->busyobj->do_stream = 0;
if (!sp->wantbody)
wrk->busyobj->do_stream = 0;
l = http_EstimateWS(wrk->busyobj->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
vary = VRY_Create(sp, wrk->busyobj->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
wrk->objcore == NULL)
wrk->storage_hint = TRANSIENT_STORAGE;
wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
if (wrk->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
if (wrk->busyobj->exp.ttl > cache_param->shortlived)
wrk->busyobj->exp.ttl = cache_param->shortlived;
wrk->busyobj->exp.grace = 0.0;
wrk->busyobj->exp.keep = 0.0;
}
if (wrk->obj == NULL) {
sp->err_code = 503;
sp->step = STP_ERROR;
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
wrk->storage_hint = NULL;
if (wrk->busyobj->do_gzip ||
(wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
wrk->obj->gziped = 1;
if (vary != NULL) {
wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
AN(wrk->obj->vary);
memcpy(wrk->obj->vary, VSB_data(vary), varyl);
VRY_Validate(wrk->obj->vary);
VSB_delete(vary);
}
wrk->obj->xid = sp->xid;
wrk->obj->response = sp->err_code;
WS_Assert(wrk->obj->ws_o);
/* Filter into object */
hp = wrk->busyobj->beresp;
hp2 = wrk->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(wrk, sp->vsl_id, hp2, hp,
pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(wrk, sp->vsl_id, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
wrk->obj->last_modified = VTIM_parse(b);
else
wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (wrk->obj->response == 200 &&
sp->http->conds &&
RFC2616_Do_Cond(sp))
wrk->busyobj->do_stream = 0;
AssertObjCorePassOrBusy(wrk->obj->objcore);
if (wrk->busyobj->do_stream) {
sp->step = STP_PREPRESP;
return (0);
}
/* Use unmodified headers*/
i = FetchBody(wrk, wrk->obj);
http_Setup(wrk->busyobj->bereq, NULL);
http_Setup(wrk->busyobj->beresp, NULL);
wrk->busyobj->vfp = NULL;
assert(WRW_IsReleased(wrk));
AZ(wrk->busyobj->vbc);
AN(sp->director);
if (i) {
HSH_Drop(wrk);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
AZ(wrk->obj);
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (wrk->obj->objcore != NULL) {
EXP_Insert(wrk->obj);
AN(wrk->obj->objcore);
AN(wrk->obj->objcore->ban);
HSH_Unbusy(wrk);
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
wrk->acct_tmp.fetch++;
sp->step = STP_PREPRESP;
return (0);
}