int
RFC2616_Do_Cond(const struct req *req)
{
char *p, *e;
double ims;
int do_cond = 0;
/* RFC 2616 13.3.4 states we need to match both ETag
and If-Modified-Since if present*/
if (http_GetHdr(req->http, H_If_Modified_Since, &p) ) {
if (!req->obj->last_modified)
return (0);
ims = VTIM_parse(p);
if (ims > req->t_req) /* [RFC2616 14.25] */
return (0);
if (req->obj->last_modified > ims)
return (0);
do_cond = 1;
}
if (http_GetHdr(req->http, H_If_None_Match, &p) &&
http_GetHdr(req->obj->http, H_ETag, &e)) {
if (strcmp(p,e) != 0)
return (0);
do_cond = 1;
}
return (do_cond);
}
vmod_time(VRT_CTX, VCL_STRING p, VCL_TIME d)
{
double r;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
if (p == NULL)
return (d);
r = VTIM_parse(p);
if (r)
return (r);
return (vmod_real(ctx, p, d));
}
int
RFC2616_Do_Cond(const struct req *req)
{
const char *p, *e, *l;
double ims, lm;
/*
* We MUST ignore If-Modified-Since if we have an If-None-Match
* header [RFC7232 3.3 p16].
*/
if (http_GetHdr(req->http, H_If_None_Match, &p)) {
if (!http_GetHdr(req->resp, H_ETag, &e))
return (0);
if (http_GetHdr(req->http, H_Range, NULL))
return (rfc2616_strong_compare(p, e));
else
return (rfc2616_weak_compare(p, e));
}
if (http_GetHdr(req->http, H_If_Modified_Since, &p)) {
ims = VTIM_parse(p);
if (!ims || ims > req->t_req) /* [RFC7232 3.3 p16] */
return (0);
if (http_GetHdr(req->resp, H_Last_Modified, &l)) {
lm = VTIM_parse(l);
if (!lm || lm > ims)
return (0);
return (1);
}
AZ(ObjGetDouble(req->wrk, req->objcore, OA_LASTMODIFIED, &lm));
if (lm > ims)
return (0);
return (1);
}
return (0);
}
static void
tst(const char *s, time_t good)
{
time_t t;
char buf[BUFSIZ];
t = VTIM_parse(s);
VTIM_format(t, buf);
printf("%-30s -> %12jd -> %s\n", s, (intmax_t)t, buf);
if (t != good) {
printf("Parse error! Got: %jd should have %jd diff %jd\n",
(intmax_t)t, (intmax_t)good, (intmax_t)(t - good));
exit(4);
}
}
int
main(int argc, char **argv)
{
time_t t;
char buf[BUFSIZ];
time(&t);
memset(buf, 0x55, sizeof buf);
VTIM_format(t, buf);
printf("scan = %d <%s>\n", VTIM_parse(buf), buf);
/* Examples from RFC2616 section 3.3.1 */
tst("Sun, 06 Nov 1994 08:49:37 GMT", 784111777);
tst("Sunday, 06-Nov-94 08:49:37 GMT", 784111777);
tst("Sun Nov 6 08:49:37 1994", 784111777);
tst_delta();
return (0);
}
static int
vbf_beresp2obj(struct busyobj *bo)
{
unsigned l, l2;
const char *b;
uint8_t *bp;
struct vsb *vary = NULL;
int varyl = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += PRNDUP((intptr_t)varyl);
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l2 = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS);
l += l2;
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (!vbf_allocobj(bo, l)) {
if (vary != NULL)
VSB_destroy(&vary);
AZ(vary);
return (-1);
}
if (vary != NULL) {
AN(ObjSetAttr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl,
VSB_data(vary)));
VSB_destroy(&vary);
}
AZ(ObjSetU32(bo->wrk, bo->fetch_objcore, OA_VXID, VXID(bo->vsl->wid)));
/* for HTTP_Encode() VSLH call */
bo->beresp->logtag = SLT_ObjMethod;
/* Filter into object */
bp = ObjSetAttr(bo->wrk, bo->fetch_objcore, OA_HEADERS, l2, NULL);
AN(bp);
HTTP_Encode(bo->beresp, bp, l2,
bo->uncacheable ? HTTPH_A_PASS : HTTPH_A_INS);
if (http_GetHdr(bo->beresp, H_Last_Modified, &b))
AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED,
VTIM_parse(b)));
else
AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED,
floor(bo->fetch_objcore->t_origin)));
return (0);
}
void
RFC2616_Ttl(struct busyobj *bo)
{
unsigned max_age, age;
double h_date, h_expires;
char *p;
const struct http *hp;
struct exp *expp;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
expp = &bo->exp;
hp = bo->beresp;
assert(bo->t_fetch != 0.0 && !isnan(bo->t_fetch));
expp->t_origin = bo->t_fetch;
expp->ttl = cache_param->default_ttl;
expp->grace = cache_param->default_grace;
expp->keep = cache_param->default_keep;
max_age = age = 0;
h_expires = 0;
h_date = 0;
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges yet, so 206 is out.
*/
if (http_GetHdr(hp, H_Age, &p)) {
/*
* We deliberately run with partial results, rather than
* reject the Age: header outright. This will be future
* compatible with fractional seconds.
*/
age = strtoul(p, NULL, 10);
expp->t_origin -= age;
}
if (http_GetHdr(hp, H_Expires, &p))
h_expires = VTIM_parse(p);
if (http_GetHdr(hp, H_Date, &p))
h_date = VTIM_parse(p);
switch (http_GetStatus(hp)) {
default:
expp->ttl = -1.;
break;
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 302: /* Moved Temporarily */
case 307: /* Temporary Redirect */
case 410: /* Gone */
case 404: /* Not Found */
/*
* First find any relative specification from the backend
* These take precedence according to RFC2616, 13.2.4
*/
if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
p != NULL) {
if (*p == '-')
max_age = 0;
else
max_age = strtoul(p, NULL, 0);
if (age > max_age)
expp->ttl = 0;
else
expp->ttl = max_age - age;
break;
}
/* No expire header, fall back to default */
if (h_expires == 0)
break;
/* If backend told us it is expired already, don't cache. */
if (h_expires < h_date) {
expp->ttl = 0;
break;
}
if (h_date == 0 ||
fabs(h_date - bo->t_fetch) < cache_param->clock_skew) {
/*
* If we have no Date: header or if it is
* sufficiently close to our clock we will
* trust Expires: relative to our own clock.
*/
if (h_expires < bo->t_fetch)
expp->ttl = 0;
else
expp->ttl = h_expires - bo->t_fetch;
break;
} else {
/*
* But even if the clocks are out of whack we can still
* derive a relative time from the two headers.
* (the negative ttl case is caught above)
*/
expp->ttl = (int)(h_expires - h_date);
}
}
/* calculated TTL, Our time, Date, Expires, max-age, age */
VSLb(bo->vsl, SLT_TTL,
"RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
expp->ttl, -1., -1., bo->t_fetch,
expp->t_origin, h_date, h_expires, max_age);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->fetch_objcore->objhead == NULL)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (bo->fetch_objcore->objhead != NULL) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
bo->storage_hint = TRANSIENT_STORAGE;
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
AZ(bo->stats);
bo->stats = &wrk->stats;
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
bo->stats = NULL;
if (obj == NULL) {
AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
bo->fetch_objcore = NULL;
VDI_CloseFd(&bo->vbc);
VBO_setstate(bo, BOS_FAILED);
return (F_STP_ABANDON);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL) {
EXP_Insert(obj);
AN(obj->objcore->ban);
AZ(obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, obj->objcore);
}
if (bo->vfp == NULL)
bo->vfp = &VFP_nop;
assert(bo->state == BOS_INVALID);
VBO_setstate(bo, BOS_FETCHING);
V1F_fetch_body(wrk, bo);
assert(bo->refcount >= 1);
if (obj->objcore->objhead != NULL)
HSH_Ref(obj->objcore);
if (bo->state == BOS_FAILED) {
/* handle early failures */
(void)HSH_Deref(&wrk->stats, NULL, &obj);
return (F_STP_ABANDON);
}
VBO_setstate(bo, BOS_FINISHED);
VBO_DerefBusyObj(wrk, &bo); // XXX ?
return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
static int
vbf_beresp2obj(struct busyobj *bo)
{
unsigned l;
char *b;
struct vsb *vary = NULL;
int varyl = 0;
uint16_t nhttp;
struct object *obj;
struct http *hp, *hp2;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
obj = vbf_allocobj(bo, l, nhttp);
if (obj == NULL)
return (-1);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = SLT_ObjMethod;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->fetch_objcore->exp.t_origin);
/* Disassociate the obj from the bo's workspace */
hp2->ws = NULL;
return (0);
}
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
bo = req->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(req->handling == VCL_RET_DELIVER);
if (req->objcore->objhead == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
bo->exp.ttl = -1.;
} else if (bo->do_pass) {
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (bo->do_esi)
bo->vfp = &vfp_esi;
else if (bo->do_gunzip)
bo->vfp = &vfp_gunzip;
else if (bo->do_gzip)
bo->vfp = &vfp_gzip;
else if (bo->is_gzip)
bo->vfp = &vfp_testgzip;
if (bo->do_esi || req->esi_level > 0)
bo->do_stream = 0;
if (!req->wantbody)
bo->do_stream = 0;
/* No reason to try streaming a non-existing body */
if (bo->body_status == BS_NONE)
bo->do_stream = 0;
l = http_EstimateWS(bo->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (req->objcore->objhead != NULL) {
CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
vary = VRY_Create(req, bo->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (bo->exp.ttl < cache_param->shortlived ||
req->objcore == NULL)
req->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
nhttp);
if (req->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
l, nhttp);
}
bo->stats = NULL;
if (req->obj == NULL) {
req->err_code = 503;
req->req_step = R_STP_ERROR;
VDI_CloseFd(&bo->vbc);
VBO_DerefBusyObj(wrk, &req->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
req->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = req->obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
req->obj->gziped = 1;
if (vary != NULL) {
req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
VSB_data(vary), varyl);
AN(req->obj->vary);
VRY_Validate(req->obj->vary);
VSB_delete(vary);
}
req->obj->vxid = bo->vsl->wid;
req->obj->response = req->err_code;
WS_Assert(req->obj->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = req->obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
req->obj->last_modified = VTIM_parse(b);
else
req->obj->last_modified = floor(bo->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (req->obj->response == 200 &&
req->http->conds &&
RFC2616_Do_Cond(req))
bo->do_stream = 0;
/*
* Ready to fetch the body
*/
bo->fetch_task.func = FetchBody;
bo->fetch_task.priv = bo;
assert(bo->refcount == 2); /* one for each thread */
if (req->obj->objcore->objhead != NULL) {
EXP_Insert(req->obj);
AN(req->obj->objcore->ban);
AZ(req->obj->ws_o->overflow);
HSH_Unbusy(&wrk->stats, req->obj->objcore);
}
if (!bo->do_stream ||
Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
FetchBody(wrk, bo);
if (req->obj->objcore->objhead != NULL)
HSH_Ref(req->obj->objcore);
if (bo->state == BOS_FINISHED) {
VBO_DerefBusyObj(wrk, &req->busyobj);
} else if (bo->state == BOS_FAILED) {
/* handle early failures */
HSH_Deref(&wrk->stats, NULL, &req->obj);
VBO_DerefBusyObj(wrk, &req->busyobj);
req->err_code = 503;
req->req_step = R_STP_ERROR;
return (0);
}
assert(WRW_IsReleased(wrk));
req->req_step = R_STP_PREPRESP;
return (0);
}
int
main(int argc, char **argv)
{
time_t t;
struct tm tm;
double tt;
char buf[BUFSIZ];
char buf1[BUFSIZ];
AZ(setenv("TZ", "UTC", 1));
assert(sizeof t >= 8);
/* Brute force test against libc version */
for (t = -2209852800; t < 20000000000; t += 3599) {
gmtime_r(&t, &tm);
strftime(buf1, sizeof buf1, "%a, %d %b %Y %T GMT", &tm);
VTIM_format(t, buf);
if (strcmp(buf, buf1)) {
printf("libc: <%s> Vtim <%s> %jd\n",
buf1, buf, (intmax_t)t);
exit(2);
}
tt = VTIM_parse(buf1);
if (tt != t) {
VTIM_format(tt, buf);
printf(" fm: %12jd <%s>\n", (intmax_t)t, buf1);
printf(" to: %12.0f <%s>\n", tt, buf);
exit(2);
}
strftime(buf1, sizeof buf1, "%a %b %e %T %Y", &tm);
tt = VTIM_parse(buf1);
if (tt != t) {
VTIM_format(tt, buf);
printf(" fm: %12jd <%s>\n", (intmax_t)t, buf1);
printf(" to: %12.0f <%s>\n", tt, buf);
exit(2);
}
strftime(buf1, sizeof buf1, "%Y-%m-%dT%T", &tm);
tt = VTIM_parse(buf1);
if (tt != t) {
VTIM_format(tt, buf);
printf(" fm: %12jd <%s>\n", (intmax_t)t, buf1);
printf(" to: %12.0f <%s>\n", tt, buf);
exit(2);
}
if (tm.tm_year >= 69 && tm.tm_year < 169) {
strftime(buf1, sizeof buf1, "%A, %d-%b-%y %T GMT", &tm);
tt = VTIM_parse(buf1);
if (tt != t) {
VTIM_format(tt, buf);
printf(" fm: %12jd <%s>\n", (intmax_t)t, buf1);
printf(" to: %12.0f <%s>\n", tt, buf);
exit(2);
}
}
}
/* Examples from RFC2616 section 3.3.1 */
tst("Sun, 06 Nov 1994 08:49:37 GMT", 784111777);
tst("Sunday, 06-Nov-94 08:49:37 GMT", 784111777);
tst("Sun Nov 6 08:49:37 1994", 784111777);
tst("1994-11-06T08:49:37", 784111777);
tst_delta();
return (0);
}
static int
cnt_fetchbody(struct sess *sp)
{
int i;
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0, pass;
struct worker *wrk;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
wrk = sp->wrk;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
sp->handling == VCL_RET_DELIVER);
if (wrk->objcore == NULL) {
/* This is a pass from vcl_recv */
pass = 1;
/* VCL may have fiddled this, but that doesn't help */
wrk->busyobj->exp.ttl = -1.;
} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
/* pass from vcl_fetch{} -> hit-for-pass */
/* XXX: the bereq was not filtered pass... */
pass = 1;
} else {
/* regular object */
pass = 0;
}
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
wrk->busyobj->is_gzip =
http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
wrk->busyobj->is_gunzip =
!http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
wrk->busyobj->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (wrk->busyobj->do_gunzip)
http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
wrk->busyobj->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (wrk->busyobj->do_gzip)
http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
"Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
/* ESI takes precedence and handles gzip/gunzip itself */
if (wrk->busyobj->do_esi)
wrk->busyobj->vfp = &vfp_esi;
else if (wrk->busyobj->do_gunzip)
wrk->busyobj->vfp = &vfp_gunzip;
else if (wrk->busyobj->do_gzip)
wrk->busyobj->vfp = &vfp_gzip;
else if (wrk->busyobj->is_gzip)
wrk->busyobj->vfp = &vfp_testgzip;
if (wrk->busyobj->do_esi || sp->esi_level > 0)
wrk->busyobj->do_stream = 0;
if (!sp->wantbody)
wrk->busyobj->do_stream = 0;
l = http_EstimateWS(wrk->busyobj->beresp,
pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
/* Create Vary instructions */
if (wrk->objcore != NULL) {
CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
vary = VRY_Create(sp, wrk->busyobj->beresp);
if (vary != NULL) {
varyl = VSB_len(vary);
assert(varyl > 0);
l += varyl;
}
}
/*
* Space for producing a Content-Length: header including padding
* A billion gigabytes is enough for anybody.
*/
l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
wrk->objcore == NULL)
wrk->storage_hint = TRANSIENT_STORAGE;
wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
if (wrk->obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
if (wrk->busyobj->exp.ttl > cache_param->shortlived)
wrk->busyobj->exp.ttl = cache_param->shortlived;
wrk->busyobj->exp.grace = 0.0;
wrk->busyobj->exp.keep = 0.0;
}
if (wrk->obj == NULL) {
sp->err_code = 503;
sp->step = STP_ERROR;
VDI_CloseFd(wrk, &wrk->busyobj->vbc);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
return (0);
}
CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
wrk->storage_hint = NULL;
if (wrk->busyobj->do_gzip ||
(wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
wrk->obj->gziped = 1;
if (vary != NULL) {
wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
AN(wrk->obj->vary);
memcpy(wrk->obj->vary, VSB_data(vary), varyl);
VRY_Validate(wrk->obj->vary);
VSB_delete(vary);
}
wrk->obj->xid = sp->xid;
wrk->obj->response = sp->err_code;
WS_Assert(wrk->obj->ws_o);
/* Filter into object */
hp = wrk->busyobj->beresp;
hp2 = wrk->obj->http;
hp2->logtag = HTTP_Obj;
http_CopyResp(hp2, hp);
http_FilterFields(wrk, sp->vsl_id, hp2, hp,
pass ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(wrk, sp->vsl_id, hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
wrk->obj->last_modified = VTIM_parse(b);
else
wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
assert(WRW_IsReleased(wrk));
/*
* If we can deliver a 304 reply, we don't bother streaming.
* Notice that vcl_deliver{} could still nuke the headers
* that allow the 304, in which case we return 200 non-stream.
*/
if (wrk->obj->response == 200 &&
sp->http->conds &&
RFC2616_Do_Cond(sp))
wrk->busyobj->do_stream = 0;
AssertObjCorePassOrBusy(wrk->obj->objcore);
if (wrk->busyobj->do_stream) {
sp->step = STP_PREPRESP;
return (0);
}
/* Use unmodified headers*/
i = FetchBody(wrk, wrk->obj);
http_Setup(wrk->busyobj->bereq, NULL);
http_Setup(wrk->busyobj->beresp, NULL);
wrk->busyobj->vfp = NULL;
assert(WRW_IsReleased(wrk));
AZ(wrk->busyobj->vbc);
AN(sp->director);
if (i) {
HSH_Drop(wrk);
VBO_DerefBusyObj(wrk, &wrk->busyobj);
AZ(wrk->obj);
sp->err_code = 503;
sp->step = STP_ERROR;
return (0);
}
if (wrk->obj->objcore != NULL) {
EXP_Insert(wrk->obj);
AN(wrk->obj->objcore);
AN(wrk->obj->objcore->ban);
HSH_Unbusy(wrk);
}
VBO_DerefBusyObj(wrk, &wrk->busyobj);
wrk->acct_tmp.fetch++;
sp->step = STP_PREPRESP;
return (0);
}
void
RFC2616_Ttl(const struct sess *sp)
{
unsigned max_age, age;
double h_date, h_expires;
char *p;
const struct http *hp;
struct exp *expp;
expp = &sp->wrk->busyobj->exp;
hp = sp->wrk->busyobj->beresp;
assert(expp->entered != 0.0 && !isnan(expp->entered));
/* If all else fails, cache using default ttl */
expp->ttl = cache_param->default_ttl;
max_age = age = 0;
h_expires = 0;
h_date = 0;
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges yet, so 206 is out.
*/
if (http_GetHdr(hp, H_Age, &p)) {
age = strtoul(p, NULL, 0);
expp->age = age;
}
if (http_GetHdr(hp, H_Expires, &p))
h_expires = VTIM_parse(p);
if (http_GetHdr(hp, H_Date, &p))
h_date = VTIM_parse(p);
switch (sp->err_code) {
default:
expp->ttl = -1.;
break;
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 302: /* Moved Temporarily */
case 307: /* Temporary Redirect */
case 410: /* Gone */
case 404: /* Not Found */
/*
* First find any relative specification from the backend
* These take precedence according to RFC2616, 13.2.4
*/
if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
p != NULL) {
if (*p == '-')
max_age = 0;
else
max_age = strtoul(p, NULL, 0);
if (age > max_age)
expp->ttl = 0;
else
expp->ttl = max_age - age;
break;
}
/* No expire header, fall back to default */
if (h_expires == 0)
break;
/* If backend told us it is expired already, don't cache. */
if (h_expires < h_date) {
expp->ttl = 0;
break;
}
if (h_date == 0 ||
fabs(h_date - expp->entered) < cache_param->clock_skew) {
/*
* If we have no Date: header or if it is
* sufficiently close to our clock we will
* trust Expires: relative to our own clock.
*/
if (h_expires < expp->entered)
expp->ttl = 0;
else
expp->ttl = h_expires -
expp->entered;
break;
} else {
/*
* But even if the clocks are out of whack we can still
* derive a relative time from the two headers.
* (the negative ttl case is caught above)
*/
expp->ttl = (int)(h_expires - h_date);
}
}
/* calculated TTL, Our time, Date, Expires, max-age, age */
WSP(sp, SLT_TTL,
"%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
sp->xid, expp->ttl, -1., -1., expp->entered,
expp->age, h_date, h_expires, max_age);
}
void
RFC2616_Ttl(struct busyobj *bo, double now, double *t_origin,
float *ttl, float *grace, float *keep)
{
unsigned max_age, age;
double h_date, h_expires;
const char *p;
const struct http *hp;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(now != 0.0 && !isnan(now));
AN(t_origin);
AN(ttl);
AN(grace);
AN(keep);
*t_origin = now;
*ttl = cache_param->default_ttl;
*grace = cache_param->default_grace;
*keep = cache_param->default_keep;
hp = bo->beresp;
max_age = age = 0;
h_expires = 0;
h_date = 0;
/*
* Initial cacheability determination per [RFC2616, 13.4]
* We do not support ranges to the backend yet, so 206 is out.
*/
if (http_GetHdr(hp, H_Age, &p)) {
/*
* We deliberately run with partial results, rather than
* reject the Age: header outright. This will be future
* compatible with fractional seconds.
*/
age = strtoul(p, NULL, 10);
*t_origin -= age;
}
if (http_GetHdr(hp, H_Expires, &p))
h_expires = VTIM_parse(p);
if (http_GetHdr(hp, H_Date, &p))
h_date = VTIM_parse(p);
switch (http_GetStatus(hp)) {
default:
*ttl = -1.;
break;
case 302: /* Moved Temporarily */
case 307: /* Temporary Redirect */
/*
* https://tools.ietf.org/html/rfc7231#section-6.1
*
* Do not apply the default ttl, only set a ttl if Cache-Control
* or Expires are present. Uncacheable otherwise.
*/
*ttl = -1.;
/* FALL-THROUGH */
case 200: /* OK */
case 203: /* Non-Authoritative Information */
case 204: /* No Content */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
case 304: /* Not Modified - handled like 200 */
case 404: /* Not Found */
case 410: /* Gone */
case 414: /* Request-URI Too Large */
/*
* First find any relative specification from the backend
* These take precedence according to RFC2616, 13.2.4
*/
if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
p != NULL) {
if (*p == '-')
max_age = 0;
else
max_age = strtoul(p, NULL, 0);
*ttl = max_age;
break;
}
/* No expire header, fall back to default */
if (h_expires == 0)
break;
/* If backend told us it is expired already, don't cache. */
if (h_expires < h_date) {
*ttl = 0;
break;
}
if (h_date == 0 ||
fabs(h_date - now) < cache_param->clock_skew) {
/*
* If we have no Date: header or if it is
* sufficiently close to our clock we will
* trust Expires: relative to our own clock.
*/
if (h_expires < now)
*ttl = 0;
else
*ttl = h_expires - now;
break;
} else {
/*
* But even if the clocks are out of whack we can still
* derive a relative time from the two headers.
* (the negative ttl case is caught above)
*/
*ttl = (int)(h_expires - h_date);
}
}
/*
* RFC5861 outlines a way to control the use of stale responses.
* We use this to initialize the grace period.
*/
if (*ttl >= 0 && http_GetHdrField(hp, H_Cache_Control,
"stale-while-revalidate", &p) && p != NULL) {
if (*p == '-')
*grace = 0;
else
*grace = strtoul(p, NULL, 0);
}
VSLb(bo->vsl, SLT_TTL,
"RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
*ttl, *grace, -1., now,
*t_origin, h_date, h_expires, max_age);
}