static void *
pool_poolherder(void *priv)
{
unsigned nwq;
VTAILQ_HEAD(,pool) pools = VTAILQ_HEAD_INITIALIZER(pools);
struct pool *pp;
uint64_t u;
THR_SetName("pool_herder");
(void)priv;
nwq = 0;
while (1) {
if (nwq < cache_param->wthread_pools) {
pp = pool_mkpool(nwq);
if (pp != NULL) {
VTAILQ_INSERT_TAIL(&pools, pp, list);
VSC_C_main->pools++;
nwq++;
continue;
}
}
/* XXX: remove pools */
if (0)
SES_DeletePool(NULL);
(void)sleep(1);
u = 0;
VTAILQ_FOREACH(pp, &pools, list)
u += pp->lqueue;
VSC_C_main->thread_queue_len = u;
}
NEEDLESS_RETURN(NULL);
}
static void
vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
{
char *t = NULL;
ssize_t s;
int fd;
CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
VTAILQ_REMOVE(&vsmw->segs, seg, list);
REPLACE(seg->class, NULL);
REPLACE(seg->id, NULL);
FREE_OBJ(seg);
if (fixidx) {
vsmw_mkent(vsmw, vsmw->idx);
REPLACE(t, VSB_data(vsmw->vsb));
AN(t);
fd = openat(vsmw->vdirfd,
t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
assert(fd >= 0);
vsmw_idx_head(vsmw, fd);
VSB_clear(vsmw->vsb);
VTAILQ_FOREACH(seg, &vsmw->segs, list)
vsmw_fmt_index(vsmw, seg);
AZ(VSB_finish(vsmw->vsb));
s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
assert(s == VSB_len(vsmw->vsb));
AZ(close(fd));
AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
REPLACE(t, NULL);
}
}
int
exec_file(const char *fn, const char *script, const char *tmpdir,
char *logbuf, unsigned loglen)
{
unsigned old_err;
FILE *f;
struct extmacro *m;
signal(SIGPIPE, SIG_IGN);
vtc_loginit(logbuf, loglen);
vltop = vtc_logopen("top");
AN(vltop);
init_macro();
init_sema();
/* Apply extmacro definitions */
VTAILQ_FOREACH(m, &extmacro_list, list)
macro_def(vltop, NULL, m->name, "%s", m->val);
/*
* We need an IP number which will not repond, ever, and that is a
* lot harder than it sounds. This IP# is from RFC5737 and a
* C-class broadcast at that.
* If tests involving ${bad_ip} fails and you run linux, you should
* check your /proc/sys/net/ipv4/ip_nonlocal_bind setting.
*/
macro_def(vltop, NULL, "bad_ip", "192.0.2.255");
/* Move into our tmpdir */
AZ(chdir(tmpdir));
macro_def(vltop, NULL, "tmpdir", "%s", tmpdir);
/* Drop file to tell what was going on here */
f = fopen("INFO", "w");
AN(f);
fprintf(f, "Test case: %s\n", fn);
AZ(fclose(f));
vtc_stop = 0;
vtc_log(vltop, 1, "TEST %s starting", fn);
vtc_thread = pthread_self();
parse_string(script, cmds, NULL, vltop);
old_err = vtc_error;
vtc_stop = 1;
vtc_log(vltop, 1, "RESETTING after %s", fn);
reset_cmds(cmds);
vtc_error = old_err;
if (vtc_error)
vtc_log(vltop, 1, "TEST %s FAILED", fn);
else
vtc_log(vltop, 1, "TEST %s completed", fn);
return (vtc_error);
}
static void
ccf_debug_vmod(struct cli *cli, const char * const *av, void *priv)
{
struct vmod *v;
(void)av;
(void)priv;
ASSERT_CLI();
VTAILQ_FOREACH(v, &vmods, list)
cli_out(cli, "%5d %s (%s)\n", v->ref, v->nm, v->path);
}
static void
pan_busyobj(struct vsb *vsb, const struct busyobj *bo)
{
struct vfp_entry *vfe;
const char *p;
VSB_printf(vsb, "busyobj = %p {\n", bo);
VSB_indent(vsb, 2);
pan_ws(vsb, bo->ws);
VSB_printf(vsb, "refcnt = %u,\n", bo->refcount);
VSB_printf(vsb, "retries = %d, ", bo->retries);
VSB_printf(vsb, "failed = %d, ", bo->vfc->failed);
VSB_printf(vsb, "state = %d,\n", (int)bo->state);
VSB_printf(vsb, "flags = {");
p = "";
/*lint -save -esym(438,p) */
#define BO_FLAG(l, r, w, d) \
if(bo->l) { VSB_printf(vsb, "%s" #l, p); p = ", "; }
#include "tbl/bo_flags.h"
#undef BO_FLAG
/*lint -restore */
VSB_printf(vsb, "},\n");
if (VALID_OBJ(bo->htc, HTTP_CONN_MAGIC))
pan_htc(vsb, bo->htc);
if (!VTAILQ_EMPTY(&bo->vfc->vfp)) {
VSB_printf(vsb, "filters =");
VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list)
VSB_printf(vsb, " %s=%d",
vfe->vfp->name, (int)vfe->closed);
VSB_printf(vsb, "\n");
}
VDI_Panic(bo->director_req, vsb, "director_req");
if (bo->director_resp == bo->director_req)
VSB_printf(vsb, "director_resp = director_req,\n");
else
VDI_Panic(bo->director_resp, vsb, "director_resp");
if (bo->bereq != NULL && bo->bereq->ws != NULL)
pan_http(vsb, "bereq", bo->bereq);
if (bo->beresp != NULL && bo->beresp->ws != NULL)
pan_http(vsb, "beresp", bo->beresp);
if (bo->fetch_objcore)
pan_objcore(vsb, "fetch", bo->fetch_objcore);
if (bo->stale_oc)
pan_objcore(vsb, "ims", bo->stale_oc);
VCL_Panic(vsb, bo->vcl);
VSB_indent(vsb, -2);
VSB_printf(vsb, "},\n");
}
static void
pan_object(const char *typ, const struct object *o)
{
const struct storage *st;
VSB_printf(pan_vsp, " obj (%s) = %p {\n", typ, o);
VSB_printf(pan_vsp, " vxid = %u,\n", VXID(vbe32dec(o->oa_vxid)));
pan_http("obj", o->http, 4);
VSB_printf(pan_vsp, " len = %jd,\n", (intmax_t)o->body->len);
VSB_printf(pan_vsp, " store = {\n");
VTAILQ_FOREACH(st, &o->body->list, list)
pan_storage(st);
VSB_printf(pan_vsp, " },\n");
VSB_printf(pan_vsp, " },\n");
}
static void
pan_object(const struct object *o)
{
const struct storage *st;
VSB_printf(pan_vsp, " obj = %p {\n", o);
VSB_printf(pan_vsp, " vxid = %u,\n", o->vxid);
pan_ws(o->ws_o, 4);
pan_http("obj", o->http, 4);
VSB_printf(pan_vsp, " len = %jd,\n", (intmax_t)o->len);
VSB_printf(pan_vsp, " store = {\n");
VTAILQ_FOREACH(st, &o->store, list)
pan_storage(st);
VSB_printf(pan_vsp, " },\n");
VSB_printf(pan_vsp, " },\n");
}
static double
ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
{
struct ban *b, *bd;
struct banhead_s obans;
double d, dt, n;
dt = 49.62; // Random, non-magic
if (cache_param->ban_lurker_sleep == 0)
return (dt);
Lck_Lock(&ban_mtx);
b = ban_start;
Lck_Unlock(&ban_mtx);
d = VTIM_real() - cache_param->ban_lurker_age;
bd = NULL;
VTAILQ_INIT(&obans);
for (; b != NULL; b = VTAILQ_NEXT(b, list)) {
if (bd != NULL)
ban_lurker_test_ban(wrk, vsl, b, &obans, bd);
if (b->flags & BANS_FLAG_COMPLETED)
continue;
if (b->flags & BANS_FLAG_REQ) {
bd = VTAILQ_NEXT(b, list);
continue;
}
n = ban_time(b->spec) - d;
if (n < 0) {
VTAILQ_INSERT_TAIL(&obans, b, l_list);
if (bd == NULL)
bd = b;
} else if (n < dt) {
dt = n;
}
}
Lck_Lock(&ban_mtx);
VTAILQ_FOREACH(b, &obans, l_list)
ban_mark_completed(b);
Lck_Unlock(&ban_mtx);
return (dt);
}
static void
pan_busyobj(const struct busyobj *bo)
{
struct vfp_entry *vfe;
VSB_printf(pan_vsp, " busyobj = %p {\n", bo);
pan_ws(bo->ws, 4);
VSB_printf(pan_vsp, " refcnt = %u\n", bo->refcount);
VSB_printf(pan_vsp, " retries = %d\n", bo->retries);
VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed);
VSB_printf(pan_vsp, " state = %d\n", (int)bo->state);
#define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, " is_" #l "\n");
#include "tbl/bo_flags.h"
#undef BO_FLAG
VSB_printf(pan_vsp, " bodystatus = %d (%s),\n",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (!VTAILQ_EMPTY(&bo->vfc->vfp)) {
VSB_printf(pan_vsp, " filters =");
VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list)
VSB_printf(pan_vsp, " %s=%d",
vfe->vfp->name, (int)vfe->closed);
VSB_printf(pan_vsp, "\n");
}
VSB_printf(pan_vsp, " },\n");
if (VALID_OBJ(bo->vbc, BACKEND_MAGIC))
pan_vbc(bo->vbc);
if (bo->bereq->ws != NULL)
pan_http("bereq", bo->bereq, 4);
if (bo->beresp->ws != NULL)
pan_http("beresp", bo->beresp, 4);
pan_ws(bo->ws_o, 4);
if (bo->fetch_objcore)
pan_objcore("FETCH", bo->fetch_objcore);
if (bo->fetch_obj)
pan_object("FETCH", bo->fetch_obj);
if (bo->ims_obj)
pan_object("IMS", bo->ims_obj);
VSB_printf(pan_vsp, " }\n");
}
void
SPT_Wakeup(struct worker *w, struct septum *st)
{
#ifdef VARNISH_DEBUG
struct septum *entry;
#endif
ssize_t l;
char m = 'R';
Lck_Lock(&w->readylist_mtx);
#ifdef VARNISH_DEBUG
/* Sanity checking */
VTAILQ_FOREACH(entry, &w->readylist, list)
assert(entry != st);
#endif
VTAILQ_INSERT_TAIL(&w->readylist, st, list);
w->nreadylist++;
Lck_Unlock(&w->readylist_mtx);
l = write(w->readypipe[1], &m, sizeof(m));
if (l != sizeof(m))
VSL_stats->readypipe_writefail++;
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
struct http *hp, *hp2;
char *b;
uint16_t nhttp;
unsigned l;
struct vsb *vary = NULL;
int varyl = 0;
struct object *obj;
ssize_t est = -1;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(wrk->handling == VCL_RET_DELIVER);
/*
* The VCL variables beresp.do_g[un]zip tells us how we want the
* object processed before it is stored.
*
* The backend Content-Encoding header tells us what we are going
* to receive, which we classify in the following three classes:
*
* "Content-Encoding: gzip" --> object is gzip'ed.
* no Content-Encoding --> object is not gzip'ed.
* anything else --> do nothing wrt gzip
*
* XXX: BS_NONE/cl==0 should avoid gzip/gunzip
*/
/* We do nothing unless the param is set */
if (!cache_param->http_gzip_support)
bo->do_gzip = bo->do_gunzip = 0;
bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
/* It can't be both */
assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
/* We won't gunzip unless it is gzip'ed */
if (bo->do_gunzip && !bo->is_gzip)
bo->do_gunzip = 0;
/* If we do gunzip, remove the C-E header */
if (bo->do_gunzip)
http_Unset(bo->beresp, H_Content_Encoding);
/* We wont gzip unless it is ungziped */
if (bo->do_gzip && !bo->is_gunzip)
bo->do_gzip = 0;
/* If we do gzip, add the C-E header */
if (bo->do_gzip)
http_SetHeader(bo->beresp, "Content-Encoding: gzip");
/* But we can't do both at the same time */
assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
if (bo->vbc != NULL)
est = V1F_Setup_Fetch(bo);
if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
RFC2616_Weaken_Etag(bo->beresp);
VFP_Push(bo, vfp_gunzip_pull, 0);
}
if (bo->do_esi && bo->do_gzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_esi_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->do_esi) {
VFP_Push(bo, vfp_esi_pull, 0);
} else if (bo->do_gzip) {
VFP_Push(bo, vfp_gzip_pull, 0);
RFC2616_Weaken_Etag(bo->beresp);
} else if (bo->is_gzip && !bo->do_gunzip) {
VFP_Push(bo, vfp_testgunzip_pull, 0);
}
if (bo->fetch_objcore->flags & OC_F_PRIVATE)
AN(bo->uncacheable);
/* No reason to try streaming a non-existing body */
if (bo->htc.body_status == BS_NONE)
bo->do_stream = 0;
l = 0;
/* Create Vary instructions */
if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
varyl = VRY_Create(bo, &vary);
if (varyl > 0) {
AN(vary);
assert(varyl == VSB_len(vary));
l += varyl;
} else if (varyl < 0) {
/*
* Vary parse error
* Complain about it, and make this a pass.
*/
VSLb(bo->vsl, SLT_Error,
"Illegal 'Vary' header from backend, "
"making this a pass.");
bo->uncacheable = 1;
AZ(vary);
} else
/* No vary */
AZ(vary);
}
l += http_EstimateWS(bo->beresp,
bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
if (bo->uncacheable)
bo->fetch_objcore->flags |= OC_F_PASS;
if (bo->uncacheable ||
bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
bo->storage_hint = TRANSIENT_STORAGE;
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->fetch_objcore);
obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
if (obj == NULL) {
/*
* Try to salvage the transaction by allocating a
* shortlived object on Transient storage.
*/
if (bo->exp.ttl > cache_param->shortlived)
bo->exp.ttl = cache_param->shortlived;
bo->exp.grace = 0.0;
bo->exp.keep = 0.0;
obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
}
if (obj == NULL) {
bo->stats = NULL;
(void)VFP_Error(bo, "Could not get storage");
VDI_CloseFd(&bo->vbc);
return (F_STP_DONE);
}
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
bo->storage_hint = NULL;
AZ(bo->fetch_obj);
bo->fetch_obj = obj;
if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
obj->gziped = 1;
if (vary != NULL) {
obj->vary = (void *)WS_Copy(obj->http->ws,
VSB_data(vary), varyl);
AN(obj->vary);
(void)VRY_Validate(obj->vary);
VSB_delete(vary);
}
obj->vxid = bo->vsl->wid;
obj->response = bo->err_code;
WS_Assert(bo->ws_o);
/* Filter into object */
hp = bo->beresp;
hp2 = obj->http;
hp2->logtag = HTTP_Obj;
http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
http_CopyHome(hp2);
if (http_GetHdr(hp, H_Last_Modified, &b))
obj->last_modified = VTIM_parse(b);
else
obj->last_modified = floor(bo->exp.t_origin);
assert(WRW_IsReleased(wrk));
/*
* Ready to fetch the body
*/
assert(bo->refcount >= 1);
AZ(WS_Overflowed(bo->ws_o));
if (bo->do_stream)
HSH_Unbusy(&wrk->stats, obj->objcore);
assert(bo->state == BOS_REQ_DONE);
VBO_setstate(bo, BOS_FETCHING);
switch (bo->htc.body_status) {
case BS_NONE:
break;
case BS_ERROR:
/* XXX: Why not earlier ? */
bo->should_close |=
VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
if (bo->vbc == NULL)
(void)VFP_Error(bo, "Backend connection gone");
else
VFP_Fetch_Body(bo, est);
}
bo->stats = NULL;
bo->t_body = VTIM_mono();
if (bo->vbc != NULL) {
if (bo->should_close)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
bo->htc.body_status, body_status_2str(bo->htc.body_status));
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
if (!bo->do_stream && bo->state != BOS_FAILED)
HSH_Unbusy(&wrk->stats, obj->objcore);
if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
EXP_Insert(obj->objcore);
AN(obj->objcore->ban);
}
HSH_Complete(obj->objcore);
assert(bo->refcount >= 1);
if (bo->state != BOS_FAILED)
VBO_setstate(bo, BOS_FINISHED);
VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
return (F_STP_DONE);
}
static void
vbf_fetch_thread(struct worker *wrk, void *priv)
{
struct busyobj *bo;
enum fetch_step stp;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
THR_SetBusyobj(bo);
stp = F_STP_MKBEREQ;
assert(bo->doclose == SC_NULL);
assert(isnan(bo->t_first));
assert(isnan(bo->t_prev));
VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk));
bo->stats = &wrk->stats;
while (stp != F_STP_DONE) {
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
assert(bo->refcount >= 1);
switch(stp) {
#define FETCH_STEP(l, U, arg) \
case F_STP_##U: \
stp = vbf_stp_##l arg; \
break;
#include "tbl/steps.h"
#undef FETCH_STEP
default:
WRONG("Illegal fetch_step");
}
}
assert(WRW_IsReleased(wrk));
bo->stats = NULL;
if (bo->vbc != NULL) {
if (bo->doclose != SC_NULL)
VDI_CloseFd(&bo->vbc, &bo->acct);
else
VDI_RecycleFd(&bo->vbc, &bo->acct);
AZ(bo->vbc);
}
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
if (bo->state == BOS_FINISHED) {
AZ(bo->fetch_objcore->flags & OC_F_FAILED);
HSH_Complete(bo->fetch_objcore);
VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
struct storage *st;
uu = 0;
VTAILQ_FOREACH(st, &bo->fetch_obj->body->list, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= bo->fetch_obj->len);
else
assert(uu == bo->fetch_obj->len);
}
}
AZ(bo->fetch_objcore->busyobj);
if (bo->ims_obj != NULL)
(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
VBO_DerefBusyObj(wrk, &bo);
THR_SetBusyobj(NULL);
}
void
V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
{
int cls;
struct storage *st;
ssize_t cl;
struct http_conn *htc;
struct object *obj;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
htc = &bo->htc;
CHECK_OBJ_ORNULL(bo->vbc, VBC_MAGIC);
obj = bo->fetch_obj;
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
assert(bo->state == BOS_FETCHING);
/*
* XXX: The busyobj needs a dstat, but it is not obvious which one
* XXX: it should be (own/borrowed). For now borrow the wrk's.
*/
AZ(bo->stats);
bo->stats = &wrk->stats;
AN(bo->vfp);
AZ(bo->vgz_rx);
assert(VTAILQ_EMPTY(&obj->store));
/* XXX: pick up estimate from objdr ? */
cl = 0;
cls = bo->should_close;
switch (htc->body_status) {
case BS_NONE:
break;
case BS_ZERO:
break;
case BS_LENGTH:
cl = vbf_fetch_number(bo->h_content_length, 10);
bo->vfp->begin(bo, cl);
if (bo->state == BOS_FETCHING && cl > 0)
cls |= vbf_fetch_straight(bo, htc, cl);
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_CHUNKED:
bo->vfp->begin(bo, cl > 0 ? cl : 0);
if (bo->state == BOS_FETCHING)
cls |= vbf_fetch_chunked(bo, htc);
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_EOF:
bo->vfp->begin(bo, cl > 0 ? cl : 0);
if (bo->state == BOS_FETCHING)
vbf_fetch_eof(bo, htc);
cls = 1;
if (bo->vfp->end(bo))
assert(bo->state == BOS_FAILED);
break;
case BS_ERROR:
cls |= VFP_Error(bo, "error incompatible Transfer-Encoding");
break;
default:
INCOMPL();
}
bo->t_body = VTIM_mono();
AZ(bo->vgz_rx);
/*
* Trim or delete the last segment, if any
*/
st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
/* XXX: Temporary: Only trim if we are not streaming */
if (st != NULL && !bo->do_stream) {
/* XXX: is any of this safe under streaming ? */
if (st->len == 0) {
VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
STV_free(st);
} else if (st->len < st->space) {
STV_trim(st, st->len, 1);
}
}
bo->vfp = NULL;
VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d",
htc->body_status, body_status_2str(htc->body_status), cls);
http_Teardown(bo->bereq);
http_Teardown(bo->beresp);
if (bo->vbc != NULL) {
if (cls)
VDI_CloseFd(&bo->vbc);
else
VDI_RecycleFd(&bo->vbc);
}
AZ(bo->vbc);
if (bo->state == BOS_FAILED) {
wrk->stats.fetch_failed++;
} else {
assert(bo->state == BOS_FETCHING);
VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert(uu <= obj->len);
else
assert(uu == obj->len);
}
}
bo->stats = NULL;
}
int
exec_file(const char *fn, const char *script, const char *tmpdir,
char *logbuf, unsigned loglen)
{
unsigned old_err;
char *cwd, *p;
FILE *f;
struct extmacro *m;
signal(SIGPIPE, SIG_IGN);
vtc_loginit(logbuf, loglen);
vltop = vtc_logopen("top");
AN(vltop);
init_macro();
init_sema();
/* Apply extmacro definitions */
VTAILQ_FOREACH(m, &extmacro_list, list)
macro_def(vltop, NULL, m->name, m->val);
/* Other macro definitions */
cwd = getcwd(NULL, PATH_MAX);
macro_def(vltop, NULL, "pwd", cwd);
macro_def(vltop, NULL, "topbuild", "%s/%s", cwd, TOP_BUILDDIR);
macro_def(vltop, NULL, "bad_ip", "10.255.255.255");
/* Move into our tmpdir */
AZ(chdir(tmpdir));
macro_def(vltop, NULL, "tmpdir", tmpdir);
/* Drop file to tell what was going on here */
f = fopen("INFO", "w");
AN(f);
fprintf(f, "Test case: %s\n", fn);
AZ(fclose(f));
vtc_stop = 0;
vtc_desc = NULL;
vtc_log(vltop, 1, "TEST %s starting", fn);
p = strdup(script);
AN(p);
vtc_thread = pthread_self();
parse_string(p, cmds, NULL, vltop);
old_err = vtc_error;
vtc_stop = 1;
vtc_log(vltop, 1, "RESETTING after %s", fn);
reset_cmds(cmds);
vtc_error = old_err;
if (vtc_error)
vtc_log(vltop, 1, "TEST %s FAILED", fn);
else
vtc_log(vltop, 1, "TEST %s completed", fn);
free(vtc_desc);
return (vtc_error);
}
int
FetchBody(struct worker *wrk, struct object *obj)
{
int cls;
struct storage *st;
int mklen;
ssize_t cl;
struct http_conn *htc;
struct busyobj *bo;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
bo = wrk->busyobj;
CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
AZ(bo->fetch_obj);
CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
htc = &bo->htc;
if (bo->vfp == NULL)
bo->vfp = &vfp_nop;
AssertObjCorePassOrBusy(obj->objcore);
AZ(bo->vgz_rx);
AZ(VTAILQ_FIRST(&obj->store));
bo->fetch_obj = obj;
bo->fetch_failed = 0;
/* XXX: pick up estimate from objdr ? */
cl = 0;
switch (bo->body_status) {
case BS_NONE:
cls = 0;
mklen = 0;
break;
case BS_ZERO:
cls = 0;
mklen = 1;
break;
case BS_LENGTH:
cl = fetch_number(bo->h_content_length, 10);
bo->vfp->begin(wrk, cl > 0 ? cl : 0);
cls = fetch_straight(wrk, htc, cl);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_CHUNKED:
bo->vfp->begin(wrk, cl);
cls = fetch_chunked(wrk, htc);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_EOF:
bo->vfp->begin(wrk, cl);
cls = fetch_eof(wrk, htc);
mklen = 1;
if (bo->vfp->end(wrk))
cls = -1;
break;
case BS_ERROR:
cls = 1;
mklen = 0;
break;
default:
cls = 0;
mklen = 0;
INCOMPL();
}
AZ(bo->vgz_rx);
/*
* It is OK for ->end to just leave the last storage segment
* sitting on wrk->storage, we will always call vfp_nop_end()
* to get it trimmed or thrown out if empty.
*/
AZ(vfp_nop_end(wrk));
bo->fetch_obj = NULL;
WSLB(wrk, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
bo->body_status, body_status(bo->body_status),
cls, mklen);
if (bo->body_status == BS_ERROR) {
VDI_CloseFd(wrk, &bo->vbc);
return (__LINE__);
}
if (cls < 0) {
wrk->stats.fetch_failed++;
/* XXX: Wouldn't this store automatically be released ? */
while (!VTAILQ_EMPTY(&obj->store)) {
st = VTAILQ_FIRST(&obj->store);
VTAILQ_REMOVE(&obj->store, st, list);
STV_free(st);
}
VDI_CloseFd(wrk, &bo->vbc);
obj->len = 0;
return (__LINE__);
}
AZ(bo->fetch_failed);
if (cls == 0 && bo->should_close)
cls = 1;
WSLB(wrk, SLT_Length, "%zd", obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &obj->store, list)
uu += st->len;
if (bo->do_stream)
/* Streaming might have started freeing stuff */
assert (uu <= obj->len);
else
assert(uu == obj->len);
}
if (mklen > 0) {
http_Unset(obj->http, H_Content_Length);
http_PrintfHeader(wrk, bo->vbc->vsl_id, obj->http,
"Content-Length: %zd", obj->len);
}
if (cls)
VDI_CloseFd(wrk, &bo->vbc);
else
VDI_RecycleFd(wrk, &bo->vbc);
return (0);
}
int
FetchBody(struct sess *sp)
{
int cls;
struct storage *st;
struct worker *w;
int mklen;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
w = sp->wrk;
CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
if (w->vfp == NULL)
w->vfp = &vfp_nop;
AN(sp->director);
AssertObjCorePassOrBusy(sp->obj->objcore);
AZ(w->vgz_rx);
AZ(VTAILQ_FIRST(&sp->obj->store));
switch (w->body_status) {
case BS_NONE:
cls = 0;
mklen = 0;
break;
case BS_ZERO:
cls = 0;
mklen = 1;
break;
case BS_LENGTH:
cls = fetch_straight(sp, w->htc,
w->h_content_length);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_CHUNKED:
cls = fetch_chunked(sp, w->htc);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_EOF:
cls = fetch_eof(sp, w->htc);
mklen = 1;
XXXAZ(w->vfp->end(sp));
break;
case BS_ERROR:
cls = 1;
mklen = 0;
break;
default:
cls = 0;
mklen = 0;
INCOMPL();
}
AZ(w->vgz_rx);
/*
* It is OK for ->end to just leave the last storage segment
* sitting on w->storage, we will always call vfp_nop_end()
* to get it trimmed or thrown out if empty.
*/
AZ(vfp_nop_end(sp));
WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
w->body_status, body_status(w->body_status),
cls, mklen);
if (w->body_status == BS_ERROR) {
VDI_CloseFd(sp);
return (__LINE__);
}
if (cls < 0) {
w->stats.fetch_failed++;
/* XXX: Wouldn't this store automatically be released ? */
while (!VTAILQ_EMPTY(&sp->obj->store)) {
st = VTAILQ_FIRST(&sp->obj->store);
VTAILQ_REMOVE(&sp->obj->store, st, list);
STV_free(st);
}
VDI_CloseFd(sp);
sp->obj->len = 0;
return (__LINE__);
}
if (cls == 0 && w->do_close)
cls = 1;
WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);
{
/* Sanity check fetch methods accounting */
ssize_t uu;
uu = 0;
VTAILQ_FOREACH(st, &sp->obj->store, list)
uu += st->len;
if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
/* Streaming might have started freeing stuff */
assert (uu <= sp->obj->len);
else
assert(uu == sp->obj->len);
}
if (mklen > 0) {
http_Unset(sp->obj->http, H_Content_Length);
http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
"Content-Length: %jd", (intmax_t)sp->obj->len);
}
if (cls)
VDI_CloseFd(sp);
else
VDI_RecycleFd(sp);
return (0);
}