Ejemplo n.º 1
0
static void *
pool_poolherder(void *priv)
{
	unsigned nwq;
	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
	struct pool *pp;
	uint64_t u;

	THR_SetName("pool_herder");
	(void)priv;

	nwq = 0;
	while (1) {
		if (nwq < cache_param->wthread_pools) {
			pp = pool_mkpool(nwq);
			if (pp != NULL) {
				VTAILQ_INSERT_TAIL(&pools, pp, list);
				VSC_C_main->pools++;
				nwq++;
				continue;
			}
		}
		/* XXX: remove pools */
		if (0)
			SES_DeletePool(NULL);
		(void)sleep(1);
		u = 0;
		VTAILQ_FOREACH(pp, &pools, list)
			u += pp->lqueue;
		VSC_C_main->thread_queue_len = u;
	}
	NEEDLESS_RETURN(NULL);
}
Ejemplo n.º 2
0
static void
vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
{
	char *t = NULL;
	ssize_t s;
	int fd;

	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);

	VTAILQ_REMOVE(&vsmw->segs, seg, list);
	REPLACE(seg->class, NULL);
	REPLACE(seg->id, NULL);
	FREE_OBJ(seg);

	if (fixidx) {
		vsmw_mkent(vsmw, vsmw->idx);
		REPLACE(t, VSB_data(vsmw->vsb));
		AN(t);
		fd = openat(vsmw->vdirfd,
		    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
		assert(fd >= 0);
		vsmw_idx_head(vsmw, fd);
		VSB_clear(vsmw->vsb);
		VTAILQ_FOREACH(seg, &vsmw->segs, list)
			vsmw_fmt_index(vsmw, seg);
		AZ(VSB_finish(vsmw->vsb));
		s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
		assert(s == VSB_len(vsmw->vsb));
		AZ(close(fd));
		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
		REPLACE(t, NULL);
	}
}
Ejemplo n.º 3
0
int
exec_file(const char *fn, const char *script, const char *tmpdir,
    char *logbuf, unsigned loglen)
{
	unsigned old_err;
	FILE *f;
	struct extmacro *m;

	signal(SIGPIPE, SIG_IGN);

	vtc_loginit(logbuf, loglen);
	vltop = vtc_logopen("top");
	AN(vltop);

	init_macro();
	init_sema();

	/* Apply extmacro definitions */
	VTAILQ_FOREACH(m, &extmacro_list, list)
		macro_def(vltop, NULL, m->name, "%s", m->val);

	/*
	 * We need an IP number which will not repond, ever, and that is a
	 * lot harder than it sounds.  This IP# is from RFC5737 and a
	 * C-class broadcast at that.
	 * If tests involving ${bad_ip} fails and you run linux, you should
	 * check your /proc/sys/net/ipv4/ip_nonlocal_bind setting.
	 */
	macro_def(vltop, NULL, "bad_ip", "192.0.2.255");

	/* Move into our tmpdir */
	AZ(chdir(tmpdir));
	macro_def(vltop, NULL, "tmpdir", "%s", tmpdir);

	/* Drop file to tell what was going on here */
	f = fopen("INFO", "w");
	AN(f);
	fprintf(f, "Test case: %s\n", fn);
	AZ(fclose(f));

	vtc_stop = 0;
	vtc_log(vltop, 1, "TEST %s starting", fn);

	vtc_thread = pthread_self();
	parse_string(script, cmds, NULL, vltop);
	old_err = vtc_error;
	vtc_stop = 1;
	vtc_log(vltop, 1, "RESETTING after %s", fn);
	reset_cmds(cmds);
	vtc_error = old_err;

	if (vtc_error)
		vtc_log(vltop, 1, "TEST %s FAILED", fn);
	else
		vtc_log(vltop, 1, "TEST %s completed", fn);

	return (vtc_error);
}
Ejemplo n.º 4
0
static void
ccf_debug_vmod(struct cli *cli, const char * const *av, void *priv)
{
	struct vmod *v;

	(void)av;
	(void)priv;
	ASSERT_CLI();
	VTAILQ_FOREACH(v, &vmods, list)
		cli_out(cli, "%5d %s (%s)\n", v->ref, v->nm, v->path);
}
Ejemplo n.º 5
0
static void
pan_busyobj(struct vsb *vsb, const struct busyobj *bo)
{
	struct vfp_entry *vfe;
	const char *p;

	VSB_printf(vsb, "busyobj = %p {\n", bo);
	VSB_indent(vsb, 2);
	pan_ws(vsb, bo->ws);
	VSB_printf(vsb, "refcnt = %u,\n", bo->refcount);
	VSB_printf(vsb, "retries = %d, ", bo->retries);
	VSB_printf(vsb, "failed = %d, ", bo->vfc->failed);
	VSB_printf(vsb, "state = %d,\n", (int)bo->state);
	VSB_printf(vsb, "flags = {");
	p = "";
	/*lint -save -esym(438,p) */
#define BO_FLAG(l, r, w, d) \
	if(bo->l) { VSB_printf(vsb,  "%s" #l, p); p = ", "; }
#include "tbl/bo_flags.h"
#undef BO_FLAG
	/*lint -restore */
	VSB_printf(vsb, "},\n");

	if (VALID_OBJ(bo->htc, HTTP_CONN_MAGIC))
		pan_htc(vsb, bo->htc);

	if (!VTAILQ_EMPTY(&bo->vfc->vfp)) {
		VSB_printf(vsb, "filters =");
		VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list)
			VSB_printf(vsb, " %s=%d",
			    vfe->vfp->name, (int)vfe->closed);
		VSB_printf(vsb, "\n");
	}

	VDI_Panic(bo->director_req, vsb, "director_req");
	if (bo->director_resp == bo->director_req)
		VSB_printf(vsb, "director_resp = director_req,\n");
	else
		VDI_Panic(bo->director_resp, vsb, "director_resp");
	if (bo->bereq != NULL && bo->bereq->ws != NULL)
		pan_http(vsb, "bereq", bo->bereq);
	if (bo->beresp != NULL && bo->beresp->ws != NULL)
		pan_http(vsb, "beresp", bo->beresp);
	if (bo->fetch_objcore)
		pan_objcore(vsb, "fetch", bo->fetch_objcore);
	if (bo->stale_oc)
		pan_objcore(vsb, "ims", bo->stale_oc);
	VCL_Panic(vsb, bo->vcl);
	VSB_indent(vsb, -2);
	VSB_printf(vsb, "},\n");
}
Ejemplo n.º 6
0
static void
pan_object(const char *typ, const struct object *o)
{
	const struct storage *st;

	VSB_printf(pan_vsp, "  obj (%s) = %p {\n", typ, o);
	VSB_printf(pan_vsp, "    vxid = %u,\n", VXID(vbe32dec(o->oa_vxid)));
	pan_http("obj", o->http, 4);
	VSB_printf(pan_vsp, "    len = %jd,\n", (intmax_t)o->body->len);
	VSB_printf(pan_vsp, "    store = {\n");
	VTAILQ_FOREACH(st, &o->body->list, list)
		pan_storage(st);
	VSB_printf(pan_vsp, "    },\n");
	VSB_printf(pan_vsp, "  },\n");
}
Ejemplo n.º 7
0
static void
pan_object(const struct object *o)
{
	const struct storage *st;

	VSB_printf(pan_vsp, "  obj = %p {\n", o);
	VSB_printf(pan_vsp, "    vxid = %u,\n", o->vxid);
	pan_ws(o->ws_o, 4);
	pan_http("obj", o->http, 4);
	VSB_printf(pan_vsp, "    len = %jd,\n", (intmax_t)o->len);
	VSB_printf(pan_vsp, "    store = {\n");
	VTAILQ_FOREACH(st, &o->store, list)
		pan_storage(st);
	VSB_printf(pan_vsp, "    },\n");
	VSB_printf(pan_vsp, "  },\n");
}
Ejemplo n.º 8
0
static double
ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
{
	struct ban *b, *bd;
	struct banhead_s obans;
	double d, dt, n;

	dt = 49.62;		// Random, non-magic
	if (cache_param->ban_lurker_sleep == 0)
		return (dt);

	Lck_Lock(&ban_mtx);
	b = ban_start;
	Lck_Unlock(&ban_mtx);
	d = VTIM_real() - cache_param->ban_lurker_age;
	bd = NULL;
	VTAILQ_INIT(&obans);
	for (; b != NULL; b = VTAILQ_NEXT(b, list)) {
		if (bd != NULL)
			ban_lurker_test_ban(wrk, vsl, b, &obans, bd);
		if (b->flags & BANS_FLAG_COMPLETED)
			continue;
		if (b->flags & BANS_FLAG_REQ) {
			bd = VTAILQ_NEXT(b, list);
			continue;
		}
		n = ban_time(b->spec) - d;
		if (n < 0) {
			VTAILQ_INSERT_TAIL(&obans, b, l_list);
			if (bd == NULL)
				bd = b;
		} else if (n < dt) {
			dt = n;
		}
	}

	Lck_Lock(&ban_mtx);
	VTAILQ_FOREACH(b, &obans, l_list)
		ban_mark_completed(b);
	Lck_Unlock(&ban_mtx);
	return (dt);
}
Ejemplo n.º 9
0
static void
pan_busyobj(const struct busyobj *bo)
{
	struct vfp_entry *vfe;

	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
	pan_ws(bo->ws, 4);
	VSB_printf(pan_vsp, "  refcnt = %u\n", bo->refcount);
	VSB_printf(pan_vsp, "  retries = %d\n", bo->retries);
	VSB_printf(pan_vsp, "  failed = %d\n", bo->vfc->failed);
	VSB_printf(pan_vsp, "  state = %d\n", (int)bo->state);
#define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, "    is_" #l "\n");
#include "tbl/bo_flags.h"
#undef BO_FLAG

	VSB_printf(pan_vsp, "    bodystatus = %d (%s),\n",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
	if (!VTAILQ_EMPTY(&bo->vfc->vfp)) {
		VSB_printf(pan_vsp, "    filters =");
		VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list)
			VSB_printf(pan_vsp, " %s=%d",
			    vfe->vfp->name, (int)vfe->closed);
		VSB_printf(pan_vsp, "\n");
	}
	VSB_printf(pan_vsp, "    },\n");
	if (VALID_OBJ(bo->vbc, BACKEND_MAGIC))
		pan_vbc(bo->vbc);
	if (bo->bereq->ws != NULL)
		pan_http("bereq", bo->bereq, 4);
	if (bo->beresp->ws != NULL)
		pan_http("beresp", bo->beresp, 4);
	pan_ws(bo->ws_o, 4);
	if (bo->fetch_objcore)
		pan_objcore("FETCH", bo->fetch_objcore);
	if (bo->fetch_obj)
		pan_object("FETCH", bo->fetch_obj);
	if (bo->ims_obj)
		pan_object("IMS", bo->ims_obj);
	VSB_printf(pan_vsp, "  }\n");
}
Ejemplo n.º 10
0
void
SPT_Wakeup(struct worker *w, struct septum *st)
{
#ifdef VARNISH_DEBUG
	struct septum *entry;
#endif
	ssize_t l;
	char m = 'R';

	Lck_Lock(&w->readylist_mtx);
#ifdef VARNISH_DEBUG
	/* Sanity checking */
	VTAILQ_FOREACH(entry, &w->readylist, list)
		assert(entry != st);
#endif

	VTAILQ_INSERT_TAIL(&w->readylist, st, list);
	w->nreadylist++;
	Lck_Unlock(&w->readylist_mtx);

	l = write(w->readypipe[1], &m, sizeof(m));
	if (l != sizeof(m))
		VSL_stats->readypipe_writefail++;
}
Ejemplo n.º 11
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct http *hp, *hp2;
	char *b;
	uint16_t nhttp;
	unsigned l;
	struct vsb *vary = NULL;
	int varyl = 0;
	struct object *obj;
	ssize_t est = -1;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 * XXX: BS_NONE/cl==0 should avoid gzip/gunzip
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* If we do gunzip, remove the C-E header */
	if (bo->do_gunzip)
		http_Unset(bo->beresp, H_Content_Encoding);

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	/* If we do gzip, add the C-E header */
	if (bo->do_gzip)
		http_SetHeader(bo->beresp, "Content-Encoding: gzip");

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->vbc != NULL)
		est = V1F_Setup_Fetch(bo);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
		RFC2616_Weaken_Etag(bo->beresp);
		VFP_Push(bo, vfp_gunzip_pull, 0);
	}

	if (bo->do_esi && bo->do_gzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi) {
		VFP_Push(bo, vfp_esi_pull, 0);
	} else if (bo->do_gzip) {
		VFP_Push(bo, vfp_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_testgunzip_pull, 0);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	l = 0;

	/* Create Vary instructions */
	if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
		varyl = VRY_Create(bo, &vary);
		if (varyl > 0) {
			AN(vary);
			assert(varyl == VSB_len(vary));
			l += varyl;
		} else if (varyl < 0) {
			/*
			 * Vary parse error
			 * Complain about it, and make this a pass.
			 */
			VSLb(bo->vsl, SLT_Error,
			    "Illegal 'Vary' header from backend, "
			    "making this a pass.");
			bo->uncacheable = 1;
			AZ(vary);
		} else
			/* No vary */
			AZ(vary);
	}

	l += http_EstimateWS(bo->beresp,
	    bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);

	if (bo->uncacheable)
		bo->fetch_objcore->flags |= OC_F_PASS;

	if (bo->uncacheable ||
	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
		bo->storage_hint = TRANSIENT_STORAGE;

	AZ(bo->stats);
	bo->stats = &wrk->stats;
	AN(bo->fetch_objcore);
	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
	if (obj == NULL) {
		/*
		 * Try to salvage the transaction by allocating a
		 * shortlived object on Transient storage.
		 */
		if (bo->exp.ttl > cache_param->shortlived)
			bo->exp.ttl = cache_param->shortlived;
		bo->exp.grace = 0.0;
		bo->exp.keep = 0.0;
		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
	}
	if (obj == NULL) {
		bo->stats = NULL;
		(void)VFP_Error(bo, "Could not get storage");
		VDI_CloseFd(&bo->vbc);
		return (F_STP_DONE);
	}
	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);

	bo->storage_hint = NULL;

	AZ(bo->fetch_obj);
	bo->fetch_obj = obj;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (vary != NULL) {
		obj->vary = (void *)WS_Copy(obj->http->ws,
		    VSB_data(vary), varyl);
		AN(obj->vary);
		(void)VRY_Validate(obj->vary);
		VSB_delete(vary);
	}

	obj->vxid = bo->vsl->wid;
	obj->response = bo->err_code;
	WS_Assert(bo->ws_o);

	/* Filter into object */
	hp = bo->beresp;
	hp2 = obj->http;

	hp2->logtag = HTTP_Obj;
	http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
	http_CopyHome(hp2);

	if (http_GetHdr(hp, H_Last_Modified, &b))
		obj->last_modified = VTIM_parse(b);
	else
		obj->last_modified = floor(bo->exp.t_origin);

	assert(WRW_IsReleased(wrk));

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	AZ(WS_Overflowed(bo->ws_o));
	if (bo->do_stream)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	assert(bo->state == BOS_REQ_DONE);
	VBO_setstate(bo, BOS_FETCHING);

	switch (bo->htc.body_status) {
	case BS_NONE:
		break;
	case BS_ERROR:
		/* XXX: Why not earlier ? */
		bo->should_close |=
		    VFP_Error(bo, "error incompatible Transfer-Encoding");
		break;
	default:
		if (bo->vbc == NULL)
			(void)VFP_Error(bo, "Backend connection gone");
		else
			VFP_Fetch_Body(bo, est);
	}

	bo->stats = NULL;

	bo->t_body = VTIM_mono();

	if (bo->vbc != NULL) {
		if (bo->should_close)
			VDI_CloseFd(&bo->vbc);
		else
			VDI_RecycleFd(&bo->vbc);
		AZ(bo->vbc);
	}

	http_Teardown(bo->bereq);
	http_Teardown(bo->beresp);

	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status));

	if (bo->state == BOS_FAILED) {
		wrk->stats.fetch_failed++;
	} else {
		assert(bo->state == BOS_FETCHING);

		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);

		{
		/* Sanity check fetch methods accounting */
			ssize_t uu;
			struct storage *st;

			uu = 0;
			VTAILQ_FOREACH(st, &obj->store, list)
				uu += st->len;
			if (bo->do_stream)
				/* Streaming might have started freeing stuff */
				assert(uu <= obj->len);

			else
				assert(uu == obj->len);
		}
	}

	if (!bo->do_stream && bo->state != BOS_FAILED)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
		EXP_Insert(obj->objcore);
		AN(obj->objcore->ban);
	}

	HSH_Complete(obj->objcore);

	assert(bo->refcount >= 1);

	if (bo->state != BOS_FAILED)
		VBO_setstate(bo, BOS_FINISHED);

VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
	return (F_STP_DONE);
}
Ejemplo n.º 12
0
static void
vbf_fetch_thread(struct worker *wrk, void *priv)
{
	struct busyobj *bo;
	enum fetch_step stp;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);

	THR_SetBusyobj(bo);
	stp = F_STP_MKBEREQ;
	assert(bo->doclose == SC_NULL);
	assert(isnan(bo->t_first));
	assert(isnan(bo->t_prev));
	VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk));

	bo->stats = &wrk->stats;

	while (stp != F_STP_DONE) {
		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
		assert(bo->refcount >= 1);
		switch(stp) {
#define FETCH_STEP(l, U, arg)						\
		case F_STP_##U:						\
			stp = vbf_stp_##l arg;				\
			break;
#include "tbl/steps.h"
#undef FETCH_STEP
		default:
			WRONG("Illegal fetch_step");
		}
	}
	assert(WRW_IsReleased(wrk));

	bo->stats = NULL;

	if (bo->vbc != NULL) {
		if (bo->doclose != SC_NULL)
			VDI_CloseFd(&bo->vbc, &bo->acct);
		else
			VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	http_Teardown(bo->bereq);
	http_Teardown(bo->beresp);

	if (bo->state == BOS_FINISHED) {
		AZ(bo->fetch_objcore->flags & OC_F_FAILED);
		HSH_Complete(bo->fetch_objcore);
		VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
		{
		/* Sanity check fetch methods accounting */
			ssize_t uu;
			struct storage *st;

			uu = 0;
			VTAILQ_FOREACH(st, &bo->fetch_obj->body->list, list)
				uu += st->len;
			if (bo->do_stream)
				/* Streaming might have started freeing stuff */
				assert(uu <= bo->fetch_obj->len);

			else
				assert(uu == bo->fetch_obj->len);
		}
	}
	AZ(bo->fetch_objcore->busyobj);

	if (bo->ims_obj != NULL)
		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);

	VBO_DerefBusyObj(wrk, &bo);
	THR_SetBusyobj(NULL);
}
Ejemplo n.º 13
0
void
V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
{
	int cls;
	struct storage *st;
	ssize_t cl;
	struct http_conn *htc;
	struct object *obj;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	htc = &bo->htc;
	CHECK_OBJ_ORNULL(bo->vbc, VBC_MAGIC);
	obj = bo->fetch_obj;
	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);

	assert(bo->state == BOS_FETCHING);

	/*
	 * XXX: The busyobj needs a dstat, but it is not obvious which one
	 * XXX: it should be (own/borrowed).  For now borrow the wrk's.
	 */
	AZ(bo->stats);
	bo->stats = &wrk->stats;

	AN(bo->vfp);
	AZ(bo->vgz_rx);
	assert(VTAILQ_EMPTY(&obj->store));

	/* XXX: pick up estimate from objdr ? */
	cl = 0;
	cls = bo->should_close;
	switch (htc->body_status) {
	case BS_NONE:
		break;
	case BS_ZERO:
		break;
	case BS_LENGTH:
		cl = vbf_fetch_number(bo->h_content_length, 10);

		bo->vfp->begin(bo, cl);
		if (bo->state == BOS_FETCHING && cl > 0)
			cls |= vbf_fetch_straight(bo, htc, cl);
		if (bo->vfp->end(bo))
			assert(bo->state == BOS_FAILED);
		break;
	case BS_CHUNKED:
		bo->vfp->begin(bo, cl > 0 ? cl : 0);
		if (bo->state == BOS_FETCHING)
			cls |= vbf_fetch_chunked(bo, htc);
		if (bo->vfp->end(bo))
			assert(bo->state == BOS_FAILED);
		break;
	case BS_EOF:
		bo->vfp->begin(bo, cl > 0 ? cl : 0);
		if (bo->state == BOS_FETCHING)
			vbf_fetch_eof(bo, htc);
		cls = 1;
		if (bo->vfp->end(bo))
			assert(bo->state == BOS_FAILED);
		break;
	case BS_ERROR:
		cls |= VFP_Error(bo, "error incompatible Transfer-Encoding");
		break;
	default:
		INCOMPL();
	}
	bo->t_body = VTIM_mono();
	AZ(bo->vgz_rx);

	/*
	 * Trim or delete the last segment, if any
	 */

	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
	/* XXX: Temporary:  Only trim if we are not streaming */
	if (st != NULL && !bo->do_stream) {
		/* XXX: is any of this safe under streaming ? */
		if (st->len == 0) {
			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
			STV_free(st);
		} else if (st->len < st->space) {
			STV_trim(st, st->len, 1);
		}
	}

	bo->vfp = NULL;

	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d",
	    htc->body_status, body_status_2str(htc->body_status), cls);

	http_Teardown(bo->bereq);
	http_Teardown(bo->beresp);

	if (bo->vbc != NULL) {
		if (cls)
			VDI_CloseFd(&bo->vbc);
		else
			VDI_RecycleFd(&bo->vbc);
	}
	AZ(bo->vbc);

	if (bo->state == BOS_FAILED) {
		wrk->stats.fetch_failed++;
	} else {
		assert(bo->state == BOS_FETCHING);

		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);

		{
		/* Sanity check fetch methods accounting */
			ssize_t uu;

			uu = 0;
			VTAILQ_FOREACH(st, &obj->store, list)
				uu += st->len;
			if (bo->do_stream)
				/* Streaming might have started freeing stuff */
				assert(uu <= obj->len);

			else
				assert(uu == obj->len);
		}
	}
	bo->stats = NULL;
}
Ejemplo n.º 14
0
int
exec_file(const char *fn, const char *script, const char *tmpdir,
    char *logbuf, unsigned loglen)
{
	unsigned old_err;
	char *cwd, *p;
	FILE *f;
	struct extmacro *m;

	signal(SIGPIPE, SIG_IGN);

	vtc_loginit(logbuf, loglen);
	vltop = vtc_logopen("top");
	AN(vltop);

	init_macro();
	init_sema();

	/* Apply extmacro definitions */
	VTAILQ_FOREACH(m, &extmacro_list, list)
		macro_def(vltop, NULL, m->name, m->val);

	/* Other macro definitions */
	cwd = getcwd(NULL, PATH_MAX);
	macro_def(vltop, NULL, "pwd", cwd);
	macro_def(vltop, NULL, "topbuild", "%s/%s", cwd, TOP_BUILDDIR);
	macro_def(vltop, NULL, "bad_ip", "10.255.255.255");

	/* Move into our tmpdir */
	AZ(chdir(tmpdir));
	macro_def(vltop, NULL, "tmpdir", tmpdir);

	/* Drop file to tell what was going on here */
	f = fopen("INFO", "w");
	AN(f);
	fprintf(f, "Test case: %s\n", fn);
	AZ(fclose(f));

	vtc_stop = 0;
	vtc_desc = NULL;
	vtc_log(vltop, 1, "TEST %s starting", fn);

	p = strdup(script);
	AN(p);

	vtc_thread = pthread_self();
	parse_string(p, cmds, NULL, vltop);
	old_err = vtc_error;
	vtc_stop = 1;
	vtc_log(vltop, 1, "RESETTING after %s", fn);
	reset_cmds(cmds);
	vtc_error = old_err;

	if (vtc_error)
		vtc_log(vltop, 1, "TEST %s FAILED", fn);
	else
		vtc_log(vltop, 1, "TEST %s completed", fn);

	free(vtc_desc);
	return (vtc_error);
}
Ejemplo n.º 15
0
int
FetchBody(struct worker *wrk, struct object *obj)
{
	int cls;
	struct storage *st;
	int mklen;
	ssize_t cl;
	struct http_conn *htc;
	struct busyobj *bo;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	bo = wrk->busyobj;
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	AZ(bo->fetch_obj);
	CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);

	htc = &bo->htc;

	if (bo->vfp == NULL)
		bo->vfp = &vfp_nop;

	AssertObjCorePassOrBusy(obj->objcore);

	AZ(bo->vgz_rx);
	AZ(VTAILQ_FIRST(&obj->store));

	bo->fetch_obj = obj;
	bo->fetch_failed = 0;

	/* XXX: pick up estimate from objdr ? */
	cl = 0;
	switch (bo->body_status) {
	case BS_NONE:
		cls = 0;
		mklen = 0;
		break;
	case BS_ZERO:
		cls = 0;
		mklen = 1;
		break;
	case BS_LENGTH:
		cl = fetch_number(bo->h_content_length, 10);
		bo->vfp->begin(wrk, cl > 0 ? cl : 0);
		cls = fetch_straight(wrk, htc, cl);
		mklen = 1;
		if (bo->vfp->end(wrk))
			cls = -1;
		break;
	case BS_CHUNKED:
		bo->vfp->begin(wrk, cl);
		cls = fetch_chunked(wrk, htc);
		mklen = 1;
		if (bo->vfp->end(wrk))
			cls = -1;
		break;
	case BS_EOF:
		bo->vfp->begin(wrk, cl);
		cls = fetch_eof(wrk, htc);
		mklen = 1;
		if (bo->vfp->end(wrk))
			cls = -1;
		break;
	case BS_ERROR:
		cls = 1;
		mklen = 0;
		break;
	default:
		cls = 0;
		mklen = 0;
		INCOMPL();
	}
	AZ(bo->vgz_rx);

	/*
	 * It is OK for ->end to just leave the last storage segment
	 * sitting on wrk->storage, we will always call vfp_nop_end()
	 * to get it trimmed or thrown out if empty.
	 */
	AZ(vfp_nop_end(wrk));

	bo->fetch_obj = NULL;

	WSLB(wrk, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
	    bo->body_status, body_status(bo->body_status),
	    cls, mklen);

	if (bo->body_status == BS_ERROR) {
		VDI_CloseFd(wrk, &bo->vbc);
		return (__LINE__);
	}

	if (cls < 0) {
		wrk->stats.fetch_failed++;
		/* XXX: Wouldn't this store automatically be released ? */
		while (!VTAILQ_EMPTY(&obj->store)) {
			st = VTAILQ_FIRST(&obj->store);
			VTAILQ_REMOVE(&obj->store, st, list);
			STV_free(st);
		}
		VDI_CloseFd(wrk, &bo->vbc);
		obj->len = 0;
		return (__LINE__);
	}
	AZ(bo->fetch_failed);

	if (cls == 0 && bo->should_close)
		cls = 1;

	WSLB(wrk, SLT_Length, "%zd", obj->len);

	{
	/* Sanity check fetch methods accounting */
		ssize_t uu;

		uu = 0;
		VTAILQ_FOREACH(st, &obj->store, list)
			uu += st->len;
		if (bo->do_stream)
			/* Streaming might have started freeing stuff */
			assert (uu <= obj->len);

		else
			assert(uu == obj->len);
	}

	if (mklen > 0) {
		http_Unset(obj->http, H_Content_Length);
		http_PrintfHeader(wrk, bo->vbc->vsl_id, obj->http,
		    "Content-Length: %zd", obj->len);
	}

	if (cls)
		VDI_CloseFd(wrk, &bo->vbc);
	else
		VDI_RecycleFd(wrk, &bo->vbc);

	return (0);
}
Ejemplo n.º 16
0
int
FetchBody(struct sess *sp)
{
	int cls;
	struct storage *st;
	struct worker *w;
	int mklen;

	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
	w = sp->wrk;
	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);

	if (w->vfp == NULL)
		w->vfp = &vfp_nop;

	AN(sp->director);
	AssertObjCorePassOrBusy(sp->obj->objcore);

	AZ(w->vgz_rx);
	AZ(VTAILQ_FIRST(&sp->obj->store));
	switch (w->body_status) {
	case BS_NONE:
		cls = 0;
		mklen = 0;
		break;
	case BS_ZERO:
		cls = 0;
		mklen = 1;
		break;
	case BS_LENGTH:
		cls = fetch_straight(sp, w->htc,
		    w->h_content_length);
		mklen = 1;
		XXXAZ(w->vfp->end(sp));
		break;
	case BS_CHUNKED:
		cls = fetch_chunked(sp, w->htc);
		mklen = 1;
		XXXAZ(w->vfp->end(sp));
		break;
	case BS_EOF:
		cls = fetch_eof(sp, w->htc);
		mklen = 1;
		XXXAZ(w->vfp->end(sp));
		break;
	case BS_ERROR:
		cls = 1;
		mklen = 0;
		break;
	default:
		cls = 0;
		mklen = 0;
		INCOMPL();
	}
	AZ(w->vgz_rx);

	/*
	 * It is OK for ->end to just leave the last storage segment
	 * sitting on w->storage, we will always call vfp_nop_end()
	 * to get it trimmed or thrown out if empty.
	 */
	AZ(vfp_nop_end(sp));

	WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
	    w->body_status, body_status(w->body_status),
	    cls, mklen);

	if (w->body_status == BS_ERROR) {
		VDI_CloseFd(sp);
		return (__LINE__);
	}

	if (cls < 0) {
		w->stats.fetch_failed++;
		/* XXX: Wouldn't this store automatically be released ? */
		while (!VTAILQ_EMPTY(&sp->obj->store)) {
			st = VTAILQ_FIRST(&sp->obj->store);
			VTAILQ_REMOVE(&sp->obj->store, st, list);
			STV_free(st);
		}
		VDI_CloseFd(sp);
		sp->obj->len = 0;
		return (__LINE__);
	}

	if (cls == 0 && w->do_close)
		cls = 1;

	WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);

	{
	/* Sanity check fetch methods accounting */
		ssize_t uu;

		uu = 0;
		VTAILQ_FOREACH(st, &sp->obj->store, list)
			uu += st->len;
		if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
			/* Streaming might have started freeing stuff */
			assert (uu <= sp->obj->len);
		else
			assert(uu == sp->obj->len);
	}

	if (mklen > 0) {
		http_Unset(sp->obj->http, H_Content_Length);
		http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
		    "Content-Length: %jd", (intmax_t)sp->obj->len);
	}

	if (cls)
		VDI_CloseFd(sp);
	else
		VDI_RecycleFd(sp);

	return (0);
}