void
VRB_Free(struct req *req)
{

	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);

	if (req->body_oc != NULL) {
		ObjFreeObj(req->wrk,req->body_oc);
		FREE_OBJ(req->body_oc);
		req->body_oc = NULL;
	}
}
Exemple #2
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	struct vsb *synth_body;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
	AN(bo->fetch_objcore->flags & OC_F_BUSY);
	assert(bo->director_state == DIR_S_NULL);

	wrk->stats->fetch_failed++;

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	if (bo->fetch_objcore->stobj->stevedore != NULL)
		ObjFreeObj(bo->wrk, bo->fetch_objcore);

	if (bo->storage == NULL)
		bo->storage = STV_next();

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	http_TimeHeader(bo->beresp, "Date: ", now);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->t_origin = now;
	if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) {
		/*
		 * If there is a waitinglist, it means that there is no
		 * grace-able object, so cache the error return for a
		 * short time, so the waiting list can drain, rather than
		 * each objcore on the waiting list sequentially attempt
		 * to fetch from the backend.
		 */
		bo->fetch_objcore->ttl = 1;
		bo->fetch_objcore->grace = 5;
		bo->fetch_objcore->keep = 5;
	} else {
		bo->fetch_objcore->ttl = 0;
		bo->fetch_objcore->grace = 0;
		bo->fetch_objcore->keep = 0;
	}

	synth_body = VSB_new_auto();
	AN(synth_body);

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);

	AZ(VSB_finish(synth_body));

	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
		VSB_destroy(&synth_body);
		return (F_STP_FAIL);
	}

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_destroy(&synth_body);
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing");
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	assert(bo->vfc->wrk == bo->wrk);
	assert(bo->vfc->oc == bo->fetch_objcore);
	assert(bo->vfc->resp == bo->beresp);
	assert(bo->vfc->req == bo->bereq);

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		VSB_destroy(&synth_body);
		return (F_STP_FAIL);
	}

	ll = VSB_len(synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		if (l > ll)
			l = ll;
		memcpy(ptr, VSB_data(synth_body) + o, l);
		VFP_Extend(bo->vfc, l);
		ll -= l;
		o += l;
	}
	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o));
	VSB_destroy(&synth_body);
	HSH_Unbusy(wrk, bo->fetch_objcore);
	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
	return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	struct vsb *synth_body;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
	assert(bo->director_state == DIR_S_NULL);

	if(bo->fetch_objcore->stobj->stevedore != NULL)
		ObjFreeObj(bo->wrk, bo->fetch_objcore);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	synth_body = VSB_new_auto();
	AN(synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	http_TimeHeader(bo->beresp, "Date: ", now);
	http_SetHeader(bo->beresp, "Server: Varnish");

	EXP_Clr(&bo->fetch_objcore->exp);
	bo->fetch_objcore->exp.t_origin = bo->t_prev;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);

	AZ(VSB_finish(synth_body));

	if (wrk->handling == VCL_RET_RETRY ||
	    wrk->handling == VCL_RET_ABANDON) {
		VSB_delete(synth_body);

		if (bo->director_state != DIR_S_NULL) {
			bo->htc->doclose = SC_RESP_CLOSE;
			VDI_Finish(bo->wrk, bo);
		}

		if (wrk->handling == VCL_RET_RETRY) {
			if (bo->retries++ < cache_param->max_retries)
				return (F_STP_RETRY);
			VSLb(bo->vsl, SLT_VCL_Error,
			    "Too many retries, delivering 503");
		}

		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	bo->vfc->bo = bo;
	bo->vfc->wrk = bo->wrk;
	bo->vfc->oc = bo->fetch_objcore;
	bo->vfc->http = bo->beresp;
	bo->vfc->esi_req = bo->bereq;

	if (vbf_beresp2obj(bo)) {
		VSB_delete(synth_body);
		return (F_STP_FAIL);
	}

	ll = VSB_len(synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		memcpy(ptr, VSB_data(synth_body) + o, l);
		VBO_extend(bo, l);
		ll -= l;
		o += l;
	}
	VSB_delete(synth_body);

	HSH_Unbusy(wrk, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Exemple #4
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct object *obj;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	AN(bo->vbc);

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
		(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);

	if (bo->do_esi && bo->do_gzip) {
		(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
		(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
	} else if (bo->do_esi) {
		(void)VFP_Push(bo->vfc, &vfp_esi, 1);
	} else if (bo->do_gzip) {
		(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
	} else if (bo->is_gzip && !bo->do_gunzip) {
		(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	if (VFP_Open(bo->vfc)) {
		(void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open");
		bo->doclose = SC_RX_BODY;
		return (F_STP_ERROR);
	}

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		bo->doclose = SC_RX_BODY;
		return (F_STP_ERROR);
	}

	assert(WRW_IsReleased(wrk));

	obj = bo->fetch_obj;
	bo->vfc->body = obj->body;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (bo->do_gzip || bo->do_gunzip)
		obj->changed_gzip = 1;

	if (bo->htc.body_status != BS_NONE)
		V1F_Setup_Fetch(bo);

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	AZ(WS_Overflowed(bo->ws_o));

	assert (bo->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		HSH_Unbusy(&wrk->stats, obj->objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc.body_status != BS_NONE) {
		assert(bo->htc.body_status != BS_ERROR);
		VFP_Fetch_Body(bo);
		bo->acct.beresp_bodybytes = bo->vfc->bodybytes;
	}

	if (bo->vfc->failed && !bo->do_stream) {
		assert(bo->state < BOS_STREAM);
		if (bo->fetch_obj != NULL) {
			ObjFreeObj(bo->fetch_objcore, bo->stats);
			bo->fetch_obj = NULL;
		}
		return (F_STP_ERROR);
	}

	if (bo->vfc->failed)
		return (F_STP_FAIL);

	if (bo->do_stream)
		assert(bo->state == BOS_STREAM);
	else {
		assert(bo->state == BOS_REQ_DONE);
		HSH_Unbusy(&wrk->stats, obj->objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	if (bo->vbc != NULL && bo->doclose == SC_NULL) {
		VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	if (bo->ims_obj != NULL)
		EXP_Rearm(bo->ims_obj->objcore,
		    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
	return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	const char *p;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	if (bo->htc->content_length == 0)
		http_Unset(bo->beresp, H_Content_Encoding);

	if (bo->htc->body_status != BS_NONE) {
		bo->is_gzip =
		    http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
		bo->is_gunzip =
		    !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
		assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
	}

	/* We won't gunzip unless it is non-empty and gzip'ed */
	if (bo->htc->body_status == BS_NONE ||
	    bo->htc->content_length == 0 ||
	    (bo->do_gunzip && !bo->is_gzip))
		bo->do_gunzip = 0;

	/* We wont gzip unless it is non-empty and ungziped */
	if (bo->htc->body_status == BS_NONE ||
	    bo->htc->content_length == 0 ||
	    (bo->do_gzip && !bo->is_gunzip))
		bo->do_gzip = 0;

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
		(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);

	if (bo->htc->content_length != 0) {
		if (bo->do_esi && bo->do_gzip) {
			(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
		} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
			(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
		} else if (bo->do_esi) {
			(void)VFP_Push(bo->vfc, &vfp_esi, 1);
		} else if (bo->do_gzip) {
			(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
		} else if (bo->is_gzip && !bo->do_gunzip) {
			(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
		}
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc->body_status == BS_NONE)
		bo->do_stream = 0;

	if (VFP_Open(bo->vfc)) {
		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
		bo->htc->doclose = SC_RX_BODY;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		bo->htc->doclose = SC_RX_BODY;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (bo->do_esi)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);

	if (bo->do_gzip || bo->do_gunzip)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);

	if (http_IsStatus(bo->beresp, 200) && (
	    http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
	    http_GetHdr(bo->beresp, H_ETag, &p)))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);

	if (bo->htc->body_status != BS_NONE)
		AZ(VDI_GetBody(bo->wrk, bo));

	assert(bo->refcount >= 1);

	assert(bo->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		HSH_Unbusy(wrk, bo->fetch_objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc->body_status, body_status_2str(bo->htc->body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc->body_status != BS_NONE) {
		assert(bo->htc->body_status != BS_ERROR);
		vbf_fetch_body_helper(bo);
	}

	if (bo->vfc->failed && !bo->do_stream) {
		assert(bo->state < BOS_STREAM);
		ObjFreeObj(bo->wrk, bo->fetch_objcore);
		// XXX: doclose = ?
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		return (F_STP_FAIL);
	}

	if (bo->do_stream)
		assert(bo->state == BOS_STREAM);
	else {
		assert(bo->state == BOS_REQ_DONE);
		HSH_Unbusy(wrk, bo->fetch_objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	if (bo->stale_oc != NULL)
		EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);
	return (F_STP_DONE);
}