static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	AZ(vbf_beresp2obj(bo));

	if (ObjHasAttr(bo->wrk, bo->stale_oc, OA_ESIDATA))
		AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc,
		    OA_ESIDATA));

	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));

	if (bo->do_stream) {
		ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
		HSH_Unbusy(wrk, bo->fetch_objcore);
		ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
	}

	if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0))
		(void)VFP_Error(bo->vfc, "Template object failed");

	if (bo->stale_oc->flags & OC_F_FAILED)
		(void)VFP_Error(bo->vfc, "Template object failed");
	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		return (F_STP_FAIL);
	}

	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
	    bo->fetch_objcore->boc->len_so_far));

	if (!bo->do_stream)
		HSH_Unbusy(wrk, bo->fetch_objcore);

	HSH_Kill(bo->stale_oc);

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Beispiel #2
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	AZ(vbf_beresp2obj(bo));

	if (ObjHasAttr(bo->wrk, bo->stale_oc, OA_ESIDATA))
		AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc,
		    OA_ESIDATA));

	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));

	if (bo->do_stream) {
		ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
		HSH_Unbusy(wrk, bo->fetch_objcore);
		ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
	}

	if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0))
		(void)VFP_Error(bo->vfc, "Template object failed");

	if (bo->stale_oc->flags & OC_F_FAILED)
		(void)VFP_Error(bo->vfc, "Template object failed");
	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		wrk->stats->fetch_failed++;
		return (F_STP_FAIL);
	}
	return (F_STP_FETCHEND);
}
Beispiel #3
0
static enum fetch_step
vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
{

	AZ(bo->vfc->failed);
	VFP_Close(bo->vfc);

	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
	    bo->fetch_objcore->boc->len_so_far));

	if (bo->do_stream)
		assert(bo->fetch_objcore->boc->state == BOS_STREAM);
	else {
		assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);
		HSH_Unbusy(wrk, bo->fetch_objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	if (bo->stale_oc != NULL)
		HSH_Kill(bo->stale_oc);
	return (F_STP_DONE);
}
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct http *hp, *hp2;
	char *b;
	uint16_t nhttp;
	unsigned l;
	struct vsb *vary = NULL;
	int varyl = 0;
	struct object *obj;


	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 * XXX: BS_NONE/cl==0 should avoid gzip/gunzip
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* If we do gunzip, remove the C-E header */
	if (bo->do_gunzip)
		http_Unset(bo->beresp, H_Content_Encoding);

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	/* If we do gzip, add the C-E header */
	if (bo->do_gzip)
		http_SetHeader(bo->beresp, "Content-Encoding: gzip");

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	/* ESI takes precedence and handles gzip/gunzip itself */
	if (bo->do_esi)
		bo->vfp = &vfp_esi;
	else if (bo->do_gunzip)
		bo->vfp = &vfp_gunzip;
	else if (bo->do_gzip)
		bo->vfp = &vfp_gzip;
	else if (bo->is_gzip)
		bo->vfp = &vfp_testgzip;

	if (bo->fetch_objcore->objhead == NULL)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	l = http_EstimateWS(bo->beresp,
	    bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);

	/* Create Vary instructions */
	if (bo->fetch_objcore->objhead != NULL) {
		varyl = VRY_Create(bo, &vary);
		if (varyl > 0) {
			AN(vary);
			assert(varyl == VSB_len(vary));
			l += varyl;
		} else if (varyl < 0) {
			/*
			 * Vary parse error
			 * Complain about it, and make this a pass.
			 */
			VSLb(bo->vsl, SLT_Error,
			    "Illegal 'Vary' header from backend, "
			    "making this a pass.");
			bo->uncacheable = 1;
			AZ(vary);
		} else
			/* No vary */
			AZ(vary);
	}

	if (bo->uncacheable)
		bo->fetch_objcore->flags |= OC_F_PASS;

	if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
		bo->storage_hint = TRANSIENT_STORAGE;

	/*
	 * Space for producing a Content-Length: header including padding
	 * A billion gigabytes is enough for anybody.
	 */
	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);

	AZ(bo->stats);
	bo->stats = &wrk->stats;
	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
	if (obj == NULL) {
		/*
		 * Try to salvage the transaction by allocating a
		 * shortlived object on Transient storage.
		 */
		if (bo->exp.ttl > cache_param->shortlived)
			bo->exp.ttl = cache_param->shortlived;
		bo->exp.grace = 0.0;
		bo->exp.keep = 0.0;
		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
	}
	bo->stats = NULL;
	if (obj == NULL) {
		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
		bo->fetch_objcore = NULL;
		VDI_CloseFd(&bo->vbc);
		VBO_setstate(bo, BOS_FAILED);
		return (F_STP_ABANDON);
	}
	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);

	bo->storage_hint = NULL;

	AZ(bo->fetch_obj);
	bo->fetch_obj = obj;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (vary != NULL) {
		obj->vary = (void *)WS_Copy(obj->http->ws,
		    VSB_data(vary), varyl);
		AN(obj->vary);
		VRY_Validate(obj->vary);
		VSB_delete(vary);
	}

	obj->vxid = bo->vsl->wid;
	obj->response = bo->err_code;
	WS_Assert(obj->ws_o);

	/* Filter into object */
	hp = bo->beresp;
	hp2 = obj->http;

	hp2->logtag = HTTP_Obj;
	http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
	http_CopyHome(hp2);

	if (http_GetHdr(hp, H_Last_Modified, &b))
		obj->last_modified = VTIM_parse(b);
	else
		obj->last_modified = floor(bo->exp.entered);

	assert(WRW_IsReleased(wrk));

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	if (obj->objcore->objhead != NULL) {
		EXP_Insert(obj);
		AN(obj->objcore->ban);
		AZ(obj->ws_o->overflow);
		HSH_Unbusy(&wrk->stats, obj->objcore);
	}

	if (bo->vfp == NULL)
		bo->vfp = &VFP_nop;

	assert(bo->state == BOS_INVALID);
	VBO_setstate(bo, BOS_FETCHING);

	V1F_fetch_body(wrk, bo);

	assert(bo->refcount >= 1);

	if (obj->objcore->objhead != NULL)
		HSH_Ref(obj->objcore);

	if (bo->state == BOS_FAILED) {
		/* handle early failures */
		(void)HSH_Deref(&wrk->stats, NULL, &obj);
		return (F_STP_ABANDON);
	}
	VBO_setstate(bo, BOS_FINISHED);

	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
	return (F_STP_DONE);
}
Beispiel #5
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	struct vsb *synth_body;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
	assert(bo->director_state == DIR_S_NULL);

	if(bo->fetch_objcore->stobj->stevedore != NULL)
		ObjFreeObj(bo->wrk, bo->fetch_objcore);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	synth_body = VSB_new_auto();
	AN(synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	http_TimeHeader(bo->beresp, "Date: ", now);
	http_SetHeader(bo->beresp, "Server: Varnish");

	EXP_Clr(&bo->fetch_objcore->exp);
	bo->fetch_objcore->exp.t_origin = bo->t_prev;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);

	AZ(VSB_finish(synth_body));

	if (wrk->handling == VCL_RET_RETRY ||
	    wrk->handling == VCL_RET_ABANDON) {
		VSB_delete(synth_body);

		if (bo->director_state != DIR_S_NULL) {
			bo->htc->doclose = SC_RESP_CLOSE;
			VDI_Finish(bo->wrk, bo);
		}

		if (wrk->handling == VCL_RET_RETRY) {
			if (bo->retries++ < cache_param->max_retries)
				return (F_STP_RETRY);
			VSLb(bo->vsl, SLT_VCL_Error,
			    "Too many retries, delivering 503");
		}

		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	bo->vfc->bo = bo;
	bo->vfc->wrk = bo->wrk;
	bo->vfc->oc = bo->fetch_objcore;
	bo->vfc->http = bo->beresp;
	bo->vfc->esi_req = bo->bereq;

	if (vbf_beresp2obj(bo)) {
		VSB_delete(synth_body);
		return (F_STP_FAIL);
	}

	ll = VSB_len(synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		memcpy(ptr, VSB_data(synth_body) + o, l);
		VBO_extend(bo, l);
		ll -= l;
		o += l;
	}
	VSB_delete(synth_body);

	HSH_Unbusy(wrk, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Beispiel #6
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	void *oi;
	void *sp;
	ssize_t sl, al, l;
	uint8_t *ptr;
	enum objiter_status ois;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	AZ(vbf_beresp2obj(bo));

	if (ObjGetattr(bo->wrk, bo->stale_oc, OA_ESIDATA, NULL) != NULL)
		AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc,
		    OA_ESIDATA));

	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));

	if (bo->do_stream) {
		HSH_Unbusy(wrk, bo->fetch_objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	al = 0;
	oi = ObjIterBegin(wrk, bo->stale_oc);
	do {
		ois = ObjIter(bo->stale_oc, oi, &sp, &sl);
		if (ois == OIS_ERROR)
			(void)VFP_Error(bo->vfc, "Template object failed");
		while (sl > 0) {
			l = ObjGetLen(bo->wrk, bo->stale_oc) - al;
			assert(l > 0);
			if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
				break;
			if (sl < l)
				l = sl;
			memcpy(ptr, sp, l);
			VBO_extend(bo, l);
			al += l;
			sp = (char *)sp + l;
			sl -= l;
		}
	} while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM));
	ObjIterEnd(bo->stale_oc, &oi);
	if (bo->stale_oc->flags & OC_F_FAILED)
		(void)VFP_Error(bo->vfc, "Template object failed");
	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		return (F_STP_FAIL);
	}

	if (!bo->do_stream)
		HSH_Unbusy(wrk, bo->fetch_objcore);

	assert(ObjGetLen(bo->wrk, bo->fetch_objcore) == al);
	EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Beispiel #7
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	const char *p;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	if (bo->htc->content_length == 0)
		http_Unset(bo->beresp, H_Content_Encoding);

	if (bo->htc->body_status != BS_NONE) {
		bo->is_gzip =
		    http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
		bo->is_gunzip =
		    !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
		assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
	}

	/* We won't gunzip unless it is non-empty and gzip'ed */
	if (bo->htc->body_status == BS_NONE ||
	    bo->htc->content_length == 0 ||
	    (bo->do_gunzip && !bo->is_gzip))
		bo->do_gunzip = 0;

	/* We wont gzip unless it is non-empty and ungziped */
	if (bo->htc->body_status == BS_NONE ||
	    bo->htc->content_length == 0 ||
	    (bo->do_gzip && !bo->is_gunzip))
		bo->do_gzip = 0;

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
		(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);

	if (bo->htc->content_length != 0) {
		if (bo->do_esi && bo->do_gzip) {
			(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
		} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
			(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
		} else if (bo->do_esi) {
			(void)VFP_Push(bo->vfc, &vfp_esi, 1);
		} else if (bo->do_gzip) {
			(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
		} else if (bo->is_gzip && !bo->do_gunzip) {
			(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
		}
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc->body_status == BS_NONE)
		bo->do_stream = 0;

	if (VFP_Open(bo->vfc)) {
		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
		bo->htc->doclose = SC_RX_BODY;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		bo->htc->doclose = SC_RX_BODY;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (bo->do_esi)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);

	if (bo->do_gzip || bo->do_gunzip)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);

	if (http_IsStatus(bo->beresp, 200) && (
	    http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
	    http_GetHdr(bo->beresp, H_ETag, &p)))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);

	if (bo->htc->body_status != BS_NONE)
		AZ(VDI_GetBody(bo->wrk, bo));

	assert(bo->refcount >= 1);

	assert(bo->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		HSH_Unbusy(wrk, bo->fetch_objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc->body_status, body_status_2str(bo->htc->body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc->body_status != BS_NONE) {
		assert(bo->htc->body_status != BS_ERROR);
		vbf_fetch_body_helper(bo);
	}

	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		if (!bo->do_stream) {
			assert(bo->state < BOS_STREAM);
			// XXX: doclose = ?
			return (F_STP_ERROR);
		} else {
			return (F_STP_FAIL);
		}
	}

	if (bo->do_stream)
		assert(bo->state == BOS_STREAM);
	else {
		assert(bo->state == BOS_REQ_DONE);
		HSH_Unbusy(wrk, bo->fetch_objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	if (bo->stale_oc != NULL)
		EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);
	return (F_STP_DONE);
}
Beispiel #8
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	unsigned l;
	uint16_t nhttp;
	struct object *obj;
	struct objiter *oi;
	void *sp;
	ssize_t sl, al, tl, vl;
	struct storage *st;
	enum objiter_status ois;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	l = 0;
	if (bo->ims_obj->vary != NULL) {
		vl = VRY_Validate(bo->ims_obj->vary);
		l += vl;
	} else
		vl = 0;
	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);

	bo->stats = &wrk->stats;
	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
	if (obj == NULL) {
		(void)VFP_Error(bo, "Could not get storage");
		VDI_CloseFd(&bo->vbc);
		return (F_STP_DONE);
	}

	AZ(bo->fetch_obj);
	bo->fetch_obj = obj;

	obj->gziped = bo->ims_obj->gziped;
	obj->gzip_start = bo->ims_obj->gzip_start;
	obj->gzip_last = bo->ims_obj->gzip_last;
	obj->gzip_stop = bo->ims_obj->gzip_stop;

	/* XXX: ESI */

	if (bo->ims_obj->vary != NULL) {
		obj->vary = (void *)WS_Copy(obj->http->ws,
		    bo->ims_obj->vary, vl);
		assert(vl == VRY_Validate(obj->vary));
	}

	obj->vxid = bo->vsl->wid;

	obj->http->logtag = HTTP_Obj;
	/* XXX: we should have our own HTTP_A_CONDFETCH */
	http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
	http_CopyHome(obj->http);


	AZ(WS_Overflowed(bo->ws_o));
	VBO_setstate(bo, BOS_FETCHING);
	HSH_Unbusy(&wrk->stats, obj->objcore);

	if (!(obj->objcore->flags & OC_F_PRIVATE)) {
		EXP_Insert(obj->objcore);
		AN(obj->objcore->ban);
	}

	st = NULL;
	al = 0;

	oi = ObjIterBegin(wrk, bo->ims_obj);
	do {
		ois = ObjIter(oi, &sp, &sl);
		while (sl > 0) {
			if (st == NULL) {
				st = VFP_GetStorage(bo, bo->ims_obj->len - al);
				XXXAN(st);
			}
			tl = sl;
			if (tl > st->space - st->len)
				tl = st->space - st->len;
			memcpy(st->ptr + st->len, sp, tl);
			al += tl;
			sp = (char *)sp + tl;
			sl -= tl;
			VBO_extend(bo, al);
			if (st->len == st->space)
				st = NULL;
		}
	} while (ois == OIS_DATA || ois == OIS_STREAM);
	ObjIterEnd(&oi);
	bo->stats = NULL;
	assert(al == bo->ims_obj->len);
	assert(obj->len == al);
	if (bo->state != BOS_FAILED)
		VBO_setstate(bo, BOS_FINISHED);
	HSH_Complete(obj->objcore);
	return (F_STP_DONE);
}
Beispiel #9
0
static int
cnt_fetchbody(struct sess *sp)
{
    int i;
    struct http *hp, *hp2;
    char *b;
    uint16_t nhttp;
    unsigned l;
    struct vsb *vary = NULL;
    int varyl = 0, pass;
    struct worker *wrk;

    CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
    wrk = sp->wrk;
    CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
    CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);

    assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
           sp->handling == VCL_RET_DELIVER);

    if (wrk->objcore == NULL) {
        /* This is a pass from vcl_recv */
        pass = 1;
        /* VCL may have fiddled this, but that doesn't help */
        wrk->busyobj->exp.ttl = -1.;
    } else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
        /* pass from vcl_fetch{} -> hit-for-pass */
        /* XXX: the bereq was not filtered pass... */
        pass = 1;
    } else {
        /* regular object */
        pass = 0;
    }

    /*
     * The VCL variables beresp.do_g[un]zip tells us how we want the
     * object processed before it is stored.
     *
     * The backend Content-Encoding header tells us what we are going
     * to receive, which we classify in the following three classes:
     *
     *	"Content-Encoding: gzip"	--> object is gzip'ed.
     *	no Content-Encoding		--> object is not gzip'ed.
     *	anything else			--> do nothing wrt gzip
     *
     */

    /* We do nothing unless the param is set */
    if (!cache_param->http_gzip_support)
        wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;

    wrk->busyobj->is_gzip =
        http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");

    wrk->busyobj->is_gunzip =
        !http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);

    /* It can't be both */
    assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);

    /* We won't gunzip unless it is gzip'ed */
    if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
        wrk->busyobj->do_gunzip = 0;

    /* If we do gunzip, remove the C-E header */
    if (wrk->busyobj->do_gunzip)
        http_Unset(wrk->busyobj->beresp, H_Content_Encoding);

    /* We wont gzip unless it is ungziped */
    if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
        wrk->busyobj->do_gzip = 0;

    /* If we do gzip, add the C-E header */
    if (wrk->busyobj->do_gzip)
        http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
                       "Content-Encoding: gzip");

    /* But we can't do both at the same time */
    assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);

    /* ESI takes precedence and handles gzip/gunzip itself */
    if (wrk->busyobj->do_esi)
        wrk->busyobj->vfp = &vfp_esi;
    else if (wrk->busyobj->do_gunzip)
        wrk->busyobj->vfp = &vfp_gunzip;
    else if (wrk->busyobj->do_gzip)
        wrk->busyobj->vfp = &vfp_gzip;
    else if (wrk->busyobj->is_gzip)
        wrk->busyobj->vfp = &vfp_testgzip;

    if (wrk->busyobj->do_esi || sp->esi_level > 0)
        wrk->busyobj->do_stream = 0;
    if (!sp->wantbody)
        wrk->busyobj->do_stream = 0;

    l = http_EstimateWS(wrk->busyobj->beresp,
                        pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);

    /* Create Vary instructions */
    if (wrk->objcore != NULL) {
        CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
        vary = VRY_Create(sp, wrk->busyobj->beresp);
        if (vary != NULL) {
            varyl = VSB_len(vary);
            assert(varyl > 0);
            l += varyl;
        }
    }

    /*
     * Space for producing a Content-Length: header including padding
     * A billion gigabytes is enough for anybody.
     */
    l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);

    if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
            wrk->objcore == NULL)
        wrk->storage_hint = TRANSIENT_STORAGE;

    wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
    if (wrk->obj == NULL) {
        /*
         * Try to salvage the transaction by allocating a
         * shortlived object on Transient storage.
         */
        wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
        if (wrk->busyobj->exp.ttl > cache_param->shortlived)
            wrk->busyobj->exp.ttl = cache_param->shortlived;
        wrk->busyobj->exp.grace = 0.0;
        wrk->busyobj->exp.keep = 0.0;
    }
    if (wrk->obj == NULL) {
        sp->err_code = 503;
        sp->step = STP_ERROR;
        VDI_CloseFd(wrk, &wrk->busyobj->vbc);
        VBO_DerefBusyObj(wrk, &wrk->busyobj);
        return (0);
    }
    CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);

    wrk->storage_hint = NULL;

    if (wrk->busyobj->do_gzip ||
            (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
        wrk->obj->gziped = 1;

    if (vary != NULL) {
        wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
        AN(wrk->obj->vary);
        memcpy(wrk->obj->vary, VSB_data(vary), varyl);
        VRY_Validate(wrk->obj->vary);
        VSB_delete(vary);
    }

    wrk->obj->xid = sp->xid;
    wrk->obj->response = sp->err_code;
    WS_Assert(wrk->obj->ws_o);

    /* Filter into object */
    hp = wrk->busyobj->beresp;
    hp2 = wrk->obj->http;

    hp2->logtag = HTTP_Obj;
    http_CopyResp(hp2, hp);
    http_FilterFields(wrk, sp->vsl_id, hp2, hp,
                      pass ? HTTPH_R_PASS : HTTPH_A_INS);
    http_CopyHome(wrk, sp->vsl_id, hp2);

    if (http_GetHdr(hp, H_Last_Modified, &b))
        wrk->obj->last_modified = VTIM_parse(b);
    else
        wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);

    assert(WRW_IsReleased(wrk));

    /*
     * If we can deliver a 304 reply, we don't bother streaming.
     * Notice that vcl_deliver{} could still nuke the headers
     * that allow the 304, in which case we return 200 non-stream.
     */
    if (wrk->obj->response == 200 &&
            sp->http->conds &&
            RFC2616_Do_Cond(sp))
        wrk->busyobj->do_stream = 0;

    AssertObjCorePassOrBusy(wrk->obj->objcore);

    if (wrk->busyobj->do_stream) {
        sp->step = STP_PREPRESP;
        return (0);
    }

    /* Use unmodified headers*/
    i = FetchBody(wrk, wrk->obj);

    http_Setup(wrk->busyobj->bereq, NULL);
    http_Setup(wrk->busyobj->beresp, NULL);
    wrk->busyobj->vfp = NULL;
    assert(WRW_IsReleased(wrk));
    AZ(wrk->busyobj->vbc);
    AN(sp->director);

    if (i) {
        HSH_Drop(wrk);
        VBO_DerefBusyObj(wrk, &wrk->busyobj);
        AZ(wrk->obj);
        sp->err_code = 503;
        sp->step = STP_ERROR;
        return (0);
    }

    if (wrk->obj->objcore != NULL) {
        EXP_Insert(wrk->obj);
        AN(wrk->obj->objcore);
        AN(wrk->obj->objcore->ban);
        HSH_Unbusy(wrk);
    }
    VBO_DerefBusyObj(wrk, &wrk->busyobj);
    wrk->acct_tmp.fetch++;
    sp->step = STP_PREPRESP;
    return (0);
}
Beispiel #10
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	struct object *obj;
	struct objiter *oi;
	void *sp;
	ssize_t sl, al, tl;
	struct storage *st;
	enum objiter_status ois;
	char *p;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	if (bo->ims_obj->changed_gzip) {
		/*
		 * If we modified the gzip status of the IMS object, that
		 * must control the C-E header, if any.
		 */
		http_Unset(bo->beresp, H_Content_Encoding);
		if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p))
			http_PrintfHeader(bo->beresp,
			    "Content-Encoding: %s", p);
	}

	AZ(vbf_beresp2obj(bo));
	obj = bo->fetch_obj;
	bo->vfc->body = obj->body;

	if (bo->ims_obj->esidata != NULL) {
		sl = bo->ims_obj->esidata->len;
		obj->esidata = STV_alloc(bo->vfc, sl);
		if (obj->esidata == NULL || obj->esidata->space < sl) {
			VSLb(bo->vsl, SLT_Error,
			    "No space for %zd bytes of ESI data", sl);
			return (F_STP_FAIL);
		}
		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
		obj->esidata->len = sl;
	}

	obj->gziped = bo->ims_obj->gziped;
	obj->gzip_start = bo->ims_obj->gzip_start;
	obj->gzip_last = bo->ims_obj->gzip_last;
	obj->gzip_stop = bo->ims_obj->gzip_stop;

	AZ(WS_Overflowed(bo->ws_o));
	if (bo->do_stream) {
		HSH_Unbusy(&wrk->stats, obj->objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	st = NULL;
	al = 0;

	oi = ObjIterBegin(wrk, bo->ims_obj);
	do {
		ois = ObjIter(oi, &sp, &sl);
		while (sl > 0) {
			if (st == NULL)
				st = VFP_GetStorage(bo->vfc,
				    bo->ims_obj->len - al);
			if (st == NULL)
				break;
			tl = sl;
			if (tl > st->space - st->len)
				tl = st->space - st->len;
			memcpy(st->ptr + st->len, sp, tl);
			al += tl;
			sp = (char *)sp + tl;
			sl -= tl;
			VBO_extend(bo, tl);
			if (st->len == st->space)
				st = NULL;
		}
	} while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM));
	ObjIterEnd(&oi);
	if (bo->vfc->failed)
		return (F_STP_FAIL);

	if (!bo->do_stream)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	assert(al == bo->ims_obj->len);
	assert(obj->len == al);
	EXP_Rearm(bo->ims_obj->objcore,
	    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	if (bo->vbc != NULL && bo->doclose == SC_NULL) {
		VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Beispiel #11
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct object *obj;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	AN(bo->vbc);

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi))
		(void)VFP_Push(bo->vfc, &vfp_gunzip, 1);

	if (bo->do_esi && bo->do_gzip) {
		(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
		(void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1);
	} else if (bo->do_esi) {
		(void)VFP_Push(bo->vfc, &vfp_esi, 1);
	} else if (bo->do_gzip) {
		(void)VFP_Push(bo->vfc, &vfp_gzip, 1);
	} else if (bo->is_gzip && !bo->do_gunzip) {
		(void)VFP_Push(bo->vfc, &vfp_testgunzip, 1);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	if (VFP_Open(bo->vfc)) {
		(void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open");
		bo->doclose = SC_RX_BODY;
		return (F_STP_ERROR);
	}

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		bo->doclose = SC_RX_BODY;
		return (F_STP_ERROR);
	}

	assert(WRW_IsReleased(wrk));

	obj = bo->fetch_obj;
	bo->vfc->body = obj->body;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (bo->do_gzip || bo->do_gunzip)
		obj->changed_gzip = 1;

	if (bo->htc.body_status != BS_NONE)
		V1F_Setup_Fetch(bo);

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	AZ(WS_Overflowed(bo->ws_o));

	assert (bo->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		HSH_Unbusy(&wrk->stats, obj->objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc.body_status != BS_NONE) {
		assert(bo->htc.body_status != BS_ERROR);
		VFP_Fetch_Body(bo);
		bo->acct.beresp_bodybytes = bo->vfc->bodybytes;
	}

	if (bo->vfc->failed && !bo->do_stream) {
		assert(bo->state < BOS_STREAM);
		if (bo->fetch_obj != NULL) {
			ObjFreeObj(bo->fetch_objcore, bo->stats);
			bo->fetch_obj = NULL;
		}
		return (F_STP_ERROR);
	}

	if (bo->vfc->failed)
		return (F_STP_FAIL);

	if (bo->do_stream)
		assert(bo->state == BOS_STREAM);
	else {
		assert(bo->state == BOS_REQ_DONE);
		HSH_Unbusy(&wrk->stats, obj->objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	if (bo->vbc != NULL && bo->doclose == SC_NULL) {
		VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	if (bo->ims_obj != NULL)
		EXP_Rearm(bo->ims_obj->objcore,
		    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
	return (F_STP_DONE);
}
Beispiel #12
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct object *obj;
	ssize_t est;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 * XXX: BS_NONE/cl==0 should avoid gzip/gunzip
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	AN(bo->vbc);
	est = V1F_Setup_Fetch(bo);

	if (est == 0) {
		/*
		 * If the length is known to be zero, it's not gziped.
		 * A similar issue exists for chunked encoding but we
		 * don't handle that.  See #1320.
		 */
		http_Unset(bo->beresp, H_Content_Encoding);
		bo->is_gzip = 0;
		bo->is_gunzip = 1;
	}

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	/* Fix Content-Encoding, as appropriate */
	if (bo->do_gzip)
		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
	else if (bo->do_gunzip)
		http_Unset(bo->beresp, H_Content_Encoding);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
		RFC2616_Weaken_Etag(bo->beresp);
		VFP_Push(bo, vfp_gunzip_pull, 0);
	}

	if (bo->do_esi && bo->do_gzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi) {
		VFP_Push(bo, vfp_esi_pull, 0);
	} else if (bo->do_gzip) {
		VFP_Push(bo, vfp_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_testgunzip_pull, 0);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo, "Could not get storage");
		VDI_CloseFd(&bo->vbc, &bo->acct);
		return (F_STP_ERROR);
	}

	assert(WRW_IsReleased(wrk));

	obj = bo->fetch_obj;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (bo->do_gzip || bo->do_gunzip)
		obj->changed_gzip = 1;

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	AZ(WS_Overflowed(bo->ws_o));

	assert (bo->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		HSH_Unbusy(&wrk->stats, obj->objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc.body_status != BS_NONE) {
		assert(bo->htc.body_status  != BS_ERROR);
		VFP_Fetch_Body(bo, est);
	}

	if (bo->failed && !bo->do_stream) {
		assert(bo->state < BOS_STREAM);
		if (bo->fetch_obj != NULL) {
			oc_freeobj(bo->fetch_objcore);
			bo->fetch_obj = NULL;
			bo->stats->n_object--;
		}
		return (F_STP_ERROR);
	}

	if (bo->failed)
		return (F_STP_FAIL);

	if (bo->do_stream)
		assert(bo->state == BOS_STREAM);
	else {
		assert(bo->state == BOS_REQ_DONE);
		HSH_Unbusy(&wrk->stats, obj->objcore);
	}

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	if (bo->vbc != NULL && !(bo->should_close)) {
		VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Beispiel #13
0
static int
cnt_fetchbody(struct worker *wrk, struct req *req)
{
	struct http *hp, *hp2;
	char *b;
	uint16_t nhttp;
	unsigned l;
	struct vsb *vary = NULL;
	int varyl = 0, pass;
	struct busyobj *bo;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
	bo = req->busyobj;
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(req->handling == VCL_RET_DELIVER);

	if (req->objcore->objhead == NULL) {
		/* This is a pass from vcl_recv */
		pass = 1;
		/* VCL may have fiddled this, but that doesn't help */
		bo->exp.ttl = -1.;
	} else if (bo->do_pass) {
		pass = 1;
	} else {
		/* regular object */
		pass = 0;
	}

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* If we do gunzip, remove the C-E header */
	if (bo->do_gunzip)
		http_Unset(bo->beresp, H_Content_Encoding);

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	/* If we do gzip, add the C-E header */
	if (bo->do_gzip)
		http_SetHeader(bo->beresp, "Content-Encoding: gzip");

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	/* ESI takes precedence and handles gzip/gunzip itself */
	if (bo->do_esi)
		bo->vfp = &vfp_esi;
	else if (bo->do_gunzip)
		bo->vfp = &vfp_gunzip;
	else if (bo->do_gzip)
		bo->vfp = &vfp_gzip;
	else if (bo->is_gzip)
		bo->vfp = &vfp_testgzip;

	if (bo->do_esi || req->esi_level > 0)
		bo->do_stream = 0;
	if (!req->wantbody)
		bo->do_stream = 0;

	/* No reason to try streaming a non-existing body */
	if (bo->body_status == BS_NONE)
		bo->do_stream = 0;

	l = http_EstimateWS(bo->beresp,
	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);

	/* Create Vary instructions */
	if (req->objcore->objhead != NULL) {
		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
		vary = VRY_Create(req, bo->beresp);
		if (vary != NULL) {
			varyl = VSB_len(vary);
			assert(varyl > 0);
			l += varyl;
		}
	}

	/*
	 * Space for producing a Content-Length: header including padding
	 * A billion gigabytes is enough for anybody.
	 */
	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);

	if (bo->exp.ttl < cache_param->shortlived ||
	    req->objcore == NULL)
		req->storage_hint = TRANSIENT_STORAGE;

	AZ(bo->stats);
	bo->stats = &wrk->stats;
	req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
	    nhttp);
	if (req->obj == NULL) {
		/*
		 * Try to salvage the transaction by allocating a
		 * shortlived object on Transient storage.
		 */
		if (bo->exp.ttl > cache_param->shortlived)
			bo->exp.ttl = cache_param->shortlived;
		bo->exp.grace = 0.0;
		bo->exp.keep = 0.0;
		req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
		    l, nhttp);
	}
	bo->stats = NULL;
	if (req->obj == NULL) {
		req->err_code = 503;
		req->req_step = R_STP_ERROR;
		VDI_CloseFd(&bo->vbc);
		VBO_DerefBusyObj(wrk, &req->busyobj);
		return (0);
	}
	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);

	req->storage_hint = NULL;

	AZ(bo->fetch_obj);
	bo->fetch_obj = req->obj;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		req->obj->gziped = 1;

	if (vary != NULL) {
		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
		    VSB_data(vary), varyl);
		AN(req->obj->vary);
		VRY_Validate(req->obj->vary);
		VSB_delete(vary);
	}

	req->obj->vxid = bo->vsl->wid;
	req->obj->response = req->err_code;
	WS_Assert(req->obj->ws_o);

	/* Filter into object */
	hp = bo->beresp;
	hp2 = req->obj->http;

	hp2->logtag = HTTP_Obj;
	http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
	http_CopyHome(hp2);

	if (http_GetHdr(hp, H_Last_Modified, &b))
		req->obj->last_modified = VTIM_parse(b);
	else
		req->obj->last_modified = floor(bo->exp.entered);

	assert(WRW_IsReleased(wrk));

	/*
	 * If we can deliver a 304 reply, we don't bother streaming.
	 * Notice that vcl_deliver{} could still nuke the headers
	 * that allow the 304, in which case we return 200 non-stream.
	 */
	if (req->obj->response == 200 &&
	    req->http->conds &&
	    RFC2616_Do_Cond(req))
		bo->do_stream = 0;

	/*
	 * Ready to fetch the body
	 */
	bo->fetch_task.func = FetchBody;
	bo->fetch_task.priv = bo;

	assert(bo->refcount == 2);	/* one for each thread */

	if (req->obj->objcore->objhead != NULL) {
		EXP_Insert(req->obj);
		AN(req->obj->objcore->ban);
		AZ(req->obj->ws_o->overflow);
		HSH_Unbusy(&wrk->stats, req->obj->objcore);
	}

	if (!bo->do_stream ||
	    Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
		FetchBody(wrk, bo);

	if (req->obj->objcore->objhead != NULL)
		HSH_Ref(req->obj->objcore);

	if (bo->state == BOS_FINISHED) {
		VBO_DerefBusyObj(wrk, &req->busyobj);
	} else if (bo->state == BOS_FAILED) {
		/* handle early failures */
		HSH_Deref(&wrk->stats, NULL, &req->obj);
		VBO_DerefBusyObj(wrk, &req->busyobj);
		req->err_code = 503;
		req->req_step = R_STP_ERROR;
		return (0);
	}

	assert(WRW_IsReleased(wrk));
	req->req_step = R_STP_PREPRESP;
	return (0);
}
static int
cnt_fetch(struct sess *sp)
{
    int i, transient;
    struct http *hp, *hp2;
    char *b;
    unsigned handling;

    CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
    CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);

    AN(sp->director);
    AZ(sp->vbe);

    /* sp->wrk->http[0] is (still) bereq */
    sp->wrk->beresp = &sp->wrk->http[1];
    http_Setup(sp->wrk->beresp, sp->wrk->ws);

    i = FetchHdr(sp);

    /*
     * Save a copy before it might get mangled in VCL.  When it comes to
     * dealing with the body, we want to see the unadultered headers.
     */
    sp->wrk->beresp1 = &sp->wrk->http[2];
    *sp->wrk->beresp1 = *sp->wrk->beresp;

    if (i) {
        if (sp->objhead) {
            CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
            CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
            HSH_DerefObjCore(sp);
        }
        AZ(sp->obj);
        sp->wrk->bereq = NULL;
        sp->wrk->beresp = NULL;
        sp->wrk->beresp1 = NULL;
        sp->err_code = 503;
        sp->step = STP_ERROR;
        return (0);
    }

    sp->err_code = http_GetStatus(sp->wrk->beresp);

    /*
     * Initial cacheability determination per [RFC2616, 13.4]
     * We do not support ranges yet, so 206 is out.
     */
    switch (sp->err_code) {
    case 200: /* OK */
    case 203: /* Non-Authoritative Information */
    case 300: /* Multiple Choices */
    case 301: /* Moved Permanently */
    case 302: /* Moved Temporarily */
    case 410: /* Gone */
    case 404: /* Not Found */
        sp->wrk->cacheable = 1;
        break;
    default:
        sp->wrk->cacheable = 0;
        break;
    }

    sp->wrk->entered = TIM_real();
    sp->wrk->age = 0;
    sp->wrk->ttl = RFC2616_Ttl(sp);

    if (sp->wrk->ttl == 0.)
        sp->wrk->cacheable = 0;

    sp->wrk->do_esi = 0;
    sp->wrk->grace = NAN;

    VCL_fetch_method(sp);

    /*
     * When we fetch the body, we may hit the LRU cleanup and that
     * will overwrite sp->handling, so we have to save our plans
     * here.
     */
    handling = sp->handling;

    if (sp->objhead == NULL)
        transient = 1;
    else if (sp->handling == VCL_RET_DELIVER)
        transient = 0;
    else
        transient = 1;

    /*
     * XXX: If we have a Length: header, we should allocate the body
     * XXX: also.
     */
    sp->obj = HSH_NewObject(sp, transient);

    if (sp->objhead != NULL) {
        CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
        CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
        sp->objcore->obj = sp->obj;
        sp->obj->objcore = sp->objcore;
        sp->obj->objhead = sp->objhead;
        sp->objhead = NULL;	/* refcnt follows pointer. */
        sp->objcore = NULL;	/* refcnt follows pointer. */
    }

    BAN_NewObj(sp->obj);

    sp->obj->xid = sp->xid;
    sp->obj->response = sp->err_code;
    sp->obj->cacheable = sp->wrk->cacheable;
    sp->obj->ttl = sp->wrk->ttl;
    sp->obj->grace = sp->wrk->grace;
    if (sp->obj->ttl == 0. && sp->obj->grace == 0.)
        sp->obj->cacheable = 0;
    sp->obj->age = sp->wrk->age;
    sp->obj->entered = sp->wrk->entered;
    WS_Assert(sp->obj->ws_o);

    /* Filter into object */
    hp = sp->wrk->beresp;
    hp2 = sp->obj->http;

    hp2->logtag = HTTP_Obj;
    http_CopyResp(hp2, hp);
    http_FilterFields(sp->wrk, sp->fd, hp2, hp, HTTPH_A_INS);
    http_CopyHome(sp->wrk, sp->fd, hp2);

    if (http_GetHdr(hp, H_Last_Modified, &b))
        sp->obj->last_modified = TIM_parse(b);

    i = FetchBody(sp);
    AZ(sp->wrk->wfd);
    AZ(sp->vbe);
    AN(sp->director);

    if (i) {
        HSH_Drop(sp);
        AZ(sp->obj);
        sp->wrk->bereq = NULL;
        sp->wrk->beresp = NULL;
        sp->wrk->beresp1 = NULL;
        sp->err_code = 503;
        sp->step = STP_ERROR;
        return (0);
    }

    if (!transient)
        HSH_Object(sp);

    if (sp->wrk->do_esi)
        ESI_Parse(sp);

    switch (handling) {
    case VCL_RET_RESTART:
        HSH_Drop(sp);
        sp->director = NULL;
        sp->restarts++;
        sp->wrk->bereq = NULL;
        sp->wrk->beresp = NULL;
        sp->wrk->beresp1 = NULL;
        sp->step = STP_RECV;
        return (0);
    case VCL_RET_PASS:
        if (sp->obj->objcore != NULL)
            sp->obj->objcore->flags |= OC_F_PASS;
        if (sp->obj->ttl - sp->t_req < params->default_ttl)
            sp->obj->ttl = sp->t_req + params->default_ttl;
        break;
    case VCL_RET_DELIVER:
        break;
    case VCL_RET_ERROR:
        HSH_Drop(sp);
        sp->wrk->bereq = NULL;
        sp->wrk->beresp = NULL;
        sp->wrk->beresp1 = NULL;
        sp->step = STP_ERROR;
        return (0);
    default:
        WRONG("Illegal action in vcl_fetch{}");
    }

    sp->obj->cacheable = 1;
    if (sp->obj->objhead != NULL) {
        VRY_Create(sp);
        EXP_Insert(sp->obj);
        AN(sp->obj->ban);
        HSH_Unbusy(sp);
    }
    sp->acct_req.fetch++;
    sp->wrk->bereq = NULL;
    sp->wrk->beresp = NULL;
    sp->wrk->beresp1 = NULL;
    sp->step = STP_DELIVER;
    return (0);
}
Beispiel #15
0
static int
cnt_streambody(struct sess *sp)
{
    int i;
    struct stream_ctx sctx;
    uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
                 cache_param->gzip_stack_buffer : 1];
    struct worker *wrk;

    CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
    wrk = sp->wrk;
    CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);

    CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
    memset(&sctx, 0, sizeof sctx);
    sctx.magic = STREAM_CTX_MAGIC;
    AZ(wrk->sctx);
    wrk->sctx = &sctx;

    if (wrk->res_mode & RES_GUNZIP) {
        sctx.vgz = VGZ_NewUngzip(wrk, "U S -");
        sctx.obuf = obuf;
        sctx.obuf_len = sizeof (obuf);
    }

    RES_StreamStart(sp);

    AssertObjCorePassOrBusy(wrk->obj->objcore);

    i = FetchBody(wrk, wrk->obj);

    http_Setup(wrk->busyobj->bereq, NULL);
    http_Setup(wrk->busyobj->beresp, NULL);
    wrk->busyobj->vfp = NULL;
    AZ(wrk->busyobj->vbc);
    AN(sp->director);

    if (!i && wrk->obj->objcore != NULL) {
        EXP_Insert(wrk->obj);
        AN(wrk->obj->objcore);
        AN(wrk->obj->objcore->ban);
        HSH_Unbusy(wrk);
    } else {
        sp->doclose = "Stream error";
    }
    wrk->acct_tmp.fetch++;
    sp->director = NULL;
    sp->restarts = 0;

    RES_StreamEnd(sp);
    if (wrk->res_mode & RES_GUNZIP)
        (void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);

    wrk->sctx = NULL;
    assert(WRW_IsReleased(wrk));
    assert(wrk->wrw.ciov == wrk->wrw.siov);
    (void)HSH_Deref(wrk, NULL, &wrk->obj);
    VBO_DerefBusyObj(wrk, &wrk->busyobj);
    http_Setup(wrk->resp, NULL);
    sp->step = STP_DONE;
    return (0);
}
Beispiel #16
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	const char *p;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	if (vbf_figure_out_vfp(bo)) {
		(bo)->htc->doclose = SC_OVERLOAD;
		VDI_Finish((bo)->wrk, bo);
		return (F_STP_ERROR);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	bo->fetch_objcore->boc->len_so_far = 0;

	if (VFP_Open(bo->vfc)) {
		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
		bo->htc->doclose = SC_RX_BODY;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		bo->htc->doclose = SC_RX_BODY;
		VFP_Close(bo->vfc);
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	if (bo->do_esi)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1);

	if (bo->do_gzip || bo->do_gunzip)
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1);

	if (!(bo->fetch_objcore->flags & OC_F_PASS) &&
	    http_IsStatus(bo->beresp, 200) && (
	      http_GetHdr(bo->beresp, H_Last_Modified, &p) ||
	      http_GetHdr(bo->beresp, H_ETag, &p)))
		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1);

	if (bo->htc->body_status != BS_NONE &&
	    VDI_GetBody(bo->wrk, bo) != 0) {
		(void)VFP_Error(bo->vfc,
		    "GetBody failed - workspace_backend overflow?");
		VFP_Close(bo->vfc);
		bo->htc->doclose = SC_OVERLOAD;
		VDI_Finish(bo->wrk, bo);
		return (F_STP_ERROR);
	}

	assert(bo->fetch_objcore->boc->refcount >= 1);

	assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE);

	if (bo->do_stream) {
		ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM);
		HSH_Unbusy(wrk, bo->fetch_objcore);
		ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM);
	}

	VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s",
	    bo->htc->body_status, body_status_2str(bo->htc->body_status),
	    bo->do_stream ? "stream" : "-");

	if (bo->htc->body_status != BS_NONE) {
		assert(bo->htc->body_status != BS_ERROR);
		return (F_STP_FETCHBODY);
	}
	AZ(bo->vfc->failed);
	return (F_STP_FETCHEND);
}
Beispiel #17
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	struct storage *st;
	ssize_t l;
	double now;
	char time_str[VTIM_FORMAT_SIZE];

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	AZ(bo->synth_body);
	bo->synth_body = VSB_new_auto();
	AN(bo->synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	VTIM_format(now, time_str);
	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->exp.t_origin = bo->t_prev;
	bo->fetch_objcore->exp.ttl = 0;
	bo->fetch_objcore->exp.grace = 0;
	bo->fetch_objcore->exp.keep = 0;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);

	AZ(VSB_finish(bo->synth_body));

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_delete(bo->synth_body);
		bo->synth_body = NULL;
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		bo->synth_body = NULL;
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	VFP_Setup(bo->vfc);
	bo->vfc->bo = bo;
	bo->vfc->http = bo->beresp;
	bo->vfc->vsl = bo->vsl;

	if (vbf_beresp2obj(bo))
		return (F_STP_FAIL);

	bo->vfc->body = bo->fetch_obj->body;

	l = VSB_len(bo->synth_body);
	if (l > 0) {
		st = VFP_GetStorage(bo->vfc, l);
		if (st != NULL) {
			if (st->space < l) {
				VSLb(bo->vsl, SLT_Error,
				    "No space for %zd bytes of synth body", l);
			} else {
				memcpy(st->ptr, VSB_data(bo->synth_body), l);
				VBO_extend(bo, l);
			}
		}
	}
	VSB_delete(bo->synth_body);
	bo->synth_body = NULL;

	HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Beispiel #18
0
static enum fetch_step
vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
{
	struct http *hp, *hp2;
	char *b;
	uint16_t nhttp;
	unsigned l;
	struct vsb *vary = NULL;
	int varyl = 0;
	struct object *obj;
	ssize_t est = -1;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	assert(wrk->handling == VCL_RET_DELIVER);

	/*
	 * The VCL variables beresp.do_g[un]zip tells us how we want the
	 * object processed before it is stored.
	 *
	 * The backend Content-Encoding header tells us what we are going
	 * to receive, which we classify in the following three classes:
	 *
	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
	 *	no Content-Encoding		--> object is not gzip'ed.
	 *	anything else			--> do nothing wrt gzip
	 *
	 * XXX: BS_NONE/cl==0 should avoid gzip/gunzip
	 */

	/* We do nothing unless the param is set */
	if (!cache_param->http_gzip_support)
		bo->do_gzip = bo->do_gunzip = 0;

	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");

	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);

	/* It can't be both */
	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);

	/* We won't gunzip unless it is gzip'ed */
	if (bo->do_gunzip && !bo->is_gzip)
		bo->do_gunzip = 0;

	/* If we do gunzip, remove the C-E header */
	if (bo->do_gunzip)
		http_Unset(bo->beresp, H_Content_Encoding);

	/* We wont gzip unless it is ungziped */
	if (bo->do_gzip && !bo->is_gunzip)
		bo->do_gzip = 0;

	/* If we do gzip, add the C-E header */
	if (bo->do_gzip)
		http_SetHeader(bo->beresp, "Content-Encoding: gzip");

	/* But we can't do both at the same time */
	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);

	if (bo->vbc != NULL)
		est = V1F_Setup_Fetch(bo);

	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
		RFC2616_Weaken_Etag(bo->beresp);
		VFP_Push(bo, vfp_gunzip_pull, 0);
	}

	if (bo->do_esi && bo->do_gzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_esi_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->do_esi) {
		VFP_Push(bo, vfp_esi_pull, 0);
	} else if (bo->do_gzip) {
		VFP_Push(bo, vfp_gzip_pull, 0);
		RFC2616_Weaken_Etag(bo->beresp);
	} else if (bo->is_gzip && !bo->do_gunzip) {
		VFP_Push(bo, vfp_testgunzip_pull, 0);
	}

	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
		AN(bo->uncacheable);

	/* No reason to try streaming a non-existing body */
	if (bo->htc.body_status == BS_NONE)
		bo->do_stream = 0;

	l = 0;

	/* Create Vary instructions */
	if (!(bo->fetch_objcore->flags & OC_F_PRIVATE)) {
		varyl = VRY_Create(bo, &vary);
		if (varyl > 0) {
			AN(vary);
			assert(varyl == VSB_len(vary));
			l += varyl;
		} else if (varyl < 0) {
			/*
			 * Vary parse error
			 * Complain about it, and make this a pass.
			 */
			VSLb(bo->vsl, SLT_Error,
			    "Illegal 'Vary' header from backend, "
			    "making this a pass.");
			bo->uncacheable = 1;
			AZ(vary);
		} else
			/* No vary */
			AZ(vary);
	}

	l += http_EstimateWS(bo->beresp,
	    bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);

	if (bo->uncacheable)
		bo->fetch_objcore->flags |= OC_F_PASS;

	if (bo->uncacheable ||
	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
		bo->storage_hint = TRANSIENT_STORAGE;

	AZ(bo->stats);
	bo->stats = &wrk->stats;
	AN(bo->fetch_objcore);
	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
	if (obj == NULL) {
		/*
		 * Try to salvage the transaction by allocating a
		 * shortlived object on Transient storage.
		 */
		if (bo->exp.ttl > cache_param->shortlived)
			bo->exp.ttl = cache_param->shortlived;
		bo->exp.grace = 0.0;
		bo->exp.keep = 0.0;
		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
	}
	if (obj == NULL) {
		bo->stats = NULL;
		(void)VFP_Error(bo, "Could not get storage");
		VDI_CloseFd(&bo->vbc);
		return (F_STP_DONE);
	}
	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);

	bo->storage_hint = NULL;

	AZ(bo->fetch_obj);
	bo->fetch_obj = obj;

	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
		obj->gziped = 1;

	if (vary != NULL) {
		obj->vary = (void *)WS_Copy(obj->http->ws,
		    VSB_data(vary), varyl);
		AN(obj->vary);
		(void)VRY_Validate(obj->vary);
		VSB_delete(vary);
	}

	obj->vxid = bo->vsl->wid;
	obj->response = bo->err_code;
	WS_Assert(bo->ws_o);

	/* Filter into object */
	hp = bo->beresp;
	hp2 = obj->http;

	hp2->logtag = HTTP_Obj;
	http_FilterResp(hp, hp2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS);
	http_CopyHome(hp2);

	if (http_GetHdr(hp, H_Last_Modified, &b))
		obj->last_modified = VTIM_parse(b);
	else
		obj->last_modified = floor(bo->exp.t_origin);

	assert(WRW_IsReleased(wrk));

	/*
	 * Ready to fetch the body
	 */

	assert(bo->refcount >= 1);

	AZ(WS_Overflowed(bo->ws_o));
	if (bo->do_stream)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	assert(bo->state == BOS_REQ_DONE);
	VBO_setstate(bo, BOS_FETCHING);

	switch (bo->htc.body_status) {
	case BS_NONE:
		break;
	case BS_ERROR:
		/* XXX: Why not earlier ? */
		bo->should_close |=
		    VFP_Error(bo, "error incompatible Transfer-Encoding");
		break;
	default:
		if (bo->vbc == NULL)
			(void)VFP_Error(bo, "Backend connection gone");
		else
			VFP_Fetch_Body(bo, est);
	}

	bo->stats = NULL;

	bo->t_body = VTIM_mono();

	if (bo->vbc != NULL) {
		if (bo->should_close)
			VDI_CloseFd(&bo->vbc);
		else
			VDI_RecycleFd(&bo->vbc);
		AZ(bo->vbc);
	}

	http_Teardown(bo->bereq);
	http_Teardown(bo->beresp);

	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
	    bo->htc.body_status, body_status_2str(bo->htc.body_status));

	if (bo->state == BOS_FAILED) {
		wrk->stats.fetch_failed++;
	} else {
		assert(bo->state == BOS_FETCHING);

		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);

		{
		/* Sanity check fetch methods accounting */
			ssize_t uu;
			struct storage *st;

			uu = 0;
			VTAILQ_FOREACH(st, &obj->store, list)
				uu += st->len;
			if (bo->do_stream)
				/* Streaming might have started freeing stuff */
				assert(uu <= obj->len);

			else
				assert(uu == obj->len);
		}
	}

	if (!bo->do_stream && bo->state != BOS_FAILED)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	if (bo->state != BOS_FAILED && !(obj->objcore->flags & OC_F_PRIVATE)) {
		EXP_Insert(obj->objcore);
		AN(obj->objcore->ban);
	}

	HSH_Complete(obj->objcore);

	assert(bo->refcount >= 1);

	if (bo->state != BOS_FAILED)
		VBO_setstate(bo, BOS_FINISHED);

VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
	return (F_STP_DONE);
}
Beispiel #19
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	struct vsb *synth_body;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
	AN(bo->fetch_objcore->flags & OC_F_BUSY);
	assert(bo->director_state == DIR_S_NULL);

	wrk->stats->fetch_failed++;

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	if (bo->fetch_objcore->stobj->stevedore != NULL)
		ObjFreeObj(bo->wrk, bo->fetch_objcore);

	if (bo->storage == NULL)
		bo->storage = STV_next();

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	http_TimeHeader(bo->beresp, "Date: ", now);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->t_origin = now;
	if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) {
		/*
		 * If there is a waitinglist, it means that there is no
		 * grace-able object, so cache the error return for a
		 * short time, so the waiting list can drain, rather than
		 * each objcore on the waiting list sequentially attempt
		 * to fetch from the backend.
		 */
		bo->fetch_objcore->ttl = 1;
		bo->fetch_objcore->grace = 5;
		bo->fetch_objcore->keep = 5;
	} else {
		bo->fetch_objcore->ttl = 0;
		bo->fetch_objcore->grace = 0;
		bo->fetch_objcore->keep = 0;
	}

	synth_body = VSB_new_auto();
	AN(synth_body);

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);

	AZ(VSB_finish(synth_body));

	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) {
		VSB_destroy(&synth_body);
		return (F_STP_FAIL);
	}

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_destroy(&synth_body);
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, failing");
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	assert(bo->vfc->wrk == bo->wrk);
	assert(bo->vfc->oc == bo->fetch_objcore);
	assert(bo->vfc->resp == bo->beresp);
	assert(bo->vfc->req == bo->bereq);

	if (vbf_beresp2obj(bo)) {
		(void)VFP_Error(bo->vfc, "Could not get storage");
		VSB_destroy(&synth_body);
		return (F_STP_FAIL);
	}

	ll = VSB_len(synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		if (l > ll)
			l = ll;
		memcpy(ptr, VSB_data(synth_body) + o, l);
		VFP_Extend(bo->vfc, l);
		ll -= l;
		o += l;
	}
	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o));
	VSB_destroy(&synth_body);
	HSH_Unbusy(wrk, bo->fetch_objcore);
	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
	return (F_STP_DONE);
}
Beispiel #20
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	char time_str[VTIM_FORMAT_SIZE];

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	assert(bo->director_state == DIR_S_NULL);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	AZ(bo->synth_body);
	bo->synth_body = VSB_new_auto();
	AN(bo->synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	VTIM_format(now, time_str);
	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->exp.t_origin = bo->t_prev;
	bo->fetch_objcore->exp.ttl = 0;
	bo->fetch_objcore->exp.grace = 0;
	bo->fetch_objcore->exp.keep = 0;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);

	AZ(VSB_finish(bo->synth_body));

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_delete(bo->synth_body);
		bo->synth_body = NULL;
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	VFP_Setup(bo->vfc);
	bo->vfc->bo = bo;
	bo->vfc->wrk = bo->wrk;
	bo->vfc->oc = bo->fetch_objcore;
	bo->vfc->http = bo->beresp;
	bo->vfc->esi_req = bo->bereq;

	if (vbf_beresp2obj(bo))
		return (F_STP_FAIL);

	ll = VSB_len(bo->synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		memcpy(ptr, VSB_data(bo->synth_body) + o, l);
		VBO_extend(bo, l);
		ll -= l;
		o += l;
	}
	VSB_delete(bo->synth_body);
	bo->synth_body = NULL;

	HSH_Unbusy(wrk, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}