Esempio n. 1
0
static int
vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef,
    struct http_conn *htc, ssize_t bytes)
{
	struct vgz *vg;
	ssize_t wl;
	enum vgzret_e vr;
	size_t dl;
	const void *dp;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
	vg = bo->vgz_rx;

	while (bytes > 0) {
		if (VGZ_IbufEmpty(vg) && bytes > 0) {
			wl = vef_read(htc, vef->ibuf, vef->ibuf_sz, bytes);
			if (wl <= 0)
				return (wl);
			VGZ_Ibuf(vg, vef->ibuf, wl);
			bytes -= wl;
		}
		if (VGZ_ObufStorage(bo, vg))
			return(-1);
		vr = VGZ_Gunzip(vg, &dp, &dl);
		if (vr < VGZ_OK)
			return (-1);
		if (dl > 0) {
			VEP_Parse(bo, dp, dl);
			VBO_extend(bo, dl);
		}
	}
	return (1);
}
Esempio n. 2
0
vfp_gzip_end(struct busyobj *bo)
{
	struct vgz *vg;
	size_t dl;
	const void *dp;
	int i;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vg = bo->vgz_rx;
	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
	bo->vgz_rx = NULL;
	if (bo->state == BOS_FAILED) {
		(void)VGZ_Destroy(&vg);
		return(0);
	}
	do {
		VGZ_Ibuf(vg, "", 0);
		if (VGZ_ObufStorage(bo, vg))
			return(-1);
		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
		VBO_extend(bo, dl);
	} while (i != Z_STREAM_END);
	VGZ_UpdateObj(vg, bo->fetch_obj);
	if (VGZ_Destroy(&vg) != VGZ_END)
		return(VFP_Error(bo, "Gzip error at the very end"));
	return (0);
}
Esempio n. 3
0
vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
{
	struct vgz *vg;
	ssize_t l, wl;
	int i = -100;
	size_t dl;
	const void *dp;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vg = bo->vgz_rx;
	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
	AZ(vg->vz.avail_in);
	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
		if (VGZ_IbufEmpty(vg) && bytes > 0) {
			l = vg->m_sz;
			if (l > bytes)
				l = bytes;
			wl = htc->read(htc, vg->m_buf, l);
			if (wl <= 0)
				return (wl);
			VGZ_Ibuf(vg, vg->m_buf, wl);
			bytes -= wl;
		}
		if (VGZ_ObufStorage(bo, vg))
			return(-1);
		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
		assert(i == Z_OK);
		VBO_extend(bo, dl);
	}
	return (1);
}
static ssize_t
vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg)
{
	struct vef_priv *vef;
	ssize_t dl;
	const void *dp;
	uint8_t *ptr;
	int i;

	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
	CAST_OBJ_NOTNULL(vef, priv, VEF_MAGIC);
	assert(l >= 0);

	if (vef->error) {
		vef->tot += l;
		return (vef->tot);
	}

	/*
	 * l == 0 is valid when 'flg' calls for action, but in the
	 * normal case we can just ignore a l==0 request.
	 * (It would cause Z_BUF_ERROR anyway)
	 */
	if (l == 0 && flg == VGZ_NORMAL)
		return (vef->tot);

	CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC);
	VGZ_Ibuf(vef->vgz, vef->ibuf_o, l);
	do {
		dl = 0;
		if (VFP_GetStorage(vc, &dl, &ptr) != VFP_OK) {
			vef->error = ENOMEM;
			vef->tot += l;
			return (vef->tot);
		}
		VGZ_Obuf(vef->vgz, ptr, dl);
		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
		VGZ_UpdateObj(vc, vef->vgz, VUA_UPDATE);
		vef->tot += dl;
		VBO_extend(vc->bo, dl);
	} while (i != VGZ_ERROR &&
	    (!VGZ_IbufEmpty(vef->vgz) || VGZ_ObufFull(vef->vgz)));
	assert(i == VGZ_ERROR || VGZ_IbufEmpty(vef->vgz));
	vef->ibuf_o += l;
	return (vef->tot);
}
Esempio n. 5
0
static ssize_t
vfp_vep_callback(struct busyobj *bo, ssize_t l, enum vgz_flag flg)
{
	struct vef_priv *vef;
	size_t dl;
	const void *dp;
	int i;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vef = bo->vef_priv;
	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
	assert(l >= 0);

	if (vef->error) {
		vef->tot += l;
		return (vef->tot);
	}

	/*
	 * l == 0 is valid when 'flg' calls for action, but in the
	 * normal case we can just ignore a l==0 request.
	 * (It would cause Z_BUF_ERROR anyway)
	 */
	if (l == 0 && flg == VGZ_NORMAL)
		return (vef->tot);

	VGZ_Ibuf(vef->vgz, vef->ibuf_o, l);
	do {
		if (VGZ_ObufStorage(bo, vef->vgz)) {
			vef->error = ENOMEM;
			vef->tot += l;
			return (vef->tot);
		}
		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
		vef->tot += dl;
		VBO_extend(bo, dl);
	} while (!VGZ_IbufEmpty(vef->vgz) ||
	    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
	assert(VGZ_IbufEmpty(vef->vgz));
	vef->ibuf_o += l;
	if (flg == VGZ_FINISH)
		assert(i == 1);			/* XXX */
	else
		assert(i == 0);			/* XXX */
	return (vef->tot);
}
Esempio n. 6
0
static ssize_t
vfp_vep_callback(struct busyobj *bo, void *priv, ssize_t l, enum vgz_flag flg)
{
    struct vef_priv *vef;
    size_t dl;
    const void *dp;
    struct storage *st;
    int i;

    CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
    CAST_OBJ_NOTNULL(vef, priv, VEF_MAGIC);
    assert(l >= 0);

    if (vef->error) {
        vef->tot += l;
        return (vef->tot);
    }

    /*
     * l == 0 is valid when 'flg' calls for action, but in the
     * normal case we can just ignore a l==0 request.
     * (It would cause Z_BUF_ERROR anyway)
     */
    if (l == 0 && flg == VGZ_NORMAL)
        return (vef->tot);

    VGZ_Ibuf(vef->vgz, vef->ibuf_o, l);
    do {
        st = VFP_GetStorage(bo, 0);
        if (st == NULL) {
            vef->error = ENOMEM;
            vef->tot += l;
            return (vef->tot);
        }
        VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
        i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
        vef->tot += dl;
        VBO_extend(bo, dl);
    } while (i != VGZ_ERROR &&
             (!VGZ_IbufEmpty(vef->vgz) || VGZ_ObufFull(vef->vgz)));
    assert(i == VGZ_ERROR || VGZ_IbufEmpty(vef->vgz));
    vef->ibuf_o += l;
    return (vef->tot);
}
Esempio n. 7
0
vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
{
	struct vgz *vg;
	ssize_t l, wl;
	int i = -100;
	size_t dl;
	const void *dp;
	struct storage *st;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vg = bo->vgz_rx;
	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
	AZ(vg->vz.avail_in);
	while (bytes > 0) {
		st = VFP_GetStorage(bo, 0);
		if (st == NULL)
			return(-1);
		l = st->space - st->len;
		if (l > bytes)
			l = bytes;
		wl = htc->read(htc, st->ptr + st->len, l);
		if (wl <= 0)
			return (wl);
		bytes -= wl;
		VGZ_Ibuf(vg, st->ptr + st->len, wl);
		VBO_extend(bo, wl);

		while (!VGZ_IbufEmpty(vg)) {
			VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
			i = VGZ_Gunzip(vg, &dp, &dl);
			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
				return(VFP_Error(bo, "Junk after gzip data"));
			if (i != VGZ_OK && i != VGZ_END)
				return(VFP_Error2(bo,
				    "Invalid Gzip data", vg->vz.msg));
		}
	}
	assert(i == VGZ_OK || i == VGZ_END);
	return (1);
}
vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
{
	ssize_t l, wl;
	struct storage *st;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	while (bytes > 0) {
		st = VFP_GetStorage(bo, 0);
		if (st == NULL)
			return(-1);
		l = st->space - st->len;
		if (l > bytes)
			l = bytes;
		wl = HTTP1_Read(htc, st->ptr + st->len, l);
		if (wl <= 0)
			return (wl);
		st->len += wl;
		VBO_extend(bo, wl);
		bytes -= wl;
	}
	return (1);
}
Esempio n. 9
0
vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
{
	struct vgz *vg;
	ssize_t l, wl;
	int i = -100;
	size_t dl;
	const void *dp;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vg = bo->vgz_rx;
	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
	AZ(vg->vz.avail_in);
	while (bytes > 0 || vg->vz.avail_in > 0) {
		if (vg->vz.avail_in == 0 && bytes > 0) {
			l = vg->m_sz;
			if (l > bytes)
				l = bytes;
			wl = htc->read(htc, vg->m_buf, l);
			if (wl <= 0)
				return (wl);
			VGZ_Ibuf(vg, vg->m_buf, wl);
			bytes -= wl;
		}

		if (VGZ_ObufStorage(bo, vg))
			return(-1);
		i = VGZ_Gunzip(vg, &dp, &dl);
		if (i != VGZ_OK && i != VGZ_END)
			return(VFP_Error(bo, "Gunzip data error"));
		if (i == VGZ_END && !VGZ_IbufEmpty(vg))
			return(VFP_Error(bo, "Junk after gzip data"));
		VBO_extend(bo, dl);
	}
	assert(i == Z_OK || i == Z_STREAM_END);
	return (1);
}
Esempio n. 10
0
static int
vfp_esi_bytes_uu(struct busyobj *bo, const struct vef_priv *vef,
    struct http_conn *htc, ssize_t bytes)
{
	ssize_t wl;
	struct storage *st;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);

	while (bytes > 0) {
		st = VFP_GetStorage(bo, 0);
		if (st == NULL)
			return (-1);
		wl = vef_read(htc,
		    st->ptr + st->len, st->space - st->len, bytes);
		if (wl <= 0)
			return (wl);
		VEP_Parse(bo, (const char *)st->ptr + st->len, wl);
		VBO_extend(bo, wl);
		bytes -= wl;
	}
	return (1);
}
Esempio n. 11
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	struct vsb *synth_body;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
	assert(bo->director_state == DIR_S_NULL);

	if(bo->fetch_objcore->stobj->stevedore != NULL)
		ObjFreeObj(bo->wrk, bo->fetch_objcore);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	synth_body = VSB_new_auto();
	AN(synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	http_TimeHeader(bo->beresp, "Date: ", now);
	http_SetHeader(bo->beresp, "Server: Varnish");

	EXP_Clr(&bo->fetch_objcore->exp);
	bo->fetch_objcore->exp.t_origin = bo->t_prev;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);

	AZ(VSB_finish(synth_body));

	if (wrk->handling == VCL_RET_RETRY ||
	    wrk->handling == VCL_RET_ABANDON) {
		VSB_delete(synth_body);

		if (bo->director_state != DIR_S_NULL) {
			bo->htc->doclose = SC_RESP_CLOSE;
			VDI_Finish(bo->wrk, bo);
		}

		if (wrk->handling == VCL_RET_RETRY) {
			if (bo->retries++ < cache_param->max_retries)
				return (F_STP_RETRY);
			VSLb(bo->vsl, SLT_VCL_Error,
			    "Too many retries, delivering 503");
		}

		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	bo->vfc->bo = bo;
	bo->vfc->wrk = bo->wrk;
	bo->vfc->oc = bo->fetch_objcore;
	bo->vfc->http = bo->beresp;
	bo->vfc->esi_req = bo->bereq;

	if (vbf_beresp2obj(bo)) {
		VSB_delete(synth_body);
		return (F_STP_FAIL);
	}

	ll = VSB_len(synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		memcpy(ptr, VSB_data(synth_body) + o, l);
		VBO_extend(bo, l);
		ll -= l;
		o += l;
	}
	VSB_delete(synth_body);

	HSH_Unbusy(wrk, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Esempio n. 12
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	void *oi;
	void *sp;
	ssize_t sl, al, l;
	uint8_t *ptr;
	enum objiter_status ois;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	AZ(vbf_beresp2obj(bo));

	if (ObjGetattr(bo->wrk, bo->stale_oc, OA_ESIDATA, NULL) != NULL)
		AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc,
		    OA_ESIDATA));

	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));

	if (bo->do_stream) {
		HSH_Unbusy(wrk, bo->fetch_objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	al = 0;
	oi = ObjIterBegin(wrk, bo->stale_oc);
	do {
		ois = ObjIter(bo->stale_oc, oi, &sp, &sl);
		if (ois == OIS_ERROR)
			(void)VFP_Error(bo->vfc, "Template object failed");
		while (sl > 0) {
			l = ObjGetLen(bo->wrk, bo->stale_oc) - al;
			assert(l > 0);
			if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
				break;
			if (sl < l)
				l = sl;
			memcpy(ptr, sp, l);
			VBO_extend(bo, l);
			al += l;
			sp = (char *)sp + l;
			sl -= l;
		}
	} while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM));
	ObjIterEnd(bo->stale_oc, &oi);
	if (bo->stale_oc->flags & OC_F_FAILED)
		(void)VFP_Error(bo->vfc, "Template object failed");
	if (bo->vfc->failed) {
		VDI_Finish(bo->wrk, bo);
		return (F_STP_FAIL);
	}

	if (!bo->do_stream)
		HSH_Unbusy(wrk, bo->fetch_objcore);

	assert(ObjGetLen(bo->wrk, bo->fetch_objcore) == al);
	EXP_Rearm(bo->stale_oc, bo->stale_oc->exp.t_origin, 0, 0, 0);

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	VDI_Finish(bo->wrk, bo);

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Esempio n. 13
0
static void
vbf_fetch_body_helper(struct busyobj *bo)
{
	ssize_t l;
	uint8_t *ptr;
	enum vfp_status vfps = VFP_ERROR;
	ssize_t est;
	struct vfp_ctx *vfc;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	vfc = bo->vfc;
	CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC);

	AN(vfc->vfp_nxt);

	est = bo->htc->content_length;
	if (est < 0)
		est = 0;

	do {
		if (bo->abandon) {
			/*
			 * A pass object and delivery was terminated
			 * We don't fail the fetch, in order for hit-for-pass
			 * objects to be created.
			 */
			AN(vfc->oc->flags & OC_F_PASS);
			VSLb(vfc->wrk->vsl, SLT_FetchError,
			    "Pass delivery abandoned");
			vfps = VFP_END;
			bo->htc->doclose = SC_RX_BODY;
			break;
		}
		AZ(vfc->failed);
		l = est;
		assert(l >= 0);
		if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) {
			bo->htc->doclose = SC_RX_BODY;
			break;
		}

		AZ(vfc->failed);
		vfps = VFP_Suck(vfc, ptr, &l);
		if (l > 0 && vfps != VFP_ERROR) {
			bo->acct.beresp_bodybytes += l;
			VBO_extend(bo, l);
			if (est >= l)
				est -= l;
			else
				est = 0;
		}
	} while (vfps == VFP_OK);

	VFP_Close(vfc);

	if (vfps == VFP_ERROR) {
		AN(vfc->failed);
		(void)VFP_Error(vfc, "Fetch pipeline failed to process");
		bo->htc->doclose = SC_RX_BODY;
	}

	if (!bo->do_stream)
		ObjTrimStore(bo->wrk, vfc->oc);
}
Esempio n. 14
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	ssize_t l, ll, o;
	double now;
	uint8_t *ptr;
	char time_str[VTIM_FORMAT_SIZE];

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
	assert(bo->director_state == DIR_S_NULL);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	AZ(bo->synth_body);
	bo->synth_body = VSB_new_auto();
	AN(bo->synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	VTIM_format(now, time_str);
	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->exp.t_origin = bo->t_prev;
	bo->fetch_objcore->exp.ttl = 0;
	bo->fetch_objcore->exp.grace = 0;
	bo->fetch_objcore->exp.keep = 0;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);

	AZ(VSB_finish(bo->synth_body));

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_delete(bo->synth_body);
		bo->synth_body = NULL;
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	VFP_Setup(bo->vfc);
	bo->vfc->bo = bo;
	bo->vfc->wrk = bo->wrk;
	bo->vfc->oc = bo->fetch_objcore;
	bo->vfc->http = bo->beresp;
	bo->vfc->esi_req = bo->bereq;

	if (vbf_beresp2obj(bo))
		return (F_STP_FAIL);

	ll = VSB_len(bo->synth_body);
	o = 0;
	while (ll > 0) {
		l = ll;
		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
			break;
		memcpy(ptr, VSB_data(bo->synth_body) + o, l);
		VBO_extend(bo, l);
		ll -= l;
		o += l;
	}
	VSB_delete(bo->synth_body);
	bo->synth_body = NULL;

	HSH_Unbusy(wrk, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Esempio n. 15
0
void
VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
{
	ssize_t l;
	enum vfp_status vfps = VFP_ERROR;
	struct storage *st = NULL;

	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	AN(bo->vfp_nxt);

	if (est < 0)
		est = 0;

	if (vfp_suck_init(bo) != VFP_OK) {
		(void)VFP_Error(bo, "Fetch Pipeline failed to initialize");
		bo->should_close = 1;
		return;
	}

	do {
		if (bo->abandon) {
			/*
			 * A pass object and delivery was terminted
			 * We don't fail the fetch, in order for hit-for-pass
			 * objects to be created.
			 */
			AN(bo->fetch_objcore->flags & OC_F_PASS);
			VSLb(bo->vsl, SLT_FetchError,
			    "Pass delivery abandoned");
			vfps = VFP_END;
			bo->should_close = 1;
			break;
		}
		AZ(bo->failed);
		if (st == NULL) {
			st = VFP_GetStorage(bo, est);
			est = 0;
		}
		if (st == NULL) {
			bo->should_close = 1;
			(void)VFP_Error(bo, "Out of storage");
			break;
		}

		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
		assert(st == VTAILQ_LAST(&bo->fetch_obj->store, storagehead));
		l = st->space - st->len;
		AZ(bo->failed);
		vfps = VFP_Suck(bo, st->ptr + st->len, &l);
		if (l > 0 && vfps != VFP_ERROR) {
			AZ(VTAILQ_EMPTY(&bo->fetch_obj->store));
			VBO_extend(bo, l);
		}
		if (st->len == st->space)
			st = NULL;
	} while (vfps == VFP_OK);

	if (vfps == VFP_ERROR) {
		AN(bo->failed);
		(void)VFP_Error(bo, "Fetch Pipeline failed to process");
		bo->should_close = 1;
	}

	vfp_suck_fini(bo);

	/*
	 * Trim or delete the last segment, if any
	 */

	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
	/* XXX: Temporary:  Only trim if we are not streaming */
	if (st != NULL && !bo->do_stream) {
		/* None of this this is safe under streaming */
		if (st->len == 0) {
			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
			STV_free(st);
		} else if (st->len < st->space) {
			STV_trim(st, st->len, 1);
		}
	}
}
Esempio n. 16
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	struct object *obj;
	struct objiter *oi;
	void *sp;
	ssize_t sl, al, tl;
	struct storage *st;
	enum objiter_status ois;
	char *p;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	if (bo->ims_obj->changed_gzip) {
		/*
		 * If we modified the gzip status of the IMS object, that
		 * must control the C-E header, if any.
		 */
		http_Unset(bo->beresp, H_Content_Encoding);
		if (http_GetHdr(bo->ims_obj->http, H_Content_Encoding, &p))
			http_PrintfHeader(bo->beresp,
			    "Content-Encoding: %s", p);
	}

	AZ(vbf_beresp2obj(bo));
	obj = bo->fetch_obj;
	bo->vfc->body = obj->body;

	if (bo->ims_obj->esidata != NULL) {
		sl = bo->ims_obj->esidata->len;
		obj->esidata = STV_alloc(bo->vfc, sl);
		if (obj->esidata == NULL || obj->esidata->space < sl) {
			VSLb(bo->vsl, SLT_Error,
			    "No space for %zd bytes of ESI data", sl);
			return (F_STP_FAIL);
		}
		memcpy(obj->esidata->ptr, bo->ims_obj->esidata->ptr, sl);
		obj->esidata->len = sl;
	}

	obj->gziped = bo->ims_obj->gziped;
	obj->gzip_start = bo->ims_obj->gzip_start;
	obj->gzip_last = bo->ims_obj->gzip_last;
	obj->gzip_stop = bo->ims_obj->gzip_stop;

	AZ(WS_Overflowed(bo->ws_o));
	if (bo->do_stream) {
		HSH_Unbusy(&wrk->stats, obj->objcore);
		VBO_setstate(bo, BOS_STREAM);
	}

	st = NULL;
	al = 0;

	oi = ObjIterBegin(wrk, bo->ims_obj);
	do {
		ois = ObjIter(oi, &sp, &sl);
		while (sl > 0) {
			if (st == NULL)
				st = VFP_GetStorage(bo->vfc,
				    bo->ims_obj->len - al);
			if (st == NULL)
				break;
			tl = sl;
			if (tl > st->space - st->len)
				tl = st->space - st->len;
			memcpy(st->ptr + st->len, sp, tl);
			al += tl;
			sp = (char *)sp + tl;
			sl -= tl;
			VBO_extend(bo, tl);
			if (st->len == st->space)
				st = NULL;
		}
	} while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM));
	ObjIterEnd(&oi);
	if (bo->vfc->failed)
		return (F_STP_FAIL);

	if (!bo->do_stream)
		HSH_Unbusy(&wrk->stats, obj->objcore);

	assert(al == bo->ims_obj->len);
	assert(obj->len == al);
	EXP_Rearm(bo->ims_obj->objcore,
	    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);

	/* Recycle the backend connection before setting BOS_FINISHED to
	   give predictable backend reuse behavior for varnishtest */
	if (bo->vbc != NULL && bo->doclose == SC_NULL) {
		VDI_RecycleFd(&bo->vbc, &bo->acct);
		AZ(bo->vbc);
	}

	VBO_setstate(bo, BOS_FINISHED);
	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
	return (F_STP_DONE);
}
Esempio n. 17
0
static enum fetch_step
vbf_stp_error(struct worker *wrk, struct busyobj *bo)
{
	struct storage *st;
	ssize_t l;
	double now;
	char time_str[VTIM_FORMAT_SIZE];

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	now = W_TIM_real(wrk);
	VSLb_ts_busyobj(bo, "Error", now);

	AN(bo->fetch_objcore->flags & OC_F_BUSY);

	AZ(bo->synth_body);
	bo->synth_body = VSB_new_auto();
	AN(bo->synth_body);

	// XXX: reset all beresp flags ?

	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
	VTIM_format(now, time_str);
	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
	http_SetHeader(bo->beresp, "Server: Varnish");

	bo->fetch_objcore->exp.t_origin = bo->t_prev;
	bo->fetch_objcore->exp.ttl = 0;
	bo->fetch_objcore->exp.grace = 0;
	bo->fetch_objcore->exp.keep = 0;

	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);

	AZ(VSB_finish(bo->synth_body));

	if (wrk->handling == VCL_RET_RETRY) {
		VSB_delete(bo->synth_body);
		bo->synth_body = NULL;
		if (bo->retries++ < cache_param->max_retries)
			return (F_STP_RETRY);
		bo->synth_body = NULL;
		return (F_STP_FAIL);
	}

	assert(wrk->handling == VCL_RET_DELIVER);

	VFP_Setup(bo->vfc);
	bo->vfc->bo = bo;
	bo->vfc->http = bo->beresp;
	bo->vfc->vsl = bo->vsl;

	if (vbf_beresp2obj(bo))
		return (F_STP_FAIL);

	bo->vfc->body = bo->fetch_obj->body;

	l = VSB_len(bo->synth_body);
	if (l > 0) {
		st = VFP_GetStorage(bo->vfc, l);
		if (st != NULL) {
			if (st->space < l) {
				VSLb(bo->vsl, SLT_Error,
				    "No space for %zd bytes of synth body", l);
			} else {
				memcpy(st->ptr, VSB_data(bo->synth_body), l);
				VBO_extend(bo, l);
			}
		}
	}
	VSB_delete(bo->synth_body);
	bo->synth_body = NULL;

	HSH_Unbusy(&wrk->stats, bo->fetch_objcore);
	VBO_setstate(bo, BOS_FINISHED);
	return (F_STP_DONE);
}
Esempio n. 18
0
static enum fetch_step
vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
{
	unsigned l;
	uint16_t nhttp;
	struct object *obj;
	struct objiter *oi;
	void *sp;
	ssize_t sl, al, tl, vl;
	struct storage *st;
	enum objiter_status ois;

	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);

	l = 0;
	if (bo->ims_obj->vary != NULL) {
		vl = VRY_Validate(bo->ims_obj->vary);
		l += vl;
	} else
		vl = 0;
	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);

	bo->stats = &wrk->stats;
	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
	if (obj == NULL) {
		(void)VFP_Error(bo, "Could not get storage");
		VDI_CloseFd(&bo->vbc);
		return (F_STP_DONE);
	}

	AZ(bo->fetch_obj);
	bo->fetch_obj = obj;

	obj->gziped = bo->ims_obj->gziped;
	obj->gzip_start = bo->ims_obj->gzip_start;
	obj->gzip_last = bo->ims_obj->gzip_last;
	obj->gzip_stop = bo->ims_obj->gzip_stop;

	/* XXX: ESI */

	if (bo->ims_obj->vary != NULL) {
		obj->vary = (void *)WS_Copy(obj->http->ws,
		    bo->ims_obj->vary, vl);
		assert(vl == VRY_Validate(obj->vary));
	}

	obj->vxid = bo->vsl->wid;

	obj->http->logtag = HTTP_Obj;
	/* XXX: we should have our own HTTP_A_CONDFETCH */
	http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
	http_CopyHome(obj->http);


	AZ(WS_Overflowed(bo->ws_o));
	VBO_setstate(bo, BOS_FETCHING);
	HSH_Unbusy(&wrk->stats, obj->objcore);

	if (!(obj->objcore->flags & OC_F_PRIVATE)) {
		EXP_Insert(obj->objcore);
		AN(obj->objcore->ban);
	}

	st = NULL;
	al = 0;

	oi = ObjIterBegin(wrk, bo->ims_obj);
	do {
		ois = ObjIter(oi, &sp, &sl);
		while (sl > 0) {
			if (st == NULL) {
				st = VFP_GetStorage(bo, bo->ims_obj->len - al);
				XXXAN(st);
			}
			tl = sl;
			if (tl > st->space - st->len)
				tl = st->space - st->len;
			memcpy(st->ptr + st->len, sp, tl);
			al += tl;
			sp = (char *)sp + tl;
			sl -= tl;
			VBO_extend(bo, al);
			if (st->len == st->space)
				st = NULL;
		}
	} while (ois == OIS_DATA || ois == OIS_STREAM);
	ObjIterEnd(&oi);
	bo->stats = NULL;
	assert(al == bo->ims_obj->len);
	assert(obj->len == al);
	if (bo->state != BOS_FAILED)
		VBO_setstate(bo, BOS_FINISHED);
	HSH_Complete(obj->objcore);
	return (F_STP_DONE);
}