static void
gen6_wa_pre_depth(struct ilo_render *r)
{
ILO_DEV_ASSERT(r->dev, 6, 6);
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 315:
*
* "Restriction: Prior to changing Depth/Stencil Buffer state (i.e.,
* any combination of 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS,
* 3DSTATE_STENCIL_BUFFER, 3DSTATE_HIER_DEPTH_BUFFER) SW must first
* issue a pipelined depth stall (PIPE_CONTROL with Depth Stall bit
* set), followed by a pipelined depth cache flush (PIPE_CONTROL with
* Depth Flush Bit set, followed by another pipelined depth stall
* (PIPE_CONTROL with Depth Stall Bit set), unless SW can otherwise
* guarantee that the pipeline from WM onwards is already flushed
* (e.g., via a preceding MI_FLUSH)."
*
* According to the classic driver, it also applies for GEN6.
*/
gen6_wa_pre_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH);
gen6_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
gen6_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH);
gen6_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
}
/**
* This should be called before any non-pipelined state command.
*/
static void
gen6_wa_pre_non_pipelined(struct ilo_render *r)
{
ILO_DEV_ASSERT(r->dev, 6, 6);
/* non-pipelined state commands produce depth stall */
gen6_wa_pre_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
}
static void
gen6_wa_post_3dstate_constant_vs(struct ilo_render *r)
{
/*
* According to upload_vs_state() of the classic driver, we need to emit a
* PIPE_CONTROL after 3DSTATE_CONSTANT_VS, otherwise the command is kept
* being buffered by VS FF, to the point that the FF dies.
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_STATE_CACHE_INVALIDATE;
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_pipe_control(r, dw1);
}
static void
gen6_wa_pre_3dstate_wm_max_threads(struct ilo_render *r)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 274:
*
* "A PIPE_CONTROL command, with only the Stall At Pixel Scoreboard
* field set (DW1 Bit 1), must be issued prior to any change to the
* value in this field (Maximum Number of Threads in 3DSTATE_WM)"
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_PIXEL_SCOREBOARD_STALL;
ILO_DEV_ASSERT(r->dev, 6, 6);
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_pipe_control(r, dw1);
}
static void
gen6_wa_post_3dstate_urb_no_gs(struct ilo_render *r)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 27:
*
* "Because of a urb corruption caused by allocating a previous
* gsunit's urb entry to vsunit software is required to send a
* "GS NULL Fence" (Send URB fence with VS URB size == 1 and GS URB
* size == 0) plus a dummy DRAW call before any case where VS will
* be taking over GS URB space."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_CS_STALL;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen6_wa_pre_3dstate_multisample(struct ilo_render *r)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 305:
*
* "Driver must guarentee that all the caches in the depth pipe are
* flushed before this command (3DSTATE_MULTISAMPLE) is parsed. This
* requires driver to send a PIPE_CONTROL with a CS stall along with a
* Depth Flush prior to this command."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH |
GEN6_PIPE_CONTROL_CS_STALL;
ILO_DEV_ASSERT(r->dev, 6, 6);
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_pipe_control(r, dw1);
}
/**
* Emit PIPE_CONTROLs to flush all caches.
*/
void
ilo_render_emit_flush(struct ilo_render *render)
{
const uint32_t dw1 = GEN6_PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_RENDER_CACHE_FLUSH |
GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH |
GEN6_PIPE_CONTROL_VF_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_CS_STALL;
const unsigned batch_used = ilo_builder_batch_used(render->builder);
ILO_DEV_ASSERT(render->dev, 6, 8);
if (ilo_dev_gen(render->dev) == ILO_GEN(6))
gen6_wa_pre_pipe_control(render, dw1);
ilo_render_pipe_control(render, dw1);
assert(ilo_builder_batch_used(render->builder) <= batch_used +
ilo_render_get_flush_len(render));
}
static void
gen6_wa_pre_3dstate_vs_toggle(struct ilo_render *r)
{
/*
* The classic driver has this undocumented WA:
*
* From the BSpec, 3D Pipeline > Geometry > Vertex Shader > State,
* 3DSTATE_VS, Dword 5.0 "VS Function Enable":
*
* [DevSNB] A pipeline flush must be programmed prior to a 3DSTATE_VS
* command that causes the VS Function Enable to toggle. Pipeline
* flush can be executed by sending a PIPE_CONTROL command with CS
* stall bit set and a post sync operation.
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_WRITE_IMM |
GEN6_PIPE_CONTROL_CS_STALL;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
/**
* Emit PIPE_CONTROLs or MI_STORE_REGISTER_MEMs to store register values.
*/
void
ilo_render_emit_query(struct ilo_render *render,
struct ilo_query *q, uint32_t offset)
{
const uint32_t pipeline_statistics_regs[11] = {
GEN6_REG_IA_VERTICES_COUNT,
GEN6_REG_IA_PRIMITIVES_COUNT,
GEN6_REG_VS_INVOCATION_COUNT,
GEN6_REG_GS_INVOCATION_COUNT,
GEN6_REG_GS_PRIMITIVES_COUNT,
GEN6_REG_CL_INVOCATION_COUNT,
GEN6_REG_CL_PRIMITIVES_COUNT,
GEN6_REG_PS_INVOCATION_COUNT,
(ilo_dev_gen(render->dev) >= ILO_GEN(7)) ?
GEN7_REG_HS_INVOCATION_COUNT : 0,
(ilo_dev_gen(render->dev) >= ILO_GEN(7)) ?
GEN7_REG_DS_INVOCATION_COUNT : 0,
0,
};
const uint32_t primitives_generated_reg =
(ilo_dev_gen(render->dev) >= ILO_GEN(7) && q->index > 0) ?
GEN7_REG_SO_PRIM_STORAGE_NEEDED(q->index) :
GEN6_REG_CL_INVOCATION_COUNT;
const uint32_t primitives_emitted_reg =
(ilo_dev_gen(render->dev) >= ILO_GEN(7)) ?
GEN7_REG_SO_NUM_PRIMS_WRITTEN(q->index) :
GEN6_REG_SO_NUM_PRIMS_WRITTEN;
const unsigned batch_used = ilo_builder_batch_used(render->builder);
const uint32_t *regs;
int reg_count = 0, i;
uint32_t pipe_control_dw1 = 0;
ILO_DEV_ASSERT(render->dev, 6, 8);
switch (q->type) {
case PIPE_QUERY_OCCLUSION_COUNTER:
case PIPE_QUERY_OCCLUSION_PREDICATE:
pipe_control_dw1 = GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_WRITE_PS_DEPTH_COUNT;
break;
case PIPE_QUERY_TIMESTAMP:
case PIPE_QUERY_TIME_ELAPSED:
pipe_control_dw1 = GEN6_PIPE_CONTROL_WRITE_TIMESTAMP;
break;
case PIPE_QUERY_PRIMITIVES_GENERATED:
regs = &primitives_generated_reg;
reg_count = 1;
break;
case PIPE_QUERY_PRIMITIVES_EMITTED:
regs = &primitives_emitted_reg;
reg_count = 1;
break;
case PIPE_QUERY_PIPELINE_STATISTICS:
regs = pipeline_statistics_regs;
reg_count = ARRAY_SIZE(pipeline_statistics_regs);
break;
default:
break;
}
if (pipe_control_dw1) {
assert(!reg_count);
if (ilo_dev_gen(render->dev) == ILO_GEN(6))
gen6_wa_pre_pipe_control(render, pipe_control_dw1);
gen6_PIPE_CONTROL(render->builder, pipe_control_dw1, q->bo, offset, 0);
render->state.current_pipe_control_dw1 |= pipe_control_dw1;
render->state.deferred_pipe_control_dw1 &= ~pipe_control_dw1;
} else if (reg_count) {
ilo_render_emit_flush(render);
}
for (i = 0; i < reg_count; i++) {
if (regs[i]) {
/* store lower 32 bits */
gen6_MI_STORE_REGISTER_MEM(render->builder, regs[i], q->bo, offset);
/* store higher 32 bits */
gen6_MI_STORE_REGISTER_MEM(render->builder, regs[i] + 4,
q->bo, offset + 4);
} else {
gen6_MI_STORE_DATA_IMM(render->builder, q->bo, offset, 0);
}
offset += 8;
}
assert(ilo_builder_batch_used(render->builder) <= batch_used +
ilo_render_get_query_len(render, q->type));
}