static void
gen6_wa_pre_depth(struct ilo_render *r)
{
ILO_DEV_ASSERT(r->dev, 6, 6);
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 315:
*
* "Restriction: Prior to changing Depth/Stencil Buffer state (i.e.,
* any combination of 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS,
* 3DSTATE_STENCIL_BUFFER, 3DSTATE_HIER_DEPTH_BUFFER) SW must first
* issue a pipelined depth stall (PIPE_CONTROL with Depth Stall bit
* set), followed by a pipelined depth cache flush (PIPE_CONTROL with
* Depth Flush Bit set, followed by another pipelined depth stall
* (PIPE_CONTROL with Depth Stall Bit set), unless SW can otherwise
* guarantee that the pipeline from WM onwards is already flushed
* (e.g., via a preceding MI_FLUSH)."
*
* According to the classic driver, it also applies for GEN6.
*/
gen6_wa_pre_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
}
/**
* This should be called before PIPE_CONTROL.
*/
void
gen6_wa_pre_pipe_control(struct ilo_render *r, uint32_t dw1)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 60:
*
* "Pipe-control with CS-stall bit set must be sent BEFORE the
* pipe-control with a post-sync op and no write-cache flushes."
*
* This WA may also be triggered indirectly by the other two WAs on the
* same page:
*
* "Before any depth stall flush (including those produced by
* non-pipelined state commands), software needs to first send a
* PIPE_CONTROL with no bits set except Post-Sync Operation != 0."
*
* "Before a PIPE_CONTROL with Write Cache Flush Enable =1, a
* PIPE_CONTROL with any non-zero post-sync-op is required."
*/
const bool direct_wa_cond = (dw1 & GEN6_PIPE_CONTROL_WRITE__MASK) &&
!(dw1 & GEN6_PIPE_CONTROL_RENDER_CACHE_FLUSH);
const bool indirect_wa_cond = (dw1 & GEN6_PIPE_CONTROL_DEPTH_STALL) |
(dw1 & GEN6_PIPE_CONTROL_RENDER_CACHE_FLUSH);
ILO_DEV_ASSERT(r->dev, 6, 6);
if (!direct_wa_cond && !indirect_wa_cond)
return;
if (!(r->state.current_pipe_control_dw1 & GEN6_PIPE_CONTROL_CS_STALL)) {
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 73:
*
* "1 of the following must also be set (when CS stall is set):
*
* - Depth Cache Flush Enable ([0] of DW1)
* - Stall at Pixel Scoreboard ([1] of DW1)
* - Depth Stall ([13] of DW1)
* - Post-Sync Operation ([13] of DW1)
* - Render Target Cache Flush Enable ([12] of DW1)
* - Notify Enable ([8] of DW1)"
*
* Because of the WAs above, we have to pick Stall at Pixel Scoreboard.
*/
const uint32_t direct_wa = GEN6_PIPE_CONTROL_CS_STALL |
GEN6_PIPE_CONTROL_PIXEL_SCOREBOARD_STALL;
ilo_render_pipe_control(r, direct_wa);
}
if (indirect_wa_cond &&
!(r->state.current_pipe_control_dw1 & GEN6_PIPE_CONTROL_WRITE__MASK)) {
const uint32_t indirect_wa = GEN6_PIPE_CONTROL_WRITE_IMM;
ilo_render_pipe_control(r, indirect_wa);
}
}
void
ilo_render_emit_rectlist_commands_gen8(struct ilo_render *r,
const struct ilo_blitter *blitter,
const struct ilo_render_rectlist_session *session)
{
uint32_t op;
ILO_DEV_ASSERT(r->dev, 8, 8);
gen8_wa_pre_depth(r);
if (blitter->uses & (ILO_BLITTER_USE_FB_DEPTH |
ILO_BLITTER_USE_FB_STENCIL)) {
gen6_3DSTATE_DEPTH_BUFFER(r->builder,
&blitter->fb.dst.u.zs, true);
}
if (blitter->uses & ILO_BLITTER_USE_FB_DEPTH) {
gen6_3DSTATE_HIER_DEPTH_BUFFER(r->builder,
&blitter->fb.dst.u.zs);
}
if (blitter->uses & ILO_BLITTER_USE_FB_STENCIL) {
gen6_3DSTATE_STENCIL_BUFFER(r->builder,
&blitter->fb.dst.u.zs);
}
gen7_3DSTATE_CLEAR_PARAMS(r->builder,
blitter->depth_clear_value);
gen6_3DSTATE_DRAWING_RECTANGLE(r->builder, 0, 0,
blitter->fb.width, blitter->fb.height);
switch (blitter->op) {
case ILO_BLITTER_RECTLIST_CLEAR_ZS:
op = 0;
if (blitter->uses & ILO_BLITTER_USE_FB_DEPTH)
op |= GEN8_WM_HZ_DW1_DEPTH_CLEAR;
if (blitter->uses & ILO_BLITTER_USE_FB_STENCIL)
op |= GEN8_WM_HZ_DW1_STENCIL_CLEAR;
break;
case ILO_BLITTER_RECTLIST_RESOLVE_Z:
op = GEN8_WM_HZ_DW1_DEPTH_RESOLVE;
break;
case ILO_BLITTER_RECTLIST_RESOLVE_HIZ:
op = GEN8_WM_HZ_DW1_HIZ_RESOLVE;
break;
default:
op = 0;
break;
}
gen8_3DSTATE_WM_HZ_OP(r->builder, op, blitter->fb.width,
blitter->fb.height, blitter->fb.num_samples);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_WRITE_IMM);
gen8_disable_3DSTATE_WM_HZ_OP(r->builder);
}
static void
gen7_wa_pre_depth(struct ilo_render *r)
{
ILO_DEV_ASSERT(r->dev, 7, 7.5);
if (ilo_dev_gen(r->dev) == ILO_GEN(7)) {
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 315:
*
* "Driver must send a least one PIPE_CONTROL command with CS Stall
* and a post sync operation prior to the group of depth
* commands(3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS,
* 3DSTATE_STENCIL_BUFFER, and 3DSTATE_HIER_DEPTH_BUFFER)."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_CS_STALL |
GEN6_PIPE_CONTROL_WRITE_IMM;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 315:
*
* "Restriction: Prior to changing Depth/Stencil Buffer state (i.e.,
* any combination of 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS,
* 3DSTATE_STENCIL_BUFFER, 3DSTATE_HIER_DEPTH_BUFFER) SW must first
* issue a pipelined depth stall (PIPE_CONTROL with Depth Stall bit
* set), followed by a pipelined depth cache flush (PIPE_CONTROL with
* Depth Flush Bit set, followed by another pipelined depth stall
* (PIPE_CONTROL with Depth Stall Bit set), unless SW can otherwise
* guarantee that the pipeline from WM onwards is already flushed
* (e.g., via a preceding MI_FLUSH)."
*/
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_DEPTH_STALL);
}
static void
gen6_wa_post_3dstate_constant_vs(struct ilo_render *r)
{
/*
* According to upload_vs_state() of the classic driver, we need to emit a
* PIPE_CONTROL after 3DSTATE_CONSTANT_VS, otherwise the command is kept
* being buffered by VS FF, to the point that the FF dies.
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_STATE_CACHE_INVALIDATE;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen7_wa_pre_3dstate_sf_depth_bias(struct ilo_render *r)
{
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 258:
*
* "Due to an HW issue driver needs to send a pipe control with stall
* when ever there is state change in depth bias related state (in
* 3DSTATE_SF)"
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_CS_STALL;
ILO_DEV_ASSERT(r->dev, 7, 7);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen7_wa_pre_3dstate_ps_max_threads(struct ilo_render *r)
{
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 286:
*
* "If this field (Maximum Number of Threads in 3DSTATE_PS) is changed
* between 3DPRIMITIVE commands, a PIPE_CONTROL command with Stall at
* Pixel Scoreboard set is required to be issued."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_PIXEL_SCOREBOARD_STALL;
ILO_DEV_ASSERT(r->dev, 7, 7.5);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen6_wa_pre_3dstate_wm_max_threads(struct ilo_render *r)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 274:
*
* "A PIPE_CONTROL command, with only the Stall At Pixel Scoreboard
* field set (DW1 Bit 1), must be issued prior to any change to the
* value in this field (Maximum Number of Threads in 3DSTATE_WM)"
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_PIXEL_SCOREBOARD_STALL;
ILO_DEV_ASSERT(r->dev, 6, 6);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen6_wa_post_3dstate_urb_no_gs(struct ilo_render *r)
{
/*
* From the Sandy Bridge PRM, volume 2 part 1, page 27:
*
* "Because of a urb corruption caused by allocating a previous
* gsunit's urb entry to vsunit software is required to send a
* "GS NULL Fence" (Send URB fence with VS URB size == 1 and GS URB
* size == 0) plus a dummy DRAW call before any case where VS will
* be taking over GS URB space."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_CS_STALL;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen7_wa_pre_3dstate_multisample(struct ilo_render *r)
{
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 304:
*
* "Driver must ierarchi that all the caches in the depth pipe are
* flushed before this command (3DSTATE_MULTISAMPLE) is parsed. This
* requires driver to send a PIPE_CONTROL with a CS stall along with a
* Depth Flush prior to this command.
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH |
GEN6_PIPE_CONTROL_CS_STALL;
ILO_DEV_ASSERT(r->dev, 7, 7.5);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
static void
gen7_wa_pre_vs(struct ilo_render *r)
{
/*
* From the Ivy Bridge PRM, volume 2 part 1, page 106:
*
* "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth stall
* needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
* 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
* 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL
* needs to be sent before any combination of VS associated 3DSTATE."
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_DEPTH_STALL |
GEN6_PIPE_CONTROL_WRITE_IMM;
ILO_DEV_ASSERT(r->dev, 7, 7);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
/**
* Emit PIPE_CONTROLs to flush all caches.
*/
void
ilo_render_emit_flush(struct ilo_render *render)
{
const uint32_t dw1 = GEN6_PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_RENDER_CACHE_FLUSH |
GEN6_PIPE_CONTROL_DEPTH_CACHE_FLUSH |
GEN6_PIPE_CONTROL_VF_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
GEN6_PIPE_CONTROL_CS_STALL;
const unsigned batch_used = ilo_builder_batch_used(render->builder);
ILO_DEV_ASSERT(render->dev, 6, 8);
if (ilo_dev_gen(render->dev) == ILO_GEN(6))
gen6_wa_pre_pipe_control(render, dw1);
ilo_render_pipe_control(render, dw1);
assert(ilo_builder_batch_used(render->builder) <= batch_used +
ilo_render_get_flush_len(render));
}
static void
gen6_wa_pre_3dstate_vs_toggle(struct ilo_render *r)
{
/*
* The classic driver has this undocumented WA:
*
* From the BSpec, 3D Pipeline > Geometry > Vertex Shader > State,
* 3DSTATE_VS, Dword 5.0 "VS Function Enable":
*
* [DevSNB] A pipeline flush must be programmed prior to a 3DSTATE_VS
* command that causes the VS Function Enable to toggle. Pipeline
* flush can be executed by sending a PIPE_CONTROL command with CS
* stall bit set and a post sync operation.
*/
const uint32_t dw1 = GEN6_PIPE_CONTROL_WRITE_IMM |
GEN6_PIPE_CONTROL_CS_STALL;
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
gen6_wa_pre_pipe_control(r, dw1);
if ((r->state.current_pipe_control_dw1 & dw1) != dw1)
ilo_render_pipe_control(r, dw1);
}
void
ilo_render_emit_rectlist_commands_gen8(struct ilo_render *r,
const struct ilo_blitter *blitter,
const struct ilo_render_rectlist_session *session)
{
ILO_DEV_ASSERT(r->dev, 8, 8);
gen8_wa_pre_depth(r);
if (blitter->uses & (ILO_BLITTER_USE_FB_DEPTH |
ILO_BLITTER_USE_FB_STENCIL))
gen6_3DSTATE_DEPTH_BUFFER(r->builder, &blitter->fb.dst.u.zs);
if (blitter->uses & ILO_BLITTER_USE_FB_DEPTH) {
gen6_3DSTATE_HIER_DEPTH_BUFFER(r->builder,
&blitter->fb.dst.u.zs);
}
if (blitter->uses & ILO_BLITTER_USE_FB_STENCIL) {
gen6_3DSTATE_STENCIL_BUFFER(r->builder,
&blitter->fb.dst.u.zs);
}
gen7_3DSTATE_CLEAR_PARAMS(r->builder,
blitter->depth_clear_value);
gen6_3DSTATE_DRAWING_RECTANGLE(r->builder, 0, 0,
blitter->fb.width, blitter->fb.height);
gen8_3DSTATE_WM_HZ_OP(r->builder, &blitter->fb.rs,
blitter->fb.width, blitter->fb.height);
ilo_render_pipe_control(r, GEN6_PIPE_CONTROL_WRITE_IMM);
gen8_disable_3DSTATE_WM_HZ_OP(r->builder);
}