static bool
is_phi_src_scalarizable(nir_phi_src *src,
struct lower_phis_to_scalar_state *state)
{
/* Don't know what to do with non-ssa sources */
if (!src->src.is_ssa)
return false;
nir_instr *src_instr = src->src.ssa->parent_instr;
switch (src_instr->type) {
case nir_instr_type_alu: {
nir_alu_instr *src_alu = nir_instr_as_alu(src_instr);
/* ALU operations with output_size == 0 should be scalarized. We
* will also see a bunch of vecN operations from scalarizing ALU
* operations and, since they can easily be copy-propagated, they
* are ok too.
*/
return nir_op_infos[src_alu->op].output_size == 0 ||
src_alu->op == nir_op_vec2 ||
src_alu->op == nir_op_vec3 ||
src_alu->op == nir_op_vec4;
}
case nir_instr_type_phi:
/* A phi is scalarizable if we're going to lower it */
return should_lower_phi(nir_instr_as_phi(src_instr), state);
case nir_instr_type_load_const:
/* These are trivially scalarizable */
return true;
case nir_instr_type_intrinsic: {
nir_intrinsic_instr *src_intrin = nir_instr_as_intrinsic(src_instr);
switch (src_intrin->intrinsic) {
case nir_intrinsic_load_var:
return src_intrin->variables[0]->var->data.mode == nir_var_shader_in ||
src_intrin->variables[0]->var->data.mode == nir_var_uniform;
case nir_intrinsic_interp_var_at_centroid:
case nir_intrinsic_interp_var_at_sample:
case nir_intrinsic_interp_var_at_offset:
case nir_intrinsic_load_uniform:
case nir_intrinsic_load_uniform_indirect:
case nir_intrinsic_load_ubo:
case nir_intrinsic_load_ubo_indirect:
case nir_intrinsic_load_input:
case nir_intrinsic_load_input_indirect:
return true;
default:
break;
}
}
default:
/* We can't scalarize this type of instruction */
return false;
}
}
static bool
remove_phis_block(nir_block *block, void *state)
{
bool *progress = state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_ssa_def *def = NULL;
bool srcs_same = true;
nir_foreach_phi_src(phi, src) {
assert(src->src.is_ssa);
/* For phi nodes at the beginning of loops, we may encounter some
* sources from backedges that point back to the destination of the
* same phi, i.e. something like:
*
* a = phi(a, b, ...)
*
* We can safely ignore these sources, since if all of the normal
* sources point to the same definition, then that definition must
* still dominate the phi node, and the phi will still always take
* the value of that definition.
*/
if (src->src.ssa == &phi->dest.ssa)
continue;
if (def == NULL) {
def = src->src.ssa;
} else {
if (src->src.ssa != def) {
srcs_same = false;
break;
}
}
}
if (!srcs_same)
continue;
/* We must have found at least one definition, since there must be at
* least one forward edge.
*/
assert(def != NULL);
assert(phi->dest.is_ssa);
nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
nir_instr_remove(instr);
*progress = true;
}
static void
opt_constant_if(nir_if *if_stmt, bool condition)
{
/* First, we need to remove any phi nodes after the if by rewriting uses to
* point to the correct source.
*/
nir_block *after = nir_cf_node_as_block(nir_cf_node_next(&if_stmt->cf_node));
nir_block *last_block =
nir_cf_node_as_block(condition ? nir_if_last_then_node(if_stmt)
: nir_if_last_else_node(if_stmt));
nir_foreach_instr_safe(after, instr) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_ssa_def *def = NULL;
nir_foreach_phi_src(phi, phi_src) {
if (phi_src->pred != last_block)
continue;
assert(phi_src->src.is_ssa);
def = phi_src->src.ssa;
}
assert(def);
assert(phi->dest.is_ssa);
nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
nir_instr_remove(instr);
}
/* The control flow list we're about to paste in may include a jump at the
* end, and in that case we have to delete the rest of the control flow
* list after the if since it's unreachable and the validator will balk if
* we don't.
*/
if (!exec_list_is_empty(&last_block->instr_list)) {
nir_instr *last_instr = nir_block_last_instr(last_block);
if (last_instr->type == nir_instr_type_jump)
remove_after_cf_node(&if_stmt->cf_node);
}
/* Finally, actually paste in the then or else branch and delete the if. */
struct exec_list *cf_list = condition ? &if_stmt->then_list
: &if_stmt->else_list;
nir_cf_list list;
nir_cf_extract(&list, nir_before_cf_list(cf_list),
nir_after_cf_list(cf_list));
nir_cf_reinsert(&list, nir_after_cf_node(&if_stmt->cf_node));
nir_cf_node_remove(&if_stmt->cf_node);
}
/** Propagates the live in of succ across the edge to the live out of pred
*
* Phi nodes exist "between" blocks and all the phi nodes at the start of a
* block act "in parallel". When we propagate from the live_in of one
* block to the live out of the other, we have to kill any writes from phis
* and make live any sources.
*
* Returns true if updating live out of pred added anything
*/
static bool
propagate_across_edge(nir_block *pred, nir_block *succ,
struct live_ssa_defs_state *state)
{
NIR_VLA(BITSET_WORD, live, state->bitset_words);
memcpy(live, succ->live_in, state->bitset_words * sizeof *live);
nir_foreach_instr(instr, succ) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
assert(phi->dest.is_ssa);
set_ssa_def_dead(&phi->dest.ssa, live);
}
nir_foreach_instr(instr, succ) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_foreach_phi_src(src, phi) {
if (src->pred == pred) {
set_src_live(&src->src, live);
break;
}
}
}
BITSET_WORD progress = 0;
for (unsigned i = 0; i < state->bitset_words; ++i) {
progress |= live[i] & ~pred->live_out[i];
pred->live_out[i] |= live[i];
}
return progress != 0;
}
static void
validate_instr(nir_instr *instr, validate_state *state)
{
assert(instr->block == state->block);
state->instr = instr;
switch (instr->type) {
case nir_instr_type_alu:
validate_alu_instr(nir_instr_as_alu(instr), state);
break;
case nir_instr_type_call:
validate_call_instr(nir_instr_as_call(instr), state);
break;
case nir_instr_type_intrinsic:
validate_intrinsic_instr(nir_instr_as_intrinsic(instr), state);
break;
case nir_instr_type_tex:
validate_tex_instr(nir_instr_as_tex(instr), state);
break;
case nir_instr_type_load_const:
validate_load_const_instr(nir_instr_as_load_const(instr), state);
break;
case nir_instr_type_phi:
validate_phi_instr(nir_instr_as_phi(instr), state);
break;
case nir_instr_type_ssa_undef:
validate_ssa_undef_instr(nir_instr_as_ssa_undef(instr), state);
break;
case nir_instr_type_jump:
break;
default:
assert(!"Invalid ALU instruction type");
break;
}
state->instr = NULL;
}
static void
propagate_invariant_instr(nir_instr *instr, struct set *invariants)
{
switch (instr->type) {
case nir_instr_type_alu: {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (!dest_is_invariant(&alu->dest.dest, invariants))
break;
alu->exact = true;
nir_foreach_src(instr, add_src_cb, invariants);
break;
}
case nir_instr_type_tex: {
nir_tex_instr *tex = nir_instr_as_tex(instr);
if (dest_is_invariant(&tex->dest, invariants))
nir_foreach_src(instr, add_src_cb, invariants);
break;
}
case nir_instr_type_intrinsic: {
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_copy_var:
/* If the destination is invariant then so is the source */
if (var_is_invariant(intrin->variables[0]->var, invariants))
add_var(intrin->variables[1]->var, invariants);
break;
case nir_intrinsic_load_var:
if (dest_is_invariant(&intrin->dest, invariants))
add_var(intrin->variables[0]->var, invariants);
break;
case nir_intrinsic_store_var:
if (var_is_invariant(intrin->variables[0]->var, invariants))
add_src(&intrin->src[0], invariants);
break;
default:
/* Nothing to do */
break;
}
}
case nir_instr_type_jump:
case nir_instr_type_ssa_undef:
case nir_instr_type_load_const:
break; /* Nothing to do */
case nir_instr_type_phi: {
nir_phi_instr *phi = nir_instr_as_phi(instr);
if (!dest_is_invariant(&phi->dest, invariants))
break;
nir_foreach_phi_src(src, phi) {
add_src(&src->src, invariants);
add_cf_node(&src->pred->cf_node, invariants);
}
break;
}
case nir_instr_type_call:
unreachable("This pass must be run after function inlining");
case nir_instr_type_parallel_copy:
default:
unreachable("Cannot have this instruction type");
}
nir_foreach_instr(succ, instr) {
if (instr->type != nir_instr_type_phi)
break;
validate_phi_src(nir_instr_as_phi(instr), block, state);
}
static bool
remove_phis_block(nir_block *block, nir_builder *b)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_ssa_def *def = NULL;
nir_alu_instr *mov = NULL;
bool srcs_same = true;
nir_foreach_phi_src(src, phi) {
assert(src->src.is_ssa);
/* For phi nodes at the beginning of loops, we may encounter some
* sources from backedges that point back to the destination of the
* same phi, i.e. something like:
*
* a = phi(a, b, ...)
*
* We can safely ignore these sources, since if all of the normal
* sources point to the same definition, then that definition must
* still dominate the phi node, and the phi will still always take
* the value of that definition.
*/
if (src->src.ssa == &phi->dest.ssa)
continue;
if (def == NULL) {
def = src->src.ssa;
mov = get_parent_mov(def);
} else {
if (src->src.ssa != def && !matching_mov(mov, src->src.ssa)) {
srcs_same = false;
break;
}
}
}
if (!srcs_same)
continue;
/* We must have found at least one definition, since there must be at
* least one forward edge.
*/
assert(def != NULL);
if (mov) {
/* If the sources were all movs from the same source with the same
* swizzle, then we can't just pick a random move because it may not
* dominate the phi node. Instead, we need to emit our own move after
* the phi which uses the shared source, and rewrite uses of the phi
* to use the move instead. This is ok, because while the movs may
* not all dominate the phi node, their shared source does.
*/
b->cursor = nir_after_phis(block);
def = nir_mov_alu(b, mov->src[0], def->num_components);
}
assert(phi->dest.is_ssa);
nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
nir_instr_remove(instr);
progress = true;
}