static bool
block_has_no_side_effects(nir_block *block, void *state)
{
(void) state;
nir_foreach_instr(block, instr) {
if (instr->type == nir_instr_type_call)
return false;
/* Return instructions can cause us to skip over other side-effecting
* instructions after the loop, so consider them to have side effects
* here.
*/
if (instr->type == nir_instr_type_jump &&
nir_instr_as_jump(instr)->type == nir_jump_return)
return false;
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (!nir_intrinsic_infos[intrin->intrinsic].flags &
NIR_INTRINSIC_CAN_ELIMINATE)
return false;
}
return true;
}
static bool
constant_fold_block(nir_block *block, void *void_state)
{
struct constant_fold_state *state = void_state;
nir_foreach_instr_safe(block, instr) {
switch (instr->type) {
case nir_instr_type_alu:
state->progress |= constant_fold_alu_instr(nir_instr_as_alu(instr),
state->mem_ctx);
break;
case nir_instr_type_intrinsic:
state->progress |=
constant_fold_intrinsic_instr(nir_instr_as_intrinsic(instr));
break;
case nir_instr_type_tex:
state->progress |= constant_fold_tex_instr(nir_instr_as_tex(instr));
break;
default:
/* Don't know how to constant fold */
break;
}
}
return true;
}
nir_foreach_instr_safe(instr, block) {
if (instr->type == nir_instr_type_intrinsic) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic == nir_intrinsic_load_var) {
nir_deref_var *dvar = intr->variables[0];
nir_variable *var = dvar->var;
if (var->data.mode == nir_var_shader_in &&
var->data.location == VARYING_SLOT_POS) {
/* gl_FragCoord should not have array/struct deref's: */
assert(dvar->deref.child == NULL);
lower_fragcoord(state, intr);
} else if (var->data.mode == nir_var_system_value &&
var->data.location == SYSTEM_VALUE_SAMPLE_POS) {
assert(dvar->deref.child == NULL);
lower_load_sample_pos(state, intr);
}
} else if (intr->intrinsic == nir_intrinsic_interp_var_at_offset) {
lower_interp_var_at_offset(state, intr);
}
} else if (instr->type == nir_instr_type_alu) {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (alu->op == nir_op_fddy ||
alu->op == nir_op_fddy_fine ||
alu->op == nir_op_fddy_coarse)
lower_fddy(state, alu);
}
}
static bool
is_phi_src_scalarizable(nir_phi_src *src,
struct lower_phis_to_scalar_state *state)
{
/* Don't know what to do with non-ssa sources */
if (!src->src.is_ssa)
return false;
nir_instr *src_instr = src->src.ssa->parent_instr;
switch (src_instr->type) {
case nir_instr_type_alu: {
nir_alu_instr *src_alu = nir_instr_as_alu(src_instr);
/* ALU operations with output_size == 0 should be scalarized. We
* will also see a bunch of vecN operations from scalarizing ALU
* operations and, since they can easily be copy-propagated, they
* are ok too.
*/
return nir_op_infos[src_alu->op].output_size == 0 ||
src_alu->op == nir_op_vec2 ||
src_alu->op == nir_op_vec3 ||
src_alu->op == nir_op_vec4;
}
case nir_instr_type_phi:
/* A phi is scalarizable if we're going to lower it */
return should_lower_phi(nir_instr_as_phi(src_instr), state);
case nir_instr_type_load_const:
/* These are trivially scalarizable */
return true;
case nir_instr_type_intrinsic: {
nir_intrinsic_instr *src_intrin = nir_instr_as_intrinsic(src_instr);
switch (src_intrin->intrinsic) {
case nir_intrinsic_load_var:
return src_intrin->variables[0]->var->data.mode == nir_var_shader_in ||
src_intrin->variables[0]->var->data.mode == nir_var_uniform;
case nir_intrinsic_interp_var_at_centroid:
case nir_intrinsic_interp_var_at_sample:
case nir_intrinsic_interp_var_at_offset:
case nir_intrinsic_load_uniform:
case nir_intrinsic_load_uniform_indirect:
case nir_intrinsic_load_ubo:
case nir_intrinsic_load_ubo_indirect:
case nir_intrinsic_load_input:
case nir_intrinsic_load_input_indirect:
return true;
default:
break;
}
}
default:
/* We can't scalarize this type of instruction */
return false;
}
}
nir_foreach_instr(instr, block) {
switch (instr->type) {
case nir_instr_type_intrinsic:
gather_intrinsic_info(nir_instr_as_intrinsic(instr), info);
break;
case nir_instr_type_tex:
gather_tex_info(nir_instr_as_tex(instr), info);
break;
default:
break;
}
}
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *copy = nir_instr_as_intrinsic(instr);
if (copy->intrinsic != nir_intrinsic_copy_var)
continue;
nir_lower_var_copy_instr(copy, mem_ctx);
nir_instr_remove(©->instr);
ralloc_free(copy);
}
nir_foreach_instr(block, instr) {
switch (instr->type) {
case nir_instr_type_intrinsic:
gather_intrinsic_info(nir_instr_as_intrinsic(instr), shader);
break;
case nir_instr_type_tex:
gather_tex_info(nir_instr_as_tex(instr), shader);
break;
case nir_instr_type_call:
assert(!"nir_shader_gather_info only works if functions are inlined");
break;
default:
break;
}
}
static void
init_instr(nir_instr *instr, struct exec_list *worklist)
{
nir_alu_instr *alu_instr;
nir_intrinsic_instr *intrin_instr;
nir_tex_instr *tex_instr;
/* We use the pass_flags to store the live/dead information. In DCE, we
* just treat it as a zero/non-zerl boolean for whether or not the
* instruction is live.
*/
instr->pass_flags = 0;
switch (instr->type) {
case nir_instr_type_call:
case nir_instr_type_jump:
worklist_push(worklist, instr);
break;
case nir_instr_type_alu:
alu_instr = nir_instr_as_alu(instr);
if (!alu_instr->dest.dest.is_ssa)
worklist_push(worklist, instr);
break;
case nir_instr_type_intrinsic:
intrin_instr = nir_instr_as_intrinsic(instr);
if (nir_intrinsic_infos[intrin_instr->intrinsic].flags &
NIR_INTRINSIC_CAN_ELIMINATE) {
if (nir_intrinsic_infos[intrin_instr->intrinsic].has_dest &&
!intrin_instr->dest.is_ssa) {
worklist_push(worklist, instr);
}
} else {
worklist_push(worklist, instr);
}
break;
case nir_instr_type_tex:
tex_instr = nir_instr_as_tex(instr);
if (!tex_instr->dest.is_ssa)
worklist_push(worklist, instr);
break;
default:
break;
}
}
static void
validate_instr(nir_instr *instr, validate_state *state)
{
assert(instr->block == state->block);
state->instr = instr;
switch (instr->type) {
case nir_instr_type_alu:
validate_alu_instr(nir_instr_as_alu(instr), state);
break;
case nir_instr_type_call:
validate_call_instr(nir_instr_as_call(instr), state);
break;
case nir_instr_type_intrinsic:
validate_intrinsic_instr(nir_instr_as_intrinsic(instr), state);
break;
case nir_instr_type_tex:
validate_tex_instr(nir_instr_as_tex(instr), state);
break;
case nir_instr_type_load_const:
validate_load_const_instr(nir_instr_as_load_const(instr), state);
break;
case nir_instr_type_phi:
validate_phi_instr(nir_instr_as_phi(instr), state);
break;
case nir_instr_type_ssa_undef:
validate_ssa_undef_instr(nir_instr_as_ssa_undef(instr), state);
break;
case nir_instr_type_jump:
break;
default:
assert(!"Invalid ALU instruction type");
break;
}
state->instr = NULL;
}
static bool
split_var_copies_block(nir_block *block, void *void_state)
{
struct split_var_copies_state *state = void_state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
if (intrinsic->intrinsic != nir_intrinsic_copy_var)
continue;
nir_deref *dest_head = &intrinsic->variables[0]->deref;
nir_deref *src_head = &intrinsic->variables[1]->deref;
nir_deref *dest_tail = get_deref_tail(dest_head);
nir_deref *src_tail = get_deref_tail(src_head);
switch (glsl_get_base_type(src_tail->type)) {
case GLSL_TYPE_ARRAY:
case GLSL_TYPE_STRUCT:
split_var_copy_instr(intrinsic, dest_head, src_head,
dest_tail, src_tail, state);
nir_instr_remove(&intrinsic->instr);
ralloc_steal(state->dead_ctx, instr);
break;
case GLSL_TYPE_FLOAT:
case GLSL_TYPE_INT:
case GLSL_TYPE_UINT:
case GLSL_TYPE_BOOL:
if (glsl_type_is_matrix(src_tail->type)) {
split_var_copy_instr(intrinsic, dest_head, src_head,
dest_tail, src_tail, state);
nir_instr_remove(&intrinsic->instr);
ralloc_steal(state->dead_ctx, instr);
}
break;
default:
unreachable("Invalid type");
break;
}
}
return true;
}
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if ((mode == nir_var_shader_in && is_input(intrin)) ||
(mode == nir_var_shader_out && is_output(intrin))) {
nir_src *offset = nir_get_io_offset_src(intrin);
nir_const_value *const_offset = nir_src_as_const_value(*offset);
if (const_offset) {
intrin->const_index[0] += const_offset->u32[0];
b->cursor = nir_before_instr(&intrin->instr);
nir_instr_rewrite_src(&intrin->instr, offset,
nir_src_for_ssa(nir_imm_int(b, 0)));
}
}
}
static bool
mark_indirect_uses_block(nir_block *block, void *void_state)
{
struct set *indirect_set = void_state;
nir_foreach_instr(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
for (unsigned i = 0;
i < nir_intrinsic_infos[intrin->intrinsic].num_variables; i++) {
if (deref_has_indirect(intrin->variables[i]))
_mesa_set_add(indirect_set, intrin->variables[i]->var);
}
}
return true;
}
static bool
lower_indirect_block(nir_block *block, nir_builder *b,
nir_variable_mode modes)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_var &&
intrin->intrinsic != nir_intrinsic_store_var)
continue;
if (!deref_has_indirect(intrin->variables[0]))
continue;
/* Only lower variables whose mode is in the mask */
if (!(modes & intrin->variables[0]->var->data.mode))
continue;
b->cursor = nir_before_instr(&intrin->instr);
if (intrin->intrinsic == nir_intrinsic_load_var) {
nir_ssa_def *result;
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, &result, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(result));
} else {
assert(intrin->src[0].is_ssa);
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, NULL, intrin->src[0].ssa);
}
nir_instr_remove(&intrin->instr);
progress = true;
}
return progress;
}
static bool
constant_fold_block(nir_block *block, void *mem_ctx)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
switch (instr->type) {
case nir_instr_type_alu:
progress |= constant_fold_alu_instr(nir_instr_as_alu(instr), mem_ctx);
break;
case nir_instr_type_intrinsic:
progress |=
constant_fold_intrinsic_instr(nir_instr_as_intrinsic(instr));
break;
default:
/* Don't know how to constant fold */
break;
}
}
return progress;
}
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var:
register_load_instr(intrin, state);
break;
case nir_intrinsic_store_var:
register_store_instr(intrin, state);
break;
case nir_intrinsic_copy_var:
register_copy_instr(intrin, state);
break;
default:
continue;
}
}
nir_foreach_instr(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_deref &&
intrin->intrinsic != nir_intrinsic_store_deref)
continue;
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (!(deref->mode & modes))
continue;
if (!deref_has_indirect(nir_src_as_deref(intrin->src[0])))
continue;
nir_variable *var = nir_deref_instr_get_variable(deref);
/* We set var->mode to 0 to indicate that a variable will be moved
* to scratch. Don't assign a scratch location twice.
*/
if (var->data.mode == 0)
continue;
unsigned var_size, var_align;
size_align(var->type, &var_size, &var_align);
if (var_size <= size_threshold)
continue;
/* Remove it from its list */
exec_node_remove(&var->node);
/* Invalid mode used to flag "moving to scratch" */
var->data.mode = 0;
var->data.location = ALIGN_POT(shader->scratch_size, var_align);
shader->scratch_size = var->data.location + var_size;
}
nir_foreach_instr_safe(instr, block) {
if (instr->type == nir_instr_type_intrinsic) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic == nir_intrinsic_load_deref) {
nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
nir_variable *var = nir_deref_instr_get_variable(deref);
if ((var->data.mode == nir_var_shader_in &&
var->data.location == VARYING_SLOT_POS) ||
(var->data.mode == nir_var_system_value &&
var->data.location == SYSTEM_VALUE_FRAG_COORD)) {
/* gl_FragCoord should not have array/struct derefs: */
lower_fragcoord(state, intr, var);
} else if (var->data.mode == nir_var_system_value &&
var->data.location == SYSTEM_VALUE_SAMPLE_POS) {
lower_load_sample_pos(state, intr);
} else if (var->data.mode == nir_var_shader_in &&
var->data.location == VARYING_SLOT_PNTC &&
state->shader->options->lower_wpos_pntc) {
lower_load_pointcoord(state, intr);
}
} else if (intr->intrinsic == nir_intrinsic_load_frag_coord) {
lower_fragcoord(state, intr, NULL);
} else if (intr->intrinsic == nir_intrinsic_load_sample_pos) {
lower_load_sample_pos(state, intr);
} else if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
lower_interp_deref_at_offset(state, intr);
}
} else if (instr->type == nir_instr_type_alu) {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (alu->op == nir_op_fddy ||
alu->op == nir_op_fddy_fine ||
alu->op == nir_op_fddy_coarse)
lower_fddy(state, alu);
}
}
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var: {
if (intrin->variables[0]->var->data.mode != nir_var_local)
continue;
nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
mov->src[0].src = get_deref_reg_src(intrin->variables[0],
&intrin->instr, state);
mov->dest.write_mask = (1 << intrin->num_components) - 1;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
intrin->num_components,
intrin->dest.ssa.bit_size, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa));
} else {
nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
}
nir_instr_insert_before(&intrin->instr, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_store_var: {
if (intrin->variables[0]->var->data.mode != nir_var_local)
continue;
nir_src reg_src = get_deref_reg_src(intrin->variables[0],
&intrin->instr, state);
nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
nir_src_copy(&mov->src[0].src, &intrin->src[0], mov);
mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
mov->dest.dest.is_ssa = false;
mov->dest.dest.reg.reg = reg_src.reg.reg;
mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
mov->dest.dest.reg.indirect = reg_src.reg.indirect;
nir_instr_insert_before(&intrin->instr, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_copy_var:
unreachable("There should be no copies whatsoever at this point");
break;
default:
continue;
}
}
static bool
nir_lower_io_block(nir_block *block, void *void_state)
{
struct lower_io_state *state = void_state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var: {
nir_variable_mode mode = intrin->variables[0]->var->data.mode;
if (mode != nir_var_shader_in && mode != nir_var_uniform)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
/* Figure out the opcode */
nir_intrinsic_op load_op;
switch (mode) {
case nir_var_shader_in:
load_op = has_indirect ? nir_intrinsic_load_input_indirect :
nir_intrinsic_load_input;
break;
case nir_var_uniform:
load_op = has_indirect ? nir_intrinsic_load_uniform_indirect :
nir_intrinsic_load_uniform;
break;
default:
unreachable("Unknown variable mode");
}
nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->mem_ctx,
load_op);
load->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
load->const_index[0] = offset;
if (has_indirect)
load->src[0] = indirect;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&load->instr, &load->dest,
intrin->num_components, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&load->dest.ssa),
state->mem_ctx);
} else {
nir_dest_copy(&load->dest, &intrin->dest, state->mem_ctx);
}
nir_instr_insert_before(&intrin->instr, &load->instr);
nir_instr_remove(&intrin->instr);
break;
}
case nir_intrinsic_store_var: {
if (intrin->variables[0]->var->data.mode != nir_var_shader_out)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
nir_intrinsic_op store_op;
if (has_indirect) {
store_op = nir_intrinsic_store_output_indirect;
} else {
store_op = nir_intrinsic_store_output;
}
nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
store_op);
store->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
store->const_index[0] = offset;
nir_src_copy(&store->src[0], &intrin->src[0], state->mem_ctx);
if (has_indirect)
store->src[1] = indirect;
nir_instr_insert_before(&intrin->instr, &store->instr);
nir_instr_remove(&intrin->instr);
break;
}
default:
break;
}
}
return true;
}
static bool
lower_locals_to_regs_block(nir_block *block,
struct locals_to_regs_state *state)
{
nir_builder *b = &state->builder;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
mov->src[0].src = get_deref_reg_src(deref, state);
mov->dest.write_mask = (1 << intrin->num_components) - 1;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
intrin->num_components,
intrin->dest.ssa.bit_size, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa));
} else {
nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
}
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_store_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_src reg_src = get_deref_reg_src(deref, state);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
nir_src_copy(&mov->src[0].src, &intrin->src[1], mov);
mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
mov->dest.dest.is_ssa = false;
mov->dest.dest.reg.reg = reg_src.reg.reg;
mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
mov->dest.dest.reg.indirect = reg_src.reg.indirect;
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_copy_deref:
unreachable("There should be no copies whatsoever at this point");
break;
default:
continue;
}
}
return true;
}
static void
propagate_invariant_instr(nir_instr *instr, struct set *invariants)
{
switch (instr->type) {
case nir_instr_type_alu: {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (!dest_is_invariant(&alu->dest.dest, invariants))
break;
alu->exact = true;
nir_foreach_src(instr, add_src_cb, invariants);
break;
}
case nir_instr_type_tex: {
nir_tex_instr *tex = nir_instr_as_tex(instr);
if (dest_is_invariant(&tex->dest, invariants))
nir_foreach_src(instr, add_src_cb, invariants);
break;
}
case nir_instr_type_intrinsic: {
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_copy_var:
/* If the destination is invariant then so is the source */
if (var_is_invariant(intrin->variables[0]->var, invariants))
add_var(intrin->variables[1]->var, invariants);
break;
case nir_intrinsic_load_var:
if (dest_is_invariant(&intrin->dest, invariants))
add_var(intrin->variables[0]->var, invariants);
break;
case nir_intrinsic_store_var:
if (var_is_invariant(intrin->variables[0]->var, invariants))
add_src(&intrin->src[0], invariants);
break;
default:
/* Nothing to do */
break;
}
}
case nir_instr_type_jump:
case nir_instr_type_ssa_undef:
case nir_instr_type_load_const:
break; /* Nothing to do */
case nir_instr_type_phi: {
nir_phi_instr *phi = nir_instr_as_phi(instr);
if (!dest_is_invariant(&phi->dest, invariants))
break;
nir_foreach_phi_src(src, phi) {
add_src(&src->src, invariants);
add_cf_node(&src->pred->cf_node, invariants);
}
break;
}
case nir_instr_type_call:
unreachable("This pass must be run after function inlining");
case nir_instr_type_parallel_copy:
default:
unreachable("Cannot have this instruction type");
}
nir_foreach_instr_safe(block, instr) {
if (instr->type == nir_instr_type_intrinsic)
lower_instr(nir_instr_as_intrinsic(instr),
(lower_atomic_state *) state);
}