static bool
mark_indirect_uses_block(nir_block *block, void *void_state)
{
struct set *indirect_set = void_state;
nir_foreach_instr(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
for (unsigned i = 0;
i < nir_intrinsic_infos[intrin->intrinsic].num_variables; i++) {
if (deref_has_indirect(intrin->variables[i]))
_mesa_set_add(indirect_set, intrin->variables[i]->var);
}
}
return true;
}
static bool
lower_indirect_block(nir_block *block, nir_builder *b,
nir_variable_mode modes)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_var &&
intrin->intrinsic != nir_intrinsic_store_var)
continue;
if (!deref_has_indirect(intrin->variables[0]))
continue;
/* Only lower variables whose mode is in the mask */
if (!(modes & intrin->variables[0]->var->data.mode))
continue;
b->cursor = nir_before_instr(&intrin->instr);
if (intrin->intrinsic == nir_intrinsic_load_var) {
nir_ssa_def *result;
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, &result, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(result));
} else {
assert(intrin->src[0].is_ssa);
emit_load_store(b, intrin, intrin->variables[0],
&intrin->variables[0]->deref, NULL, intrin->src[0].ssa);
}
nir_instr_remove(&intrin->instr);
progress = true;
}
return progress;
}
nir_foreach_instr(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_deref &&
intrin->intrinsic != nir_intrinsic_store_deref)
continue;
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (!(deref->mode & modes))
continue;
if (!deref_has_indirect(nir_src_as_deref(intrin->src[0])))
continue;
nir_variable *var = nir_deref_instr_get_variable(deref);
/* We set var->mode to 0 to indicate that a variable will be moved
* to scratch. Don't assign a scratch location twice.
*/
if (var->data.mode == 0)
continue;
unsigned var_size, var_align;
size_align(var->type, &var_size, &var_align);
if (var_size <= size_threshold)
continue;
/* Remove it from its list */
exec_node_remove(&var->node);
/* Invalid mode used to flag "moving to scratch" */
var->data.mode = 0;
var->data.location = ALIGN_POT(shader->scratch_size, var_align);
shader->scratch_size = var->data.location + var_size;
}
static bool
nir_lower_io_block(nir_block *block, void *void_state)
{
struct lower_io_state *state = void_state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var: {
nir_variable_mode mode = intrin->variables[0]->var->data.mode;
if (mode != nir_var_shader_in && mode != nir_var_uniform)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
/* Figure out the opcode */
nir_intrinsic_op load_op;
switch (mode) {
case nir_var_shader_in:
load_op = has_indirect ? nir_intrinsic_load_input_indirect :
nir_intrinsic_load_input;
break;
case nir_var_uniform:
load_op = has_indirect ? nir_intrinsic_load_uniform_indirect :
nir_intrinsic_load_uniform;
break;
default:
unreachable("Unknown variable mode");
}
nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->mem_ctx,
load_op);
load->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
load->const_index[0] = offset;
if (has_indirect)
load->src[0] = indirect;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&load->instr, &load->dest,
intrin->num_components, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&load->dest.ssa),
state->mem_ctx);
} else {
nir_dest_copy(&load->dest, &intrin->dest, state->mem_ctx);
}
nir_instr_insert_before(&intrin->instr, &load->instr);
nir_instr_remove(&intrin->instr);
break;
}
case nir_intrinsic_store_var: {
if (intrin->variables[0]->var->data.mode != nir_var_shader_out)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
nir_intrinsic_op store_op;
if (has_indirect) {
store_op = nir_intrinsic_store_output_indirect;
} else {
store_op = nir_intrinsic_store_output;
}
nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
store_op);
store->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
store->const_index[0] = offset;
nir_src_copy(&store->src[0], &intrin->src[0], state->mem_ctx);
if (has_indirect)
store->src[1] = indirect;
nir_instr_insert_before(&intrin->instr, &store->instr);
nir_instr_remove(&intrin->instr);
break;
}
default:
break;
}
}
return true;
}
