static void
lower_load_store(nir_builder *b,
nir_intrinsic_instr *intrin,
glsl_type_size_align_func size_align)
{
b->cursor = nir_before_instr(&intrin->instr);
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
nir_variable *var = nir_deref_instr_get_variable(deref);
nir_ssa_def *offset =
nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
var->data.location);
unsigned align, UNUSED size;
size_align(deref->type, &size, &align);
if (intrin->intrinsic == nir_intrinsic_load_deref) {
nir_intrinsic_instr *load =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
load->num_components = intrin->num_components;
load->src[0] = nir_src_for_ssa(offset);
nir_intrinsic_set_align(load, align, 0);
nir_ssa_dest_init(&load->instr, &load->dest,
intrin->dest.ssa.num_components,
intrin->dest.ssa.bit_size, NULL);
nir_builder_instr_insert(b, &load->instr);
nir_ssa_def *value = &load->dest.ssa;
if (glsl_type_is_boolean(deref->type))
value = nir_b2i32(b, value);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&load->dest.ssa));
} else {
assert(intrin->intrinsic == nir_intrinsic_store_deref);
assert(intrin->src[1].is_ssa);
nir_ssa_def *value = intrin->src[1].ssa;
if (glsl_type_is_boolean(deref->type))
value = nir_i2b(b, value);
nir_intrinsic_instr *store =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
store->num_components = intrin->num_components;
store->src[0] = nir_src_for_ssa(value);
store->src[1] = nir_src_for_ssa(offset);
nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
nir_intrinsic_set_align(store, align, 0);
nir_builder_instr_insert(b, &store->instr);
}
nir_instr_remove(&intrin->instr);
nir_deref_instr_remove_if_unused(deref);
}
nir_foreach_instr(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if (intrin->intrinsic != nir_intrinsic_load_deref &&
intrin->intrinsic != nir_intrinsic_store_deref)
continue;
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (!(deref->mode & modes))
continue;
if (!deref_has_indirect(nir_src_as_deref(intrin->src[0])))
continue;
nir_variable *var = nir_deref_instr_get_variable(deref);
/* We set var->mode to 0 to indicate that a variable will be moved
* to scratch. Don't assign a scratch location twice.
*/
if (var->data.mode == 0)
continue;
unsigned var_size, var_align;
size_align(var->type, &var_size, &var_align);
if (var_size <= size_threshold)
continue;
/* Remove it from its list */
exec_node_remove(&var->node);
/* Invalid mode used to flag "moving to scratch" */
var->data.mode = 0;
var->data.location = ALIGN_POT(shader->scratch_size, var_align);
shader->scratch_size = var->data.location + var_size;
}
nir_foreach_instr_safe(instr, block) {
if (instr->type == nir_instr_type_intrinsic) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic == nir_intrinsic_load_deref) {
nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
nir_variable *var = nir_deref_instr_get_variable(deref);
if ((var->data.mode == nir_var_shader_in &&
var->data.location == VARYING_SLOT_POS) ||
(var->data.mode == nir_var_system_value &&
var->data.location == SYSTEM_VALUE_FRAG_COORD)) {
/* gl_FragCoord should not have array/struct derefs: */
lower_fragcoord(state, intr, var);
} else if (var->data.mode == nir_var_system_value &&
var->data.location == SYSTEM_VALUE_SAMPLE_POS) {
lower_load_sample_pos(state, intr);
} else if (var->data.mode == nir_var_shader_in &&
var->data.location == VARYING_SLOT_PNTC &&
state->shader->options->lower_wpos_pntc) {
lower_load_pointcoord(state, intr);
}
} else if (intr->intrinsic == nir_intrinsic_load_frag_coord) {
lower_fragcoord(state, intr, NULL);
} else if (intr->intrinsic == nir_intrinsic_load_sample_pos) {
lower_load_sample_pos(state, intr);
} else if (intr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
lower_interp_deref_at_offset(state, intr);
}
} else if (instr->type == nir_instr_type_alu) {
nir_alu_instr *alu = nir_instr_as_alu(instr);
if (alu->op == nir_op_fddy ||
alu->op == nir_op_fddy_fine ||
alu->op == nir_op_fddy_coarse)
lower_fddy(state, alu);
}
}
static bool
lower_locals_to_regs_block(nir_block *block,
struct locals_to_regs_state *state)
{
nir_builder *b = &state->builder;
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
mov->src[0].src = get_deref_reg_src(deref, state);
mov->dest.write_mask = (1 << intrin->num_components) - 1;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
intrin->num_components,
intrin->dest.ssa.bit_size, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa));
} else {
nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
}
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_store_deref: {
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
if (deref->mode != nir_var_function_temp)
continue;
b->cursor = nir_before_instr(&intrin->instr);
nir_src reg_src = get_deref_reg_src(deref, state);
nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
nir_src_copy(&mov->src[0].src, &intrin->src[1], mov);
mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
mov->dest.dest.is_ssa = false;
mov->dest.dest.reg.reg = reg_src.reg.reg;
mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
mov->dest.dest.reg.indirect = reg_src.reg.indirect;
nir_builder_instr_insert(b, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_copy_deref:
unreachable("There should be no copies whatsoever at this point");
break;
default:
continue;
}
}
return true;
}
static void
gather_intrinsic_info(nir_intrinsic_instr *instr, nir_shader *shader,
void *dead_ctx)
{
switch (instr->intrinsic) {
case nir_intrinsic_discard:
case nir_intrinsic_discard_if:
assert(shader->info.stage == MESA_SHADER_FRAGMENT);
shader->info.fs.uses_discard = true;
break;
case nir_intrinsic_interp_deref_at_centroid:
case nir_intrinsic_interp_deref_at_sample:
case nir_intrinsic_interp_deref_at_offset:
case nir_intrinsic_load_deref:
case nir_intrinsic_store_deref:{
nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
nir_variable *var = nir_deref_instr_get_variable(deref);
if (var->data.mode == nir_var_shader_in ||
var->data.mode == nir_var_shader_out) {
bool is_output_read = false;
if (var->data.mode == nir_var_shader_out &&
instr->intrinsic == nir_intrinsic_load_deref)
is_output_read = true;
if (!try_mask_partial_io(shader, var, deref, is_output_read))
mark_whole_variable(shader, var, is_output_read);
/* We need to track which input_reads bits correspond to a
* dvec3/dvec4 input attribute */
if (shader->info.stage == MESA_SHADER_VERTEX &&
var->data.mode == nir_var_shader_in &&
glsl_type_is_dual_slot(glsl_without_array(var->type))) {
for (unsigned i = 0; i < glsl_count_attribute_slots(var->type, false); i++) {
int idx = var->data.location + i;
shader->info.vs.double_inputs |= BITFIELD64_BIT(idx);
}
}
}
break;
}
case nir_intrinsic_load_draw_id:
case nir_intrinsic_load_frag_coord:
case nir_intrinsic_load_front_face:
case nir_intrinsic_load_vertex_id:
case nir_intrinsic_load_vertex_id_zero_base:
case nir_intrinsic_load_base_vertex:
case nir_intrinsic_load_first_vertex:
case nir_intrinsic_load_is_indexed_draw:
case nir_intrinsic_load_base_instance:
case nir_intrinsic_load_instance_id:
case nir_intrinsic_load_sample_id:
case nir_intrinsic_load_sample_pos:
case nir_intrinsic_load_sample_mask_in:
case nir_intrinsic_load_primitive_id:
case nir_intrinsic_load_invocation_id:
case nir_intrinsic_load_local_invocation_id:
case nir_intrinsic_load_local_invocation_index:
case nir_intrinsic_load_work_group_id:
case nir_intrinsic_load_num_work_groups:
case nir_intrinsic_load_tess_coord:
case nir_intrinsic_load_tess_level_outer:
case nir_intrinsic_load_tess_level_inner:
case nir_intrinsic_load_patch_vertices_in:
shader->info.system_values_read |=
(1ull << nir_system_value_from_intrinsic(instr->intrinsic));
break;
case nir_intrinsic_end_primitive:
case nir_intrinsic_end_primitive_with_counter:
assert(shader->info.stage == MESA_SHADER_GEOMETRY);
shader->info.gs.uses_end_primitive = 1;
/* fall through */
case nir_intrinsic_emit_vertex:
if (nir_intrinsic_stream_id(instr) > 0)
shader->info.gs.uses_streams = true;
break;
default:
break;
}
}
