static bool
constant_fold_intrinsic_instr(nir_intrinsic_instr *instr)
{
bool progress = false;
if (instr->intrinsic == nir_intrinsic_discard_if &&
nir_src_is_const(instr->src[0])) {
if (nir_src_as_bool(instr->src[0])) {
/* This method of getting a nir_shader * from a nir_instr is
* admittedly gross, but given the rarity of hitting this case I think
* it's preferable to plumbing an otherwise unused nir_shader *
* parameter through four functions to get here.
*/
nir_cf_node *cf_node = &instr->instr.block->cf_node;
nir_function_impl *impl = nir_cf_node_get_function(cf_node);
nir_shader *shader = impl->function->shader;
nir_intrinsic_instr *discard =
nir_intrinsic_instr_create(shader, nir_intrinsic_discard);
nir_instr_insert_before(&instr->instr, &discard->instr);
nir_instr_remove(&instr->instr);
progress = true;
} else {
/* We're not discarding, just delete the instruction */
nir_instr_remove(&instr->instr);
progress = true;
}
}
return progress;
}
/**
* For a given starting writemask channel and corresponding source index in
* the vec instruction, insert a MOV to the vec instruction's dest of all the
* writemask channels that get read from the same src reg.
*
* Returns the writemask of our MOV, so the parent loop calling this knows
* which ones have been processed.
*/
static unsigned
insert_mov(nir_alu_instr *vec, unsigned start_channel,
unsigned start_src_idx, nir_shader *shader)
{
unsigned src_idx = start_src_idx;
assert(src_idx < nir_op_infos[vec->op].num_inputs);
nir_alu_instr *mov = nir_alu_instr_create(shader, nir_op_imov);
nir_alu_src_copy(&mov->src[0], &vec->src[src_idx], mov);
nir_alu_dest_copy(&mov->dest, &vec->dest, mov);
mov->dest.write_mask = (1u << start_channel);
mov->src[0].swizzle[start_channel] = vec->src[src_idx].swizzle[0];
src_idx++;
for (unsigned i = start_channel + 1; i < 4; i++) {
if (!(vec->dest.write_mask & (1 << i)))
continue;
if (nir_srcs_equal(vec->src[src_idx].src, vec->src[start_src_idx].src)) {
mov->dest.write_mask |= (1 << i);
mov->src[0].swizzle[i] = vec->src[src_idx].swizzle[0];
}
src_idx++;
}
nir_instr_insert_before(&vec->instr, &mov->instr);
return mov->dest.write_mask;
}
static void
lower_reduction(nir_alu_instr *instr, nir_op chan_op, nir_op merge_op,
void *mem_ctx)
{
unsigned num_components = nir_op_infos[instr->op].input_sizes[0];
nir_ssa_def *last = NULL;
for (unsigned i = 0; i < num_components; i++) {
nir_alu_instr *chan = nir_alu_instr_create(mem_ctx, chan_op);
nir_alu_ssa_dest_init(chan, 1);
nir_alu_src_copy(&chan->src[0], &instr->src[0], mem_ctx);
chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
if (nir_op_infos[chan_op].num_inputs > 1) {
assert(nir_op_infos[chan_op].num_inputs == 2);
nir_alu_src_copy(&chan->src[1], &instr->src[1], mem_ctx);
chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
}
nir_instr_insert_before(&instr->instr, &chan->instr);
if (i == 0) {
last = &chan->dest.dest.ssa;
} else {
nir_alu_instr *merge = nir_alu_instr_create(mem_ctx, merge_op);
nir_alu_ssa_dest_init(merge, 1);
merge->dest.write_mask = 1;
merge->src[0].src = nir_src_for_ssa(last);
merge->src[1].src = nir_src_for_ssa(&chan->dest.dest.ssa);
nir_instr_insert_before(&instr->instr, &merge->instr);
last = &merge->dest.dest.ssa;
}
}
assert(instr->dest.write_mask == 1);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(last),
mem_ctx);
nir_instr_remove(&instr->instr);
}
static bool
constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx)
{
nir_const_value src[4];
if (!instr->dest.dest.is_ssa)
return false;
for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
if (!instr->src[i].src.is_ssa)
return false;
nir_instr *src_instr = instr->src[i].src.ssa->parent_instr;
if (src_instr->type != nir_instr_type_load_const)
return false;
nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr);
for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i);
j++) {
src[i].u[j] = load_const->value.u[instr->src[i].swizzle[j]];
}
/* We shouldn't have any source modifiers in the optimization loop. */
assert(!instr->src[i].abs && !instr->src[i].negate);
}
/* We shouldn't have any saturate modifiers in the optimization loop. */
assert(!instr->dest.saturate);
nir_const_value dest =
nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components,
src);
nir_load_const_instr *new_instr =
nir_load_const_instr_create(mem_ctx,
instr->dest.dest.ssa.num_components);
new_instr->value = dest;
nir_instr_insert_before(&instr->instr, &new_instr->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(&new_instr->def),
mem_ctx);
nir_instr_remove(&instr->instr);
ralloc_free(instr);
return true;
}
nir_alu_instr *
nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search,
const nir_search_value *replace, void *mem_ctx)
{
uint8_t swizzle[4] = { 0, 0, 0, 0 };
for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i)
swizzle[i] = i;
assert(instr->dest.dest.is_ssa);
struct match_state state;
state.variables_seen = 0;
if (!match_expression(search, instr, instr->dest.dest.ssa.num_components,
swizzle, &state))
return NULL;
/* Inserting a mov may be unnecessary. However, it's much easier to
* simply let copy propagation clean this up than to try to go through
* and rewrite swizzles ourselves.
*/
nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov);
mov->dest.write_mask = instr->dest.write_mask;
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
instr->dest.dest.ssa.num_components, NULL);
mov->src[0] = construct_value(replace, nir_op_infos[instr->op].output_type,
instr->dest.dest.ssa.num_components, &state,
&instr->instr, mem_ctx);
nir_instr_insert_before(&instr->instr, &mov->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa), mem_ctx);
/* We know this one has no more uses because we just rewrote them all,
* so we can remove it. The rest of the matched expression, however, we
* don't know so much about. We'll just let dead code clean them up.
*/
nir_instr_remove(&instr->instr);
return mov;
}
/* This function recursively walks the given deref chain and replaces the
* given copy instruction with an equivalent sequence load/store
* operations.
*
* @copy_instr The copy instruction to replace; new instructions will be
* inserted before this one
*
* @dest_head The head of the destination variable deref chain
*
* @src_head The head of the source variable deref chain
*
* @dest_tail The current tail of the destination variable deref chain;
* this is used for recursion and external callers of this
* function should call it with tail == head
*
* @src_tail The current tail of the source variable deref chain;
* this is used for recursion and external callers of this
* function should call it with tail == head
*
* @state The current variable lowering state
*/
static void
emit_copy_load_store(nir_intrinsic_instr *copy_instr,
nir_deref_var *dest_head, nir_deref_var *src_head,
nir_deref *dest_tail, nir_deref *src_tail, void *mem_ctx)
{
/* Find the next pair of wildcards */
nir_deref *src_arr_parent = deref_next_wildcard_parent(src_tail);
nir_deref *dest_arr_parent = deref_next_wildcard_parent(dest_tail);
if (src_arr_parent || dest_arr_parent) {
/* Wildcards had better come in matched pairs */
assert(dest_arr_parent && dest_arr_parent);
nir_deref_array *src_arr = nir_deref_as_array(src_arr_parent->child);
nir_deref_array *dest_arr = nir_deref_as_array(dest_arr_parent->child);
unsigned length = glsl_get_length(src_arr_parent->type);
/* The wildcards should represent the same number of elements */
assert(length == glsl_get_length(dest_arr_parent->type));
assert(length > 0);
/* Walk over all of the elements that this wildcard refers to and
* call emit_copy_load_store on each one of them */
src_arr->deref_array_type = nir_deref_array_type_direct;
dest_arr->deref_array_type = nir_deref_array_type_direct;
for (unsigned i = 0; i < length; i++) {
src_arr->base_offset = i;
dest_arr->base_offset = i;
emit_copy_load_store(copy_instr, dest_head, src_head,
&dest_arr->deref, &src_arr->deref, mem_ctx);
}
src_arr->deref_array_type = nir_deref_array_type_wildcard;
dest_arr->deref_array_type = nir_deref_array_type_wildcard;
} else {
/* In this case, we have no wildcards anymore, so all we have to do
* is just emit the load and store operations. */
src_tail = nir_deref_tail(src_tail);
dest_tail = nir_deref_tail(dest_tail);
assert(src_tail->type == dest_tail->type);
unsigned num_components = glsl_get_vector_elements(src_tail->type);
nir_intrinsic_instr *load =
nir_intrinsic_instr_create(mem_ctx, nir_intrinsic_load_var);
load->num_components = num_components;
load->variables[0] = nir_deref_as_var(nir_copy_deref(load, &src_head->deref));
nir_ssa_dest_init(&load->instr, &load->dest, num_components, NULL);
nir_instr_insert_before(©_instr->instr, &load->instr);
nir_intrinsic_instr *store =
nir_intrinsic_instr_create(mem_ctx, nir_intrinsic_store_var);
store->num_components = num_components;
store->const_index[0] = (1 << num_components) - 1;
store->variables[0] = nir_deref_as_var(nir_copy_deref(store, &dest_head->deref));
store->src[0].is_ssa = true;
store->src[0].ssa = &load->dest.ssa;
nir_instr_insert_before(©_instr->instr, &store->instr);
}
}
static void
lower_alu_instr_scalar(nir_alu_instr *instr, void *mem_ctx)
{
unsigned num_src = nir_op_infos[instr->op].num_inputs;
unsigned i, chan;
assert(instr->dest.dest.is_ssa);
assert(instr->dest.write_mask != 0);
#define LOWER_REDUCTION(name, chan, merge) \
case name##2: \
case name##3: \
case name##4: \
lower_reduction(instr, chan, merge, mem_ctx); \
break;
switch (instr->op) {
case nir_op_vec4:
case nir_op_vec3:
case nir_op_vec2:
/* We don't need to scalarize these ops, they're the ones generated to
* group up outputs into a value that can be SSAed.
*/
return;
LOWER_REDUCTION(nir_op_fdot, nir_op_fmul, nir_op_fadd);
LOWER_REDUCTION(nir_op_ball_fequal, nir_op_feq, nir_op_iand);
LOWER_REDUCTION(nir_op_ball_iequal, nir_op_ieq, nir_op_iand);
LOWER_REDUCTION(nir_op_bany_fnequal, nir_op_fne, nir_op_ior);
LOWER_REDUCTION(nir_op_bany_inequal, nir_op_ine, nir_op_ior);
LOWER_REDUCTION(nir_op_fall_equal, nir_op_seq, nir_op_fand);
LOWER_REDUCTION(nir_op_fany_nequal, nir_op_sne, nir_op_for);
LOWER_REDUCTION(nir_op_ball, nir_op_imov, nir_op_iand);
LOWER_REDUCTION(nir_op_bany, nir_op_imov, nir_op_ior);
LOWER_REDUCTION(nir_op_fall, nir_op_fmov, nir_op_fand);
LOWER_REDUCTION(nir_op_fany, nir_op_fmov, nir_op_for);
default:
break;
}
if (instr->dest.dest.ssa.num_components == 1)
return;
unsigned num_components = instr->dest.dest.ssa.num_components;
static const nir_op nir_op_map[] = {nir_op_vec2, nir_op_vec3, nir_op_vec4};
nir_alu_instr *vec_instr =
nir_alu_instr_create(mem_ctx, nir_op_map[num_components - 2]);
nir_alu_ssa_dest_init(vec_instr, num_components);
for (chan = 0; chan < 4; chan++) {
if (!(instr->dest.write_mask & (1 << chan)))
continue;
nir_alu_instr *lower = nir_alu_instr_create(mem_ctx, instr->op);
for (i = 0; i < num_src; i++) {
/* We only handle same-size-as-dest (input_sizes[] == 0) or scalar
* args (input_sizes[] == 1).
*/
assert(nir_op_infos[instr->op].input_sizes[i] < 2);
unsigned src_chan = (nir_op_infos[instr->op].input_sizes[i] == 1 ?
0 : chan);
nir_alu_src_copy(&lower->src[i], &instr->src[i], mem_ctx);
for (int j = 0; j < 4; j++)
lower->src[i].swizzle[j] = instr->src[i].swizzle[src_chan];
}
nir_alu_ssa_dest_init(lower, 1);
lower->dest.saturate = instr->dest.saturate;
vec_instr->src[chan].src = nir_src_for_ssa(&lower->dest.dest.ssa);
nir_instr_insert_before(&instr->instr, &lower->instr);
}
nir_instr_insert_before(&instr->instr, &vec_instr->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
nir_src_for_ssa(&vec_instr->dest.dest.ssa),
mem_ctx);
nir_instr_remove(&instr->instr);
}
static void
lower_instr(nir_intrinsic_instr *instr,
lower_atomic_state *state)
{
nir_intrinsic_op op;
switch (instr->intrinsic) {
case nir_intrinsic_atomic_counter_read_var:
op = nir_intrinsic_atomic_counter_read;
break;
case nir_intrinsic_atomic_counter_inc_var:
op = nir_intrinsic_atomic_counter_inc;
break;
case nir_intrinsic_atomic_counter_dec_var:
op = nir_intrinsic_atomic_counter_dec;
break;
default:
return;
}
if (instr->variables[0]->var->data.mode != nir_var_uniform &&
instr->variables[0]->var->data.mode != nir_var_shader_storage)
return; /* atomics passed as function arguments can't be lowered */
void *mem_ctx = ralloc_parent(instr);
unsigned uniform_loc = instr->variables[0]->var->data.location;
nir_intrinsic_instr *new_instr = nir_intrinsic_instr_create(mem_ctx, op);
new_instr->const_index[0] =
state->shader_program->UniformStorage[uniform_loc].opaque[state->shader->stage].index;
nir_load_const_instr *offset_const = nir_load_const_instr_create(mem_ctx, 1);
offset_const->value.u[0] = instr->variables[0]->var->data.atomic.offset;
nir_instr_insert_before(&instr->instr, &offset_const->instr);
nir_ssa_def *offset_def = &offset_const->def;
nir_deref *tail = &instr->variables[0]->deref;
while (tail->child != NULL) {
assert(tail->child->deref_type == nir_deref_type_array);
nir_deref_array *deref_array = nir_deref_as_array(tail->child);
tail = tail->child;
unsigned child_array_elements = tail->child != NULL ?
glsl_get_aoa_size(tail->type) : 1;
offset_const->value.u[0] += deref_array->base_offset *
child_array_elements * ATOMIC_COUNTER_SIZE;
if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
nir_load_const_instr *atomic_counter_size =
nir_load_const_instr_create(mem_ctx, 1);
atomic_counter_size->value.u[0] = child_array_elements * ATOMIC_COUNTER_SIZE;
nir_instr_insert_before(&instr->instr, &atomic_counter_size->instr);
nir_alu_instr *mul = nir_alu_instr_create(mem_ctx, nir_op_imul);
nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, NULL);
mul->dest.write_mask = 0x1;
nir_src_copy(&mul->src[0].src, &deref_array->indirect, mul);
mul->src[1].src.is_ssa = true;
mul->src[1].src.ssa = &atomic_counter_size->def;
nir_instr_insert_before(&instr->instr, &mul->instr);
nir_alu_instr *add = nir_alu_instr_create(mem_ctx, nir_op_iadd);
nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, NULL);
add->dest.write_mask = 0x1;
add->src[0].src.is_ssa = true;
add->src[0].src.ssa = &mul->dest.dest.ssa;
add->src[1].src.is_ssa = true;
add->src[1].src.ssa = offset_def;
nir_instr_insert_before(&instr->instr, &add->instr);
offset_def = &add->dest.dest.ssa;
}
}
new_instr->src[0].is_ssa = true;
new_instr->src[0].ssa = offset_def;
if (instr->dest.is_ssa) {
nir_ssa_dest_init(&new_instr->instr, &new_instr->dest,
instr->dest.ssa.num_components, NULL);
nir_ssa_def_rewrite_uses(&instr->dest.ssa,
nir_src_for_ssa(&new_instr->dest.ssa));
} else {
nir_dest_copy(&new_instr->dest, &instr->dest, mem_ctx);
}
nir_instr_insert_before(&instr->instr, &new_instr->instr);
nir_instr_remove(&instr->instr);
}
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var: {
if (intrin->variables[0]->var->data.mode != nir_var_local)
continue;
nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
mov->src[0].src = get_deref_reg_src(intrin->variables[0],
&intrin->instr, state);
mov->dest.write_mask = (1 << intrin->num_components) - 1;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
intrin->num_components,
intrin->dest.ssa.bit_size, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa));
} else {
nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
}
nir_instr_insert_before(&intrin->instr, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_store_var: {
if (intrin->variables[0]->var->data.mode != nir_var_local)
continue;
nir_src reg_src = get_deref_reg_src(intrin->variables[0],
&intrin->instr, state);
nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
nir_src_copy(&mov->src[0].src, &intrin->src[0], mov);
mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
mov->dest.dest.is_ssa = false;
mov->dest.dest.reg.reg = reg_src.reg.reg;
mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
mov->dest.dest.reg.indirect = reg_src.reg.indirect;
nir_instr_insert_before(&intrin->instr, &mov->instr);
nir_instr_remove(&intrin->instr);
state->progress = true;
break;
}
case nir_intrinsic_copy_var:
unreachable("There should be no copies whatsoever at this point");
break;
default:
continue;
}
}
static nir_src
get_deref_reg_src(nir_deref_var *deref, nir_instr *instr,
struct locals_to_regs_state *state)
{
nir_src src;
src.is_ssa = false;
src.reg.reg = get_reg_for_deref(deref, state);
src.reg.base_offset = 0;
src.reg.indirect = NULL;
/* It is possible for a user to create a shader that has an array with a
* single element and then proceed to access it indirectly. Indirectly
* accessing a non-array register is not allowed in NIR. In order to
* handle this case we just convert it to a direct reference.
*/
if (src.reg.reg->num_array_elems == 0)
return src;
nir_deref *tail = &deref->deref;
while (tail->child != NULL) {
const struct glsl_type *parent_type = tail->type;
tail = tail->child;
if (tail->deref_type != nir_deref_type_array)
continue;
nir_deref_array *deref_array = nir_deref_as_array(tail);
src.reg.base_offset *= glsl_get_length(parent_type);
src.reg.base_offset += deref_array->base_offset;
if (src.reg.indirect) {
nir_load_const_instr *load_const =
nir_load_const_instr_create(state->shader, 1, 32);
load_const->value.u32[0] = glsl_get_length(parent_type);
nir_instr_insert_before(instr, &load_const->instr);
nir_alu_instr *mul = nir_alu_instr_create(state->shader, nir_op_imul);
mul->src[0].src = *src.reg.indirect;
mul->src[1].src.is_ssa = true;
mul->src[1].src.ssa = &load_const->def;
mul->dest.write_mask = 1;
nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, 32, NULL);
nir_instr_insert_before(instr, &mul->instr);
src.reg.indirect->is_ssa = true;
src.reg.indirect->ssa = &mul->dest.dest.ssa;
}
if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
if (src.reg.indirect == NULL) {
src.reg.indirect = ralloc(state->shader, nir_src);
nir_src_copy(src.reg.indirect, &deref_array->indirect,
state->shader);
} else {
nir_alu_instr *add = nir_alu_instr_create(state->shader,
nir_op_iadd);
add->src[0].src = *src.reg.indirect;
nir_src_copy(&add->src[1].src, &deref_array->indirect, add);
add->dest.write_mask = 1;
nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, 32, NULL);
nir_instr_insert_before(instr, &add->instr);
src.reg.indirect->is_ssa = true;
src.reg.indirect->ssa = &add->dest.dest.ssa;
}
}
}
return src;
}
static bool
constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx)
{
nir_const_value src[NIR_MAX_VEC_COMPONENTS];
if (!instr->dest.dest.is_ssa)
return false;
/* In the case that any outputs/inputs have unsized types, then we need to
* guess the bit-size. In this case, the validator ensures that all
* bit-sizes match so we can just take the bit-size from first
* output/input with an unsized type. If all the outputs/inputs are sized
* then we don't need to guess the bit-size at all because the code we
* generate for constant opcodes in this case already knows the sizes of
* the types involved and does not need the provided bit-size for anything
* (although it still requires to receive a valid bit-size).
*/
unsigned bit_size = 0;
if (!nir_alu_type_get_type_size(nir_op_infos[instr->op].output_type))
bit_size = instr->dest.dest.ssa.bit_size;
for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
if (!instr->src[i].src.is_ssa)
return false;
if (bit_size == 0 &&
!nir_alu_type_get_type_size(nir_op_infos[instr->op].input_sizes[i])) {
bit_size = instr->src[i].src.ssa->bit_size;
}
nir_instr *src_instr = instr->src[i].src.ssa->parent_instr;
if (src_instr->type != nir_instr_type_load_const)
return false;
nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr);
for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i);
j++) {
switch(load_const->def.bit_size) {
case 64:
src[i].u64[j] = load_const->value.u64[instr->src[i].swizzle[j]];
break;
case 32:
src[i].u32[j] = load_const->value.u32[instr->src[i].swizzle[j]];
break;
case 16:
src[i].u16[j] = load_const->value.u16[instr->src[i].swizzle[j]];
break;
case 8:
src[i].u8[j] = load_const->value.u8[instr->src[i].swizzle[j]];
break;
default:
unreachable("Invalid bit size");
}
}
/* We shouldn't have any source modifiers in the optimization loop. */
assert(!instr->src[i].abs && !instr->src[i].negate);
}
if (bit_size == 0)
bit_size = 32;
/* We shouldn't have any saturate modifiers in the optimization loop. */
assert(!instr->dest.saturate);
nir_const_value dest =
nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components,
bit_size, src);
nir_load_const_instr *new_instr =
nir_load_const_instr_create(mem_ctx,
instr->dest.dest.ssa.num_components,
instr->dest.dest.ssa.bit_size);
new_instr->value = dest;
nir_instr_insert_before(&instr->instr, &new_instr->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
nir_src_for_ssa(&new_instr->def));
nir_instr_remove(&instr->instr);
ralloc_free(instr);
return true;
}
static void
lower_instr(nir_intrinsic_instr *instr, nir_function_impl *impl)
{
nir_intrinsic_op op;
switch (instr->intrinsic) {
case nir_intrinsic_atomic_counter_read_var:
op = nir_intrinsic_atomic_counter_read;
break;
case nir_intrinsic_atomic_counter_inc_var:
op = nir_intrinsic_atomic_counter_inc;
break;
case nir_intrinsic_atomic_counter_dec_var:
op = nir_intrinsic_atomic_counter_dec;
break;
default:
return;
}
if (instr->variables[0]->var->data.mode != nir_var_uniform)
return; /* atomics passed as function arguments can't be lowered */
void *mem_ctx = ralloc_parent(instr);
nir_intrinsic_instr *new_instr = nir_intrinsic_instr_create(mem_ctx, op);
new_instr->const_index[0] =
(int) instr->variables[0]->var->data.atomic.buffer_index;
nir_load_const_instr *offset_const = nir_load_const_instr_create(mem_ctx, 1);
offset_const->value.u[0] = instr->variables[0]->var->data.atomic.offset;
nir_instr_insert_before(&instr->instr, &offset_const->instr);
nir_ssa_def *offset_def = &offset_const->def;
if (instr->variables[0]->deref.child != NULL) {
assert(instr->variables[0]->deref.child->deref_type ==
nir_deref_type_array);
nir_deref_array *deref_array =
nir_deref_as_array(instr->variables[0]->deref.child);
assert(deref_array->deref.child == NULL);
offset_const->value.u[0] +=
deref_array->base_offset * ATOMIC_COUNTER_SIZE;
if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
nir_load_const_instr *atomic_counter_size =
nir_load_const_instr_create(mem_ctx, 1);
atomic_counter_size->value.u[0] = ATOMIC_COUNTER_SIZE;
nir_instr_insert_before(&instr->instr, &atomic_counter_size->instr);
nir_alu_instr *mul = nir_alu_instr_create(mem_ctx, nir_op_imul);
nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, NULL);
mul->dest.write_mask = 0x1;
nir_src_copy(&mul->src[0].src, &deref_array->indirect, mem_ctx);
mul->src[1].src.is_ssa = true;
mul->src[1].src.ssa = &atomic_counter_size->def;
nir_instr_insert_before(&instr->instr, &mul->instr);
nir_alu_instr *add = nir_alu_instr_create(mem_ctx, nir_op_iadd);
nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, NULL);
add->dest.write_mask = 0x1;
add->src[0].src.is_ssa = true;
add->src[0].src.ssa = &mul->dest.dest.ssa;
add->src[1].src.is_ssa = true;
add->src[1].src.ssa = &offset_const->def;
nir_instr_insert_before(&instr->instr, &add->instr);
offset_def = &add->dest.dest.ssa;
}
}
new_instr->src[0].is_ssa = true;
new_instr->src[0].ssa = offset_def;;
if (instr->dest.is_ssa) {
nir_ssa_dest_init(&new_instr->instr, &new_instr->dest,
instr->dest.ssa.num_components, NULL);
nir_ssa_def_rewrite_uses(&instr->dest.ssa,
nir_src_for_ssa(&new_instr->dest.ssa),
mem_ctx);
} else {
nir_dest_copy(&new_instr->dest, &instr->dest, mem_ctx);
}
nir_instr_insert_before(&instr->instr, &new_instr->instr);
nir_instr_remove(&instr->instr);
}
static bool
nir_lower_io_block(nir_block *block, void *void_state)
{
struct lower_io_state *state = void_state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_load_var: {
nir_variable_mode mode = intrin->variables[0]->var->data.mode;
if (mode != nir_var_shader_in && mode != nir_var_uniform)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
/* Figure out the opcode */
nir_intrinsic_op load_op;
switch (mode) {
case nir_var_shader_in:
load_op = has_indirect ? nir_intrinsic_load_input_indirect :
nir_intrinsic_load_input;
break;
case nir_var_uniform:
load_op = has_indirect ? nir_intrinsic_load_uniform_indirect :
nir_intrinsic_load_uniform;
break;
default:
unreachable("Unknown variable mode");
}
nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->mem_ctx,
load_op);
load->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
load->const_index[0] = offset;
if (has_indirect)
load->src[0] = indirect;
if (intrin->dest.is_ssa) {
nir_ssa_dest_init(&load->instr, &load->dest,
intrin->num_components, NULL);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&load->dest.ssa),
state->mem_ctx);
} else {
nir_dest_copy(&load->dest, &intrin->dest, state->mem_ctx);
}
nir_instr_insert_before(&intrin->instr, &load->instr);
nir_instr_remove(&intrin->instr);
break;
}
case nir_intrinsic_store_var: {
if (intrin->variables[0]->var->data.mode != nir_var_shader_out)
continue;
bool has_indirect = deref_has_indirect(intrin->variables[0]);
nir_intrinsic_op store_op;
if (has_indirect) {
store_op = nir_intrinsic_store_output_indirect;
} else {
store_op = nir_intrinsic_store_output;
}
nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
store_op);
store->num_components = intrin->num_components;
nir_src indirect;
unsigned offset = get_io_offset(intrin->variables[0],
&intrin->instr, &indirect, state);
offset += intrin->variables[0]->var->data.driver_location;
store->const_index[0] = offset;
nir_src_copy(&store->src[0], &intrin->src[0], state->mem_ctx);
if (has_indirect)
store->src[1] = indirect;
nir_instr_insert_before(&intrin->instr, &store->instr);
nir_instr_remove(&intrin->instr);
break;
}
default:
break;
}
}
return true;
}
static unsigned
get_io_offset(nir_deref_var *deref, nir_instr *instr, nir_src *indirect,
struct lower_io_state *state)
{
bool found_indirect = false;
unsigned base_offset = 0;
nir_deref *tail = &deref->deref;
while (tail->child != NULL) {
const struct glsl_type *parent_type = tail->type;
tail = tail->child;
if (tail->deref_type == nir_deref_type_array) {
nir_deref_array *deref_array = nir_deref_as_array(tail);
unsigned size = type_size(tail->type);
base_offset += size * deref_array->base_offset;
if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
nir_load_const_instr *load_const =
nir_load_const_instr_create(state->mem_ctx, 1);
load_const->value.u[0] = size;
nir_instr_insert_before(instr, &load_const->instr);
nir_alu_instr *mul = nir_alu_instr_create(state->mem_ctx,
nir_op_imul);
mul->src[0].src.is_ssa = true;
mul->src[0].src.ssa = &load_const->def;
nir_src_copy(&mul->src[1].src, &deref_array->indirect,
state->mem_ctx);
mul->dest.write_mask = 1;
nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, NULL);
nir_instr_insert_before(instr, &mul->instr);
if (found_indirect) {
nir_alu_instr *add = nir_alu_instr_create(state->mem_ctx,
nir_op_iadd);
add->src[0].src = *indirect;
add->src[1].src.is_ssa = true;
add->src[1].src.ssa = &mul->dest.dest.ssa;
add->dest.write_mask = 1;
nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, NULL);
nir_instr_insert_before(instr, &add->instr);
indirect->is_ssa = true;
indirect->ssa = &add->dest.dest.ssa;
} else {
indirect->is_ssa = true;
indirect->ssa = &mul->dest.dest.ssa;
found_indirect = true;
}
}
} else if (tail->deref_type == nir_deref_type_struct) {
nir_deref_struct *deref_struct = nir_deref_as_struct(tail);
for (unsigned i = 0; i < deref_struct->index; i++)
base_offset += type_size(glsl_get_struct_field(parent_type, i));
}
}
return base_offset;
}
static nir_alu_src
construct_value(const nir_search_value *value, nir_alu_type type,
unsigned num_components, struct match_state *state,
nir_instr *instr, void *mem_ctx)
{
switch (value->type) {
case nir_search_value_expression: {
const nir_search_expression *expr = nir_search_value_as_expression(value);
if (nir_op_infos[expr->opcode].output_size != 0)
num_components = nir_op_infos[expr->opcode].output_size;
nir_alu_instr *alu = nir_alu_instr_create(mem_ctx, expr->opcode);
nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components, NULL);
alu->dest.write_mask = (1 << num_components) - 1;
alu->dest.saturate = false;
for (unsigned i = 0; i < nir_op_infos[expr->opcode].num_inputs; i++) {
/* If the source is an explicitly sized source, then we need to reset
* the number of components to match.
*/
if (nir_op_infos[alu->op].input_sizes[i] != 0)
num_components = nir_op_infos[alu->op].input_sizes[i];
alu->src[i] = construct_value(expr->srcs[i],
nir_op_infos[alu->op].input_types[i],
num_components,
state, instr, mem_ctx);
}
nir_instr_insert_before(instr, &alu->instr);
nir_alu_src val;
val.src = nir_src_for_ssa(&alu->dest.dest.ssa);
val.negate = false;
val.abs = false,
memcpy(val.swizzle, identity_swizzle, sizeof val.swizzle);
return val;
}
case nir_search_value_variable: {
const nir_search_variable *var = nir_search_value_as_variable(value);
assert(state->variables_seen & (1 << var->variable));
nir_alu_src val = { NIR_SRC_INIT };
nir_alu_src_copy(&val, &state->variables[var->variable], mem_ctx);
assert(!var->is_constant);
return val;
}
case nir_search_value_constant: {
const nir_search_constant *c = nir_search_value_as_constant(value);
nir_load_const_instr *load = nir_load_const_instr_create(mem_ctx, 1);
switch (type) {
case nir_type_float:
load->def.name = ralloc_asprintf(mem_ctx, "%f", c->data.f);
load->value.f[0] = c->data.f;
break;
case nir_type_int:
load->def.name = ralloc_asprintf(mem_ctx, "%d", c->data.i);
load->value.i[0] = c->data.i;
break;
case nir_type_unsigned:
case nir_type_bool:
load->value.u[0] = c->data.u;
break;
default:
unreachable("Invalid alu source type");
}
nir_instr_insert_before(instr, &load->instr);
nir_alu_src val;
val.src = nir_src_for_ssa(&load->def);
val.negate = false;
val.abs = false,
memset(val.swizzle, 0, sizeof val.swizzle);
return val;
}
default:
unreachable("Invalid search value type");
}
}