/* Recursively constructs deref chains to split a copy instruction into
* multiple (if needed) copy instructions with full-length deref chains.
* External callers of this function should pass the tail and head of the
* deref chains found as the source and destination of the copy instruction
* into this function.
*
* \param old_copy The copy instruction we are splitting
* \param dest_head The head of the destination deref chain we are building
* \param src_head The head of the source deref chain we are building
* \param dest_tail The tail of the destination deref chain we are building
* \param src_tail The tail of the source deref chain we are building
* \param state The current split_var_copies_state object
*/
static void
split_var_copy_instr(nir_intrinsic_instr *old_copy,
nir_deref *dest_head, nir_deref *src_head,
nir_deref *dest_tail, nir_deref *src_tail,
struct split_var_copies_state *state)
{
assert(src_tail->type == dest_tail->type);
/* Make sure these really are the tails of the deref chains */
assert(dest_tail->child == NULL);
assert(src_tail->child == NULL);
switch (glsl_get_base_type(src_tail->type)) {
case GLSL_TYPE_ARRAY: {
/* Make a wildcard dereference */
nir_deref_array *deref = nir_deref_array_create(state->dead_ctx);
deref->deref.type = glsl_get_array_element(src_tail->type);
deref->deref_array_type = nir_deref_array_type_wildcard;
/* Set the tail of both as the newly created wildcard deref. It is
* safe to use the same wildcard in both places because a) we will be
* copying it before we put it in an actual instruction and b)
* everything that will potentially add another link in the deref
* chain will also add the same thing to both chains.
*/
src_tail->child = &deref->deref;
dest_tail->child = &deref->deref;
split_var_copy_instr(old_copy, dest_head, src_head,
dest_tail->child, src_tail->child, state);
/* Set it back to the way we found it */
src_tail->child = NULL;
dest_tail->child = NULL;
break;
}
case GLSL_TYPE_STRUCT:
/* This is the only part that actually does any interesting
* splitting. For array types, we just use wildcards and resolve
* them later. For structure types, we need to emit one copy
* instruction for every structure element. Because we may have
* structs inside structs, we just recurse and let the next level
* take care of any additional structures.
*/
for (unsigned i = 0; i < glsl_get_length(src_tail->type); i++) {
nir_deref_struct *deref = nir_deref_struct_create(state->dead_ctx, i);
deref->deref.type = glsl_get_struct_field(src_tail->type, i);
/* Set the tail of both as the newly created structure deref. It
* is safe to use the same wildcard in both places because a) we
* will be copying it before we put it in an actual instruction
* and b) everything that will potentially add another link in the
* deref chain will also add the same thing to both chains.
*/
src_tail->child = &deref->deref;
dest_tail->child = &deref->deref;
split_var_copy_instr(old_copy, dest_head, src_head,
dest_tail->child, src_tail->child, state);
}
/* Set it back to the way we found it */
src_tail->child = NULL;
dest_tail->child = NULL;
break;
case GLSL_TYPE_UINT:
case GLSL_TYPE_INT:
case GLSL_TYPE_FLOAT:
case GLSL_TYPE_BOOL:
if (glsl_type_is_matrix(src_tail->type)) {
nir_deref_array *deref = nir_deref_array_create(state->dead_ctx);
deref->deref.type = glsl_get_column_type(src_tail->type);
deref->deref_array_type = nir_deref_array_type_wildcard;
/* Set the tail of both as the newly created wildcard deref. It
* is safe to use the same wildcard in both places because a) we
* will be copying it before we put it in an actual instruction
* and b) everything that will potentially add another link in the
* deref chain will also add the same thing to both chains.
*/
src_tail->child = &deref->deref;
dest_tail->child = &deref->deref;
split_var_copy_instr(old_copy, dest_head, src_head,
dest_tail->child, src_tail->child, state);
/* Set it back to the way we found it */
src_tail->child = NULL;
dest_tail->child = NULL;
} else {
/* At this point, we have fully built our deref chains and can
* actually add the new copy instruction.
*/
nir_intrinsic_instr *new_copy =
nir_intrinsic_instr_create(state->mem_ctx, nir_intrinsic_copy_var);
/* We need to make copies because a) this deref chain actually
* belongs to the copy instruction and b) the deref chains may
* have some of the same links due to the way we constructed them
*/
nir_deref *src = nir_copy_deref(new_copy, src_head);
nir_deref *dest = nir_copy_deref(new_copy, dest_head);
new_copy->variables[0] = nir_deref_as_var(dest);
new_copy->variables[1] = nir_deref_as_var(src);
/* Emit the copy instruction after the old instruction. We'll
* remove the old one later.
*/
nir_instr_insert_after(&old_copy->instr, &new_copy->instr);
state->progress = true;
}
break;
case GLSL_TYPE_SAMPLER:
case GLSL_TYPE_IMAGE:
case GLSL_TYPE_ATOMIC_UINT:
case GLSL_TYPE_INTERFACE:
default:
unreachable("Cannot copy these types");
}
}
static void
_vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
nir_ssa_def *index, nir_ssa_def *offset,
struct vtn_access_chain *chain, unsigned chain_idx,
struct vtn_type *type, struct vtn_ssa_value **inout)
{
if (chain && chain_idx >= chain->length)
chain = NULL;
if (load && chain == NULL && *inout == NULL)
*inout = vtn_create_ssa_value(b, type->type);
enum glsl_base_type base_type = glsl_get_base_type(type->type);
switch (base_type) {
case GLSL_TYPE_UINT:
case GLSL_TYPE_INT:
case GLSL_TYPE_FLOAT:
case GLSL_TYPE_BOOL:
/* This is where things get interesting. At this point, we've hit
* a vector, a scalar, or a matrix.
*/
if (glsl_type_is_matrix(type->type)) {
if (chain == NULL) {
/* Loading the whole matrix */
struct vtn_ssa_value *transpose;
unsigned num_ops, vec_width;
if (type->row_major) {
num_ops = glsl_get_vector_elements(type->type);
vec_width = glsl_get_matrix_columns(type->type);
if (load) {
const struct glsl_type *transpose_type =
glsl_matrix_type(base_type, vec_width, num_ops);
*inout = vtn_create_ssa_value(b, transpose_type);
} else {
transpose = vtn_ssa_transpose(b, *inout);
inout = &transpose;
}
} else {
num_ops = glsl_get_matrix_columns(type->type);
vec_width = glsl_get_vector_elements(type->type);
}
for (unsigned i = 0; i < num_ops; i++) {
nir_ssa_def *elem_offset =
nir_iadd(&b->nb, offset,
nir_imm_int(&b->nb, i * type->stride));
_vtn_load_store_tail(b, op, load, index, elem_offset,
&(*inout)->elems[i],
glsl_vector_type(base_type, vec_width));
}
if (load && type->row_major)
*inout = vtn_ssa_transpose(b, *inout);
} else if (type->row_major) {
/* Row-major but with an access chiain. */
nir_ssa_def *col_offset =
vtn_access_link_as_ssa(b, chain->link[chain_idx],
type->array_element->stride);
offset = nir_iadd(&b->nb, offset, col_offset);
if (chain_idx + 1 < chain->length) {
/* Picking off a single element */
nir_ssa_def *row_offset =
vtn_access_link_as_ssa(b, chain->link[chain_idx + 1],
type->stride);
offset = nir_iadd(&b->nb, offset, row_offset);
if (load)
*inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type));
_vtn_load_store_tail(b, op, load, index, offset, inout,
glsl_scalar_type(base_type));
} else {
/* Grabbing a column; picking one element off each row */
unsigned num_comps = glsl_get_vector_elements(type->type);
const struct glsl_type *column_type =
glsl_get_column_type(type->type);
nir_ssa_def *comps[4];
for (unsigned i = 0; i < num_comps; i++) {
nir_ssa_def *elem_offset =
nir_iadd(&b->nb, offset,
nir_imm_int(&b->nb, i * type->stride));
struct vtn_ssa_value *comp, temp_val;
if (!load) {
temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
temp_val.type = glsl_scalar_type(base_type);
}
comp = &temp_val;
_vtn_load_store_tail(b, op, load, index, elem_offset,
&comp, glsl_scalar_type(base_type));
comps[i] = comp->def;
}
if (load) {
if (*inout == NULL)
*inout = vtn_create_ssa_value(b, column_type);
(*inout)->def = nir_vec(&b->nb, comps, num_comps);
}
}
} else {
/* Column-major with a deref. Fall through to array case. */
nir_ssa_def *col_offset =
vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
offset = nir_iadd(&b->nb, offset, col_offset);
_vtn_block_load_store(b, op, load, index, offset,
chain, chain_idx + 1,
type->array_element, inout);
}
} else if (chain == NULL) {
/* Single whole vector */
assert(glsl_type_is_vector_or_scalar(type->type));
_vtn_load_store_tail(b, op, load, index, offset, inout, type->type);
} else {
/* Single component of a vector. Fall through to array case. */
nir_ssa_def *elem_offset =
vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
offset = nir_iadd(&b->nb, offset, elem_offset);
_vtn_block_load_store(b, op, load, index, offset, NULL, 0,
type->array_element, inout);
}
return;
case GLSL_TYPE_ARRAY: {
unsigned elems = glsl_get_length(type->type);
for (unsigned i = 0; i < elems; i++) {
nir_ssa_def *elem_off =
nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
_vtn_block_load_store(b, op, load, index, elem_off, NULL, 0,
type->array_element, &(*inout)->elems[i]);
}
return;
}
case GLSL_TYPE_STRUCT: {
unsigned elems = glsl_get_length(type->type);
for (unsigned i = 0; i < elems; i++) {
nir_ssa_def *elem_off =
nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
_vtn_block_load_store(b, op, load, index, elem_off, NULL, 0,
type->members[i], &(*inout)->elems[i]);
}
return;
}
default:
unreachable("Invalid block member type");
}
}
