static void
lower_load_store(nir_builder *b,
nir_intrinsic_instr *intrin,
glsl_type_size_align_func size_align)
{
b->cursor = nir_before_instr(&intrin->instr);
nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
nir_variable *var = nir_deref_instr_get_variable(deref);
nir_ssa_def *offset =
nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
var->data.location);
unsigned align, UNUSED size;
size_align(deref->type, &size, &align);
if (intrin->intrinsic == nir_intrinsic_load_deref) {
nir_intrinsic_instr *load =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
load->num_components = intrin->num_components;
load->src[0] = nir_src_for_ssa(offset);
nir_intrinsic_set_align(load, align, 0);
nir_ssa_dest_init(&load->instr, &load->dest,
intrin->dest.ssa.num_components,
intrin->dest.ssa.bit_size, NULL);
nir_builder_instr_insert(b, &load->instr);
nir_ssa_def *value = &load->dest.ssa;
if (glsl_type_is_boolean(deref->type))
value = nir_b2i32(b, value);
nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
nir_src_for_ssa(&load->dest.ssa));
} else {
assert(intrin->intrinsic == nir_intrinsic_store_deref);
assert(intrin->src[1].is_ssa);
nir_ssa_def *value = intrin->src[1].ssa;
if (glsl_type_is_boolean(deref->type))
value = nir_i2b(b, value);
nir_intrinsic_instr *store =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
store->num_components = intrin->num_components;
store->src[0] = nir_src_for_ssa(value);
store->src[1] = nir_src_for_ssa(offset);
nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
nir_intrinsic_set_align(store, align, 0);
nir_builder_instr_insert(b, &store->instr);
}
nir_instr_remove(&intrin->instr);
nir_deref_instr_remove_if_unused(deref);
}
static nir_ssa_def *
get_texture_size(nir_builder *b, nir_tex_instr *tex)
{
b->cursor = nir_before_instr(&tex->instr);
/* RECT textures should not be array: */
assert(!tex->is_array);
nir_tex_instr *txs;
txs = nir_tex_instr_create(b->shader, 1);
txs->op = nir_texop_txs;
txs->sampler_dim = GLSL_SAMPLER_DIM_RECT;
txs->texture_index = tex->texture_index;
txs->dest_type = nir_type_int;
/* only single src, the lod: */
txs->src[0].src = nir_src_for_ssa(nir_imm_int(b, 0));
txs->src[0].src_type = nir_tex_src_lod;
nir_ssa_dest_init(&txs->instr, &txs->dest, 2, 32, NULL);
nir_builder_instr_insert(b, &txs->instr);
return nir_i2f(b, &txs->dest.ssa);
}
static nir_ssa_def *
sample_plane(nir_builder *b, nir_tex_instr *tex, int plane)
{
assert(tex->dest.is_ssa);
assert(nir_tex_instr_dest_size(tex) == 4);
assert(nir_alu_type_get_base_type(tex->dest_type) == nir_type_float);
assert(tex->op == nir_texop_tex);
assert(tex->coord_components == 2);
nir_tex_instr *plane_tex =
nir_tex_instr_create(b->shader, tex->num_srcs + 1);
for (unsigned i = 0; i < tex->num_srcs; i++) {
nir_src_copy(&plane_tex->src[i].src, &tex->src[i].src, plane_tex);
plane_tex->src[i].src_type = tex->src[i].src_type;
}
plane_tex->src[tex->num_srcs].src = nir_src_for_ssa(nir_imm_int(b, plane));
plane_tex->src[tex->num_srcs].src_type = nir_tex_src_plane;
plane_tex->op = nir_texop_tex;
plane_tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
plane_tex->dest_type = nir_type_float;
plane_tex->coord_components = 2;
plane_tex->texture_index = tex->texture_index;
plane_tex->sampler_index = tex->sampler_index;
nir_ssa_dest_init(&plane_tex->instr, &plane_tex->dest, 4, 32, NULL);
nir_builder_instr_insert(b, &plane_tex->instr);
return &plane_tex->dest.ssa;
}
static bool
repair_ssa_def(nir_ssa_def *def, void *void_state)
{
struct repair_ssa_state *state = void_state;
bool is_valid = true;
nir_foreach_use(def, src) {
if (!nir_block_dominates(def->parent_instr->block, get_src_block(src))) {
is_valid = false;
break;
}
}
if (is_valid)
return true;
struct nir_phi_builder *pb = prep_build_phi(state);
BITSET_SET(state->def_set, def->parent_instr->block->index);
struct nir_phi_builder_value *val =
nir_phi_builder_add_value(pb, def->num_components, def->bit_size,
state->def_set);
nir_phi_builder_value_set_block_def(val, def->parent_instr->block, def);
nir_foreach_use_safe(def, src) {
nir_block *src_block = get_src_block(src);
if (!nir_block_dominates(def->parent_instr->block, src_block)) {
nir_instr_rewrite_src(src->parent_instr, src, nir_src_for_ssa(
nir_phi_builder_value_get_block_def(val, src_block)));
}
}
static nir_ssa_def *
sample_plane(nir_builder *b, nir_tex_instr *tex, int plane)
{
assert(tex->dest.is_ssa);
assert(nir_tex_instr_dest_size(tex) == 4);
assert(nir_alu_type_get_base_type(tex->dest_type) == nir_type_float);
assert(tex->op == nir_texop_tex);
assert(tex->coord_components == 2);
nir_tex_instr *plane_tex = nir_tex_instr_create(b->shader, 2);
nir_src_copy(&plane_tex->src[0].src, &tex->src[0].src, plane_tex);
plane_tex->src[0].src_type = nir_tex_src_coord;
plane_tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, plane));
plane_tex->src[1].src_type = nir_tex_src_plane;
plane_tex->op = nir_texop_tex;
plane_tex->sampler_dim = 2;
plane_tex->dest_type = nir_type_float;
plane_tex->coord_components = 2;
plane_tex->texture_index = tex->texture_index;
plane_tex->texture = (nir_deref_var *)
nir_copy_deref(plane_tex, &tex->texture->deref);
plane_tex->sampler_index = tex->sampler_index;
plane_tex->sampler = (nir_deref_var *)
nir_copy_deref(plane_tex, &tex->sampler->deref);
nir_ssa_dest_init(&plane_tex->instr, &plane_tex->dest, 4, 32, NULL);
nir_builder_instr_insert(b, &plane_tex->instr);
return &plane_tex->dest.ssa;
}
static void
lower_load_const_instr_scalar(nir_load_const_instr *lower)
{
if (lower->def.num_components == 1)
return;
nir_builder b;
nir_builder_init(&b, nir_cf_node_get_function(&lower->instr.block->cf_node));
b.cursor = nir_before_instr(&lower->instr);
/* Emit the individual loads. */
nir_ssa_def *loads[4];
for (unsigned i = 0; i < lower->def.num_components; i++) {
nir_load_const_instr *load_comp = nir_load_const_instr_create(b.shader, 1);
load_comp->value.u[0] = lower->value.u[i];
nir_builder_instr_insert(&b, &load_comp->instr);
loads[i] = &load_comp->def;
}
/* Batch things back together into a vector. */
nir_ssa_def *vec = nir_vec(&b, loads, lower->def.num_components);
/* Replace the old load with a reference to our reconstructed vector. */
nir_ssa_def_rewrite_uses(&lower->def, nir_src_for_ssa(vec));
nir_instr_remove(&lower->instr);
}
static nir_ssa_def *
get_texture_size(nir_builder *b, nir_tex_instr *tex)
{
b->cursor = nir_before_instr(&tex->instr);
nir_tex_instr *txs;
txs = nir_tex_instr_create(b->shader, 1);
txs->op = nir_texop_txs;
txs->sampler_dim = tex->sampler_dim;
txs->is_array = tex->is_array;
txs->is_shadow = tex->is_shadow;
txs->is_new_style_shadow = tex->is_new_style_shadow;
txs->texture_index = tex->texture_index;
txs->texture = nir_deref_var_clone(tex->texture, txs);
txs->sampler_index = tex->sampler_index;
txs->sampler = nir_deref_var_clone(tex->sampler, txs);
txs->dest_type = nir_type_int;
/* only single src, the lod: */
txs->src[0].src = nir_src_for_ssa(nir_imm_int(b, 0));
txs->src[0].src_type = nir_tex_src_lod;
nir_ssa_dest_init(&txs->instr, &txs->dest, tex->coord_components, 32, NULL);
nir_builder_instr_insert(b, &txs->instr);
return nir_i2f(b, &txs->dest.ssa);
}
static nir_ssa_def *
get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain,
struct vtn_type **type, unsigned *chain_idx)
{
/* Push constants have no explicit binding */
if (chain->var->mode == vtn_variable_mode_push_constant) {
*chain_idx = 0;
*type = chain->var->type;
return NULL;
}
nir_ssa_def *array_index;
if (glsl_type_is_array(chain->var->type->type)) {
assert(chain->length > 0);
array_index = vtn_access_link_as_ssa(b, chain->link[0], 1);
*chain_idx = 1;
*type = chain->var->type->array_element;
} else {
array_index = nir_imm_int(&b->nb, 0);
*chain_idx = 0;
*type = chain->var->type;
}
nir_intrinsic_instr *instr =
nir_intrinsic_instr_create(b->nb.shader,
nir_intrinsic_vulkan_resource_index);
instr->src[0] = nir_src_for_ssa(array_index);
nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set);
nir_intrinsic_set_binding(instr, chain->var->binding);
nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
nir_builder_instr_insert(&b->nb, &instr->instr);
return &instr->dest.ssa;
}
static void
lower_implicit_lod(nir_builder *b, nir_tex_instr *tex)
{
assert(tex->op == nir_texop_tex || tex->op == nir_texop_txb);
assert(nir_tex_instr_src_index(tex, nir_tex_src_lod) < 0);
assert(nir_tex_instr_src_index(tex, nir_tex_src_ddx) < 0);
assert(nir_tex_instr_src_index(tex, nir_tex_src_ddy) < 0);
b->cursor = nir_before_instr(&tex->instr);
nir_ssa_def *lod = get_texture_lod(b, tex);
int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias);
if (bias_idx >= 0) {
/* If we have a bias, add it in */
lod = nir_fadd(b, lod, nir_ssa_for_src(b, tex->src[bias_idx].src, 1));
nir_tex_instr_remove_src(tex, bias_idx);
}
int min_lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod);
if (min_lod_idx >= 0) {
/* If we have a minimum LOD, clamp LOD accordingly */
lod = nir_fmax(b, lod, nir_ssa_for_src(b, tex->src[min_lod_idx].src, 1));
nir_tex_instr_remove_src(tex, min_lod_idx);
}
nir_tex_instr_add_src(tex, nir_tex_src_lod, nir_src_for_ssa(lod));
tex->op = nir_texop_txl;
}
static bool
constant_fold_deref(nir_instr *instr, nir_deref_var *deref)
{
bool progress = false;
for (nir_deref *tail = deref->deref.child; tail; tail = tail->child) {
if (tail->deref_type != nir_deref_type_array)
continue;
nir_deref_array *arr = nir_deref_as_array(tail);
if (arr->deref_array_type == nir_deref_array_type_indirect &&
arr->indirect.is_ssa &&
arr->indirect.ssa->parent_instr->type == nir_instr_type_load_const) {
nir_load_const_instr *indirect =
nir_instr_as_load_const(arr->indirect.ssa->parent_instr);
arr->base_offset += indirect->value.u[0];
/* Clear out the source */
nir_instr_rewrite_src(instr, &arr->indirect, nir_src_for_ssa(NULL));
arr->deref_array_type = nir_deref_array_type_direct;
progress = true;
}
}
return progress;
}
static void
vtn_build_subgroup_instr(struct vtn_builder *b,
nir_intrinsic_op nir_op,
struct vtn_ssa_value *dst,
struct vtn_ssa_value *src0,
nir_ssa_def *index,
unsigned const_idx0,
unsigned const_idx1)
{
/* Some of the subgroup operations take an index. SPIR-V allows this to be
* any integer type. To make things simpler for drivers, we only support
* 32-bit indices.
*/
if (index && index->bit_size != 32)
index = nir_u2u32(&b->nb, index);
vtn_assert(dst->type == src0->type);
if (!glsl_type_is_vector_or_scalar(dst->type)) {
for (unsigned i = 0; i < glsl_get_length(dst->type); i++) {
vtn_build_subgroup_instr(b, nir_op, dst->elems[i],
src0->elems[i], index,
const_idx0, const_idx1);
}
return;
}
nir_intrinsic_instr *intrin =
nir_intrinsic_instr_create(b->nb.shader, nir_op);
nir_ssa_dest_init_for_type(&intrin->instr, &intrin->dest,
dst->type, NULL);
intrin->num_components = intrin->dest.ssa.num_components;
intrin->src[0] = nir_src_for_ssa(src0->def);
if (index)
intrin->src[1] = nir_src_for_ssa(index);
intrin->const_index[0] = const_idx0;
intrin->const_index[1] = const_idx1;
nir_builder_instr_insert(&b->nb, &intrin->instr);
dst->def = &intrin->dest.ssa;
}
static bool
remove_phis_block(nir_block *block, void *state)
{
bool *progress = state;
nir_foreach_instr_safe(block, instr) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_ssa_def *def = NULL;
bool srcs_same = true;
nir_foreach_phi_src(phi, src) {
assert(src->src.is_ssa);
/* For phi nodes at the beginning of loops, we may encounter some
* sources from backedges that point back to the destination of the
* same phi, i.e. something like:
*
* a = phi(a, b, ...)
*
* We can safely ignore these sources, since if all of the normal
* sources point to the same definition, then that definition must
* still dominate the phi node, and the phi will still always take
* the value of that definition.
*/
if (src->src.ssa == &phi->dest.ssa)
continue;
if (def == NULL) {
def = src->src.ssa;
} else {
if (src->src.ssa != def) {
srcs_same = false;
break;
}
}
}
if (!srcs_same)
continue;
/* We must have found at least one definition, since there must be at
* least one forward edge.
*/
assert(def != NULL);
assert(phi->dest.is_ssa);
nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
nir_instr_remove(instr);
*progress = true;
}
static void
build_color_shaders(struct nir_shader **out_vs,
struct nir_shader **out_fs,
uint32_t frag_output)
{
nir_builder vs_b;
nir_builder fs_b;
nir_builder_init_simple_shader(&vs_b, NULL, MESA_SHADER_VERTEX, NULL);
nir_builder_init_simple_shader(&fs_b, NULL, MESA_SHADER_FRAGMENT, NULL);
vs_b.shader->info.name = ralloc_strdup(vs_b.shader, "meta_clear_color_vs");
fs_b.shader->info.name = ralloc_strdup(fs_b.shader, "meta_clear_color_fs");
const struct glsl_type *position_type = glsl_vec4_type();
const struct glsl_type *color_type = glsl_vec4_type();
nir_variable *vs_out_pos =
nir_variable_create(vs_b.shader, nir_var_shader_out, position_type,
"gl_Position");
vs_out_pos->data.location = VARYING_SLOT_POS;
nir_intrinsic_instr *in_color_load = nir_intrinsic_instr_create(fs_b.shader, nir_intrinsic_load_push_constant);
nir_intrinsic_set_base(in_color_load, 0);
nir_intrinsic_set_range(in_color_load, 16);
in_color_load->src[0] = nir_src_for_ssa(nir_imm_int(&fs_b, 0));
in_color_load->num_components = 4;
nir_ssa_dest_init(&in_color_load->instr, &in_color_load->dest, 4, 32, "clear color");
nir_builder_instr_insert(&fs_b, &in_color_load->instr);
nir_variable *fs_out_color =
nir_variable_create(fs_b.shader, nir_var_shader_out, color_type,
"f_color");
fs_out_color->data.location = FRAG_RESULT_DATA0 + frag_output;
nir_store_var(&fs_b, fs_out_color, &in_color_load->dest.ssa, 0xf);
nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&vs_b);
nir_store_var(&vs_b, vs_out_pos, outvec, 0xf);
const struct glsl_type *layer_type = glsl_int_type();
nir_variable *vs_out_layer =
nir_variable_create(vs_b.shader, nir_var_shader_out, layer_type,
"v_layer");
vs_out_layer->data.location = VARYING_SLOT_LAYER;
vs_out_layer->data.interpolation = INTERP_MODE_FLAT;
nir_ssa_def *inst_id = nir_load_system_value(&vs_b, nir_intrinsic_load_instance_id, 0);
nir_ssa_def *base_instance = nir_load_system_value(&vs_b, nir_intrinsic_load_base_instance, 0);
nir_ssa_def *layer_id = nir_iadd(&vs_b, inst_id, base_instance);
nir_store_var(&vs_b, vs_out_layer, layer_id, 0x1);
*out_vs = vs_b.shader;
*out_fs = fs_b.shader;
}
static bool
lower_offset(nir_builder *b, nir_tex_instr *tex)
{
int offset_index = nir_tex_instr_src_index(tex, nir_tex_src_offset);
if (offset_index < 0)
return false;
int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord);
assert(coord_index >= 0);
assert(tex->src[offset_index].src.is_ssa);
assert(tex->src[coord_index].src.is_ssa);
nir_ssa_def *offset = tex->src[offset_index].src.ssa;
nir_ssa_def *coord = tex->src[coord_index].src.ssa;
b->cursor = nir_before_instr(&tex->instr);
nir_ssa_def *offset_coord;
if (nir_tex_instr_src_type(tex, coord_index) == nir_type_float) {
if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) {
offset_coord = nir_fadd(b, coord, nir_i2f32(b, offset));
} else {
nir_ssa_def *txs = get_texture_size(b, tex);
nir_ssa_def *scale = nir_frcp(b, txs);
offset_coord = nir_fadd(b, coord,
nir_fmul(b,
nir_i2f32(b, offset),
scale));
}
} else {
offset_coord = nir_iadd(b, coord, offset);
}
if (tex->is_array) {
/* The offset is not applied to the array index */
if (tex->coord_components == 2) {
offset_coord = nir_vec2(b, nir_channel(b, offset_coord, 0),
nir_channel(b, coord, 1));
} else if (tex->coord_components == 3) {
offset_coord = nir_vec3(b, nir_channel(b, offset_coord, 0),
nir_channel(b, offset_coord, 1),
nir_channel(b, coord, 2));
} else {
unreachable("Invalid number of components");
}
}
nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src,
nir_src_for_ssa(offset_coord));
nir_tex_instr_remove_src(tex, offset_index);
return true;
}
static void
lower_sampler(nir_tex_instr *instr, const struct gl_shader_program *shader_program,
gl_shader_stage stage, nir_builder *builder)
{
if (instr->sampler == NULL)
return;
instr->sampler_index = 0;
unsigned location = instr->sampler->var->data.location;
unsigned array_elements = 1;
nir_ssa_def *indirect = NULL;
builder->cursor = nir_before_instr(&instr->instr);
calc_sampler_offsets(&instr->sampler->deref, instr, &array_elements,
&indirect, builder, &location);
if (indirect) {
/* First, we have to resize the array of texture sources */
nir_tex_src *new_srcs = rzalloc_array(instr, nir_tex_src,
instr->num_srcs + 1);
for (unsigned i = 0; i < instr->num_srcs; i++) {
new_srcs[i].src_type = instr->src[i].src_type;
nir_instr_move_src(&instr->instr, &new_srcs[i].src,
&instr->src[i].src);
}
ralloc_free(instr->src);
instr->src = new_srcs;
/* Now we can go ahead and move the source over to being a
* first-class texture source.
*/
instr->src[instr->num_srcs].src_type = nir_tex_src_sampler_offset;
instr->num_srcs++;
nir_instr_rewrite_src(&instr->instr,
&instr->src[instr->num_srcs - 1].src,
nir_src_for_ssa(indirect));
instr->sampler_array_size = array_elements;
}
if (location > shader_program->NumUniformStorage - 1 ||
!shader_program->UniformStorage[location].opaque[stage].active) {
assert(!"cannot return a sampler");
return;
}
instr->sampler_index +=
shader_program->UniformStorage[location].opaque[stage].index;
instr->sampler = NULL;
}
static void
opt_constant_if(nir_if *if_stmt, bool condition)
{
/* First, we need to remove any phi nodes after the if by rewriting uses to
* point to the correct source.
*/
nir_block *after = nir_cf_node_as_block(nir_cf_node_next(&if_stmt->cf_node));
nir_block *last_block =
nir_cf_node_as_block(condition ? nir_if_last_then_node(if_stmt)
: nir_if_last_else_node(if_stmt));
nir_foreach_instr_safe(after, instr) {
if (instr->type != nir_instr_type_phi)
break;
nir_phi_instr *phi = nir_instr_as_phi(instr);
nir_ssa_def *def = NULL;
nir_foreach_phi_src(phi, phi_src) {
if (phi_src->pred != last_block)
continue;
assert(phi_src->src.is_ssa);
def = phi_src->src.ssa;
}
assert(def);
assert(phi->dest.is_ssa);
nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
nir_instr_remove(instr);
}
/* The control flow list we're about to paste in may include a jump at the
* end, and in that case we have to delete the rest of the control flow
* list after the if since it's unreachable and the validator will balk if
* we don't.
*/
if (!exec_list_is_empty(&last_block->instr_list)) {
nir_instr *last_instr = nir_block_last_instr(last_block);
if (last_instr->type == nir_instr_type_jump)
remove_after_cf_node(&if_stmt->cf_node);
}
/* Finally, actually paste in the then or else branch and delete the if. */
struct exec_list *cf_list = condition ? &if_stmt->then_list
: &if_stmt->else_list;
nir_cf_list list;
nir_cf_extract(&list, nir_before_cf_list(cf_list),
nir_after_cf_list(cf_list));
nir_cf_reinsert(&list, nir_after_cf_node(&if_stmt->cf_node));
nir_cf_node_remove(&if_stmt->cf_node);
}
static void
_vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
nir_ssa_def *index, nir_ssa_def *offset,
struct vtn_ssa_value **inout, const struct glsl_type *type)
{
nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
instr->num_components = glsl_get_vector_elements(type);
int src = 0;
if (!load) {
nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
instr->src[src++] = nir_src_for_ssa((*inout)->def);
}
/* We set the base and size for push constant load to the entire push
* constant block for now.
*/
if (op == nir_intrinsic_load_push_constant) {
nir_intrinsic_set_base(instr, 0);
nir_intrinsic_set_range(instr, 128);
}
if (index)
instr->src[src++] = nir_src_for_ssa(index);
instr->src[src++] = nir_src_for_ssa(offset);
if (load) {
nir_ssa_dest_init(&instr->instr, &instr->dest,
instr->num_components,
glsl_get_bit_size(glsl_get_base_type(type)), NULL);
(*inout)->def = &instr->dest.ssa;
}
nir_builder_instr_insert(&b->nb, &instr->instr);
if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
(*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
}
static nir_ssa_def *
get_texture_size(nir_builder *b, nir_tex_instr *tex)
{
b->cursor = nir_before_instr(&tex->instr);
nir_tex_instr *txs;
unsigned num_srcs = 1; /* One for the LOD */
for (unsigned i = 0; i < tex->num_srcs; i++) {
if (tex->src[i].src_type == nir_tex_src_texture_deref ||
tex->src[i].src_type == nir_tex_src_sampler_deref ||
tex->src[i].src_type == nir_tex_src_texture_offset ||
tex->src[i].src_type == nir_tex_src_sampler_offset ||
tex->src[i].src_type == nir_tex_src_texture_handle ||
tex->src[i].src_type == nir_tex_src_sampler_handle)
num_srcs++;
}
txs = nir_tex_instr_create(b->shader, num_srcs);
txs->op = nir_texop_txs;
txs->sampler_dim = tex->sampler_dim;
txs->is_array = tex->is_array;
txs->is_shadow = tex->is_shadow;
txs->is_new_style_shadow = tex->is_new_style_shadow;
txs->texture_index = tex->texture_index;
txs->sampler_index = tex->sampler_index;
txs->dest_type = nir_type_int;
unsigned idx = 0;
for (unsigned i = 0; i < tex->num_srcs; i++) {
if (tex->src[i].src_type == nir_tex_src_texture_deref ||
tex->src[i].src_type == nir_tex_src_sampler_deref ||
tex->src[i].src_type == nir_tex_src_texture_offset ||
tex->src[i].src_type == nir_tex_src_sampler_offset ||
tex->src[i].src_type == nir_tex_src_texture_handle ||
tex->src[i].src_type == nir_tex_src_sampler_handle) {
nir_src_copy(&txs->src[idx].src, &tex->src[i].src, txs);
txs->src[idx].src_type = tex->src[i].src_type;
idx++;
}
}
/* Add in an LOD because some back-ends require it */
txs->src[idx].src = nir_src_for_ssa(nir_imm_int(b, 0));
txs->src[idx].src_type = nir_tex_src_lod;
nir_ssa_dest_init(&txs->instr, &txs->dest,
nir_tex_instr_dest_size(txs), 32, NULL);
nir_builder_instr_insert(b, &txs->instr);
return nir_i2f32(b, &txs->dest.ssa);
}
static void
lower_reduction(nir_alu_instr *instr, nir_op chan_op, nir_op merge_op,
void *mem_ctx)
{
unsigned num_components = nir_op_infos[instr->op].input_sizes[0];
nir_ssa_def *last = NULL;
for (unsigned i = 0; i < num_components; i++) {
nir_alu_instr *chan = nir_alu_instr_create(mem_ctx, chan_op);
nir_alu_ssa_dest_init(chan, 1);
nir_alu_src_copy(&chan->src[0], &instr->src[0], mem_ctx);
chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
if (nir_op_infos[chan_op].num_inputs > 1) {
assert(nir_op_infos[chan_op].num_inputs == 2);
nir_alu_src_copy(&chan->src[1], &instr->src[1], mem_ctx);
chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
}
nir_instr_insert_before(&instr->instr, &chan->instr);
if (i == 0) {
last = &chan->dest.dest.ssa;
} else {
nir_alu_instr *merge = nir_alu_instr_create(mem_ctx, merge_op);
nir_alu_ssa_dest_init(merge, 1);
merge->dest.write_mask = 1;
merge->src[0].src = nir_src_for_ssa(last);
merge->src[1].src = nir_src_for_ssa(&chan->dest.dest.ssa);
nir_instr_insert_before(&instr->instr, &merge->instr);
last = &merge->dest.dest.ssa;
}
}
assert(instr->dest.write_mask == 1);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(last),
mem_ctx);
nir_instr_remove(&instr->instr);
}
static bool
constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx)
{
nir_const_value src[4];
if (!instr->dest.dest.is_ssa)
return false;
for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
if (!instr->src[i].src.is_ssa)
return false;
nir_instr *src_instr = instr->src[i].src.ssa->parent_instr;
if (src_instr->type != nir_instr_type_load_const)
return false;
nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr);
for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i);
j++) {
src[i].u[j] = load_const->value.u[instr->src[i].swizzle[j]];
}
/* We shouldn't have any source modifiers in the optimization loop. */
assert(!instr->src[i].abs && !instr->src[i].negate);
}
/* We shouldn't have any saturate modifiers in the optimization loop. */
assert(!instr->dest.saturate);
nir_const_value dest =
nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components,
src);
nir_load_const_instr *new_instr =
nir_load_const_instr_create(mem_ctx,
instr->dest.dest.ssa.num_components);
new_instr->value = dest;
nir_instr_insert_before(&instr->instr, &new_instr->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(&new_instr->def),
mem_ctx);
nir_instr_remove(&instr->instr);
ralloc_free(instr);
return true;
}
static nir_src
get_deref_reg_src(nir_deref_instr *deref, struct locals_to_regs_state *state)
{
nir_builder *b = &state->builder;
nir_src src;
src.is_ssa = false;
src.reg.reg = get_reg_for_deref(deref, state);
src.reg.base_offset = 0;
src.reg.indirect = NULL;
/* It is possible for a user to create a shader that has an array with a
* single element and then proceed to access it indirectly. Indirectly
* accessing a non-array register is not allowed in NIR. In order to
* handle this case we just convert it to a direct reference.
*/
if (src.reg.reg->num_array_elems == 0)
return src;
unsigned inner_array_size = 1;
for (const nir_deref_instr *d = deref; d; d = nir_deref_instr_parent(d)) {
if (d->deref_type != nir_deref_type_array)
continue;
if (nir_src_is_const(d->arr.index) && !src.reg.indirect) {
src.reg.base_offset += nir_src_as_uint(d->arr.index) *
inner_array_size;
} else {
if (src.reg.indirect) {
assert(src.reg.base_offset == 0);
} else {
src.reg.indirect = ralloc(b->shader, nir_src);
*src.reg.indirect =
nir_src_for_ssa(nir_imm_int(b, src.reg.base_offset));
src.reg.base_offset = 0;
}
assert(src.reg.indirect->is_ssa);
nir_ssa_def *index = nir_i2i(b, nir_ssa_for_src(b, d->arr.index, 1), 32);
src.reg.indirect->ssa =
nir_iadd(b, src.reg.indirect->ssa,
nir_imul(b, index, nir_imm_int(b, inner_array_size)));
}
inner_array_size *= glsl_get_length(nir_deref_instr_parent(d)->type);
}
return src;
}
static void
handle_glsl450_interpolation(struct vtn_builder *b, enum GLSLstd450 opcode,
const uint32_t *w, unsigned count)
{
const struct glsl_type *dest_type =
vtn_value(b, w[1], vtn_value_type_type)->type->type;
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
val->ssa = vtn_create_ssa_value(b, dest_type);
nir_intrinsic_op op;
switch (opcode) {
case GLSLstd450InterpolateAtCentroid:
op = nir_intrinsic_interp_var_at_centroid;
break;
case GLSLstd450InterpolateAtSample:
op = nir_intrinsic_interp_var_at_sample;
break;
case GLSLstd450InterpolateAtOffset:
op = nir_intrinsic_interp_var_at_offset;
break;
default:
unreachable("Invalid opcode");
}
nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->nb.shader, op);
nir_deref_var *deref = vtn_nir_deref(b, w[5]);
intrin->variables[0] =
nir_deref_as_var(nir_copy_deref(intrin, &deref->deref));
switch (opcode) {
case GLSLstd450InterpolateAtCentroid:
break;
case GLSLstd450InterpolateAtSample:
case GLSLstd450InterpolateAtOffset:
intrin->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[6])->def);
break;
default:
unreachable("Invalid opcode");
}
intrin->num_components = glsl_get_vector_elements(dest_type);
nir_ssa_dest_init(&intrin->instr, &intrin->dest,
glsl_get_vector_elements(dest_type),
glsl_get_bit_size(dest_type), NULL);
val->ssa->def = &intrin->dest.ssa;
nir_builder_instr_insert(&b->nb, &intrin->instr);
}
static nir_shader *
build_resolve_fragment_shader(struct radv_device *dev, bool is_integer, int samples)
{
nir_builder b;
char name[64];
const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
const struct glsl_type *vec4 = glsl_vec4_type();
const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_MS,
false,
false,
GLSL_TYPE_FLOAT);
snprintf(name, 64, "meta_resolve_fs-%d-%s", samples, is_integer ? "int" : "float");
nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
b.shader->info.name = ralloc_strdup(b.shader, name);
nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform,
sampler_type, "s_tex");
input_img->data.descriptor_set = 0;
input_img->data.binding = 0;
nir_variable *fs_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "fs_pos_in");
fs_pos_in->data.location = VARYING_SLOT_POS;
nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
vec4, "f_color");
color_out->data.location = FRAG_RESULT_DATA0;
nir_ssa_def *pos_in = nir_load_var(&b, fs_pos_in);
nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant);
nir_intrinsic_set_base(src_offset, 0);
nir_intrinsic_set_range(src_offset, 8);
src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
src_offset->num_components = 2;
nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset");
nir_builder_instr_insert(&b, &src_offset->instr);
nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3);
nir_variable *color = nir_local_variable_create(b.impl, glsl_vec4_type(), "color");
radv_meta_build_resolve_shader_core(&b, is_integer, samples, input_img,
color, img_coord);
nir_ssa_def *outval = nir_load_var(&b, color);
nir_store_var(&b, color_out, outval, 0xf);
return b.shader;
}
static void
emit_load_store(nir_builder *b, nir_intrinsic_instr *orig_instr,
nir_deref_var *deref, nir_deref *tail,
nir_ssa_def **dest, nir_ssa_def *src)
{
for (; tail->child; tail = tail->child) {
if (tail->child->deref_type != nir_deref_type_array)
continue;
nir_deref_array *arr = nir_deref_as_array(tail->child);
if (arr->deref_array_type != nir_deref_array_type_indirect)
continue;
int length = glsl_get_length(tail->type);
emit_indirect_load_store(b, orig_instr, deref, tail, -arr->base_offset,
length - arr->base_offset, dest, src);
return;
}
assert(tail && tail->child == NULL);
/* We reached the end of the deref chain. Emit the instruction */
if (src == NULL) {
/* This is a load instruction */
nir_intrinsic_instr *load =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_var);
load->num_components = orig_instr->num_components;
load->variables[0] =
nir_deref_as_var(nir_copy_deref(load, &deref->deref));
unsigned bit_size = orig_instr->dest.ssa.bit_size;
nir_ssa_dest_init(&load->instr, &load->dest,
load->num_components, bit_size, NULL);
nir_builder_instr_insert(b, &load->instr);
*dest = &load->dest.ssa;
} else {
/* This is a store instruction */
nir_intrinsic_instr *store =
nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_var);
store->num_components = orig_instr->num_components;
nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(orig_instr));
store->variables[0] =
nir_deref_as_var(nir_copy_deref(store, &deref->deref));
store->src[0] = nir_src_for_ssa(src);
nir_builder_instr_insert(b, &store->instr);
}
}
static void
saturate_src(nir_builder *b, nir_tex_instr *tex, unsigned sat_mask)
{
b->cursor = nir_before_instr(&tex->instr);
/* Walk through the sources saturating the requested arguments. */
for (unsigned i = 0; i < tex->num_srcs; i++) {
if (tex->src[i].src_type != nir_tex_src_coord)
continue;
nir_ssa_def *src =
nir_ssa_for_src(b, tex->src[i].src, tex->coord_components);
/* split src into components: */
nir_ssa_def *comp[4];
for (unsigned j = 0; j < tex->coord_components; j++)
comp[j] = nir_channel(b, src, j);
/* clamp requested components, array index does not get clamped: */
unsigned ncomp = tex->coord_components;
if (tex->is_array)
ncomp--;
for (unsigned j = 0; j < ncomp; j++) {
if ((1 << j) & sat_mask) {
if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) {
/* non-normalized texture coords, so clamp to texture
* size rather than [0.0, 1.0]
*/
nir_ssa_def *txs = get_texture_size(b, tex);
comp[j] = nir_fmax(b, comp[j], nir_imm_float(b, 0.0));
comp[j] = nir_fmin(b, comp[j], nir_channel(b, txs, j));
} else {
comp[j] = nir_fsat(b, comp[j]);
}
}
}
/* and move the result back into a single vecN: */
src = nir_vec(b, comp, tex->coord_components);
nir_instr_rewrite_src(&tex->instr,
&tex->src[i].src,
nir_src_for_ssa(src));
}
}
/* Multiply interp_var_at_offset's offset by transform.x to flip it. */
static void
lower_interp_var_at_offset(lower_wpos_ytransform_state *state,
nir_intrinsic_instr *interp)
{
nir_builder *b = &state->b;
nir_ssa_def *offset;
nir_ssa_def *flip_y;
b->cursor = nir_before_instr(&interp->instr);
offset = nir_ssa_for_src(b, interp->src[0], 2);
flip_y = nir_fmul(b, nir_channel(b, offset, 1),
nir_channel(b, get_transform(state), 0));
nir_instr_rewrite_src(&interp->instr, &interp->src[0],
nir_src_for_ssa(nir_vec2(b, nir_channel(b, offset, 0),
flip_y)));
}
nir_foreach_instr_safe(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
if ((mode == nir_var_shader_in && is_input(intrin)) ||
(mode == nir_var_shader_out && is_output(intrin))) {
nir_src *offset = nir_get_io_offset_src(intrin);
nir_const_value *const_offset = nir_src_as_const_value(*offset);
if (const_offset) {
intrin->const_index[0] += const_offset->u32[0];
b->cursor = nir_before_instr(&intrin->instr);
nir_instr_rewrite_src(&intrin->instr, offset,
nir_src_for_ssa(nir_imm_int(b, 0)));
}
}
}
nir_alu_instr *
nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search,
const nir_search_value *replace, void *mem_ctx)
{
uint8_t swizzle[4] = { 0, 0, 0, 0 };
for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i)
swizzle[i] = i;
assert(instr->dest.dest.is_ssa);
struct match_state state;
state.variables_seen = 0;
if (!match_expression(search, instr, instr->dest.dest.ssa.num_components,
swizzle, &state))
return NULL;
/* Inserting a mov may be unnecessary. However, it's much easier to
* simply let copy propagation clean this up than to try to go through
* and rewrite swizzles ourselves.
*/
nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov);
mov->dest.write_mask = instr->dest.write_mask;
nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
instr->dest.dest.ssa.num_components, NULL);
mov->src[0] = construct_value(replace, nir_op_infos[instr->op].output_type,
instr->dest.dest.ssa.num_components, &state,
&instr->instr, mem_ctx);
nir_instr_insert_before(&instr->instr, &mov->instr);
nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
nir_src_for_ssa(&mov->dest.dest.ssa), mem_ctx);
/* We know this one has no more uses because we just rewrote them all,
* so we can remove it. The rest of the matched expression, however, we
* don't know so much about. We'll just let dead code clean them up.
*/
nir_instr_remove(&instr->instr);
return mov;
}
static void
lower_load_sample_pos(lower_wpos_ytransform_state *state,
nir_intrinsic_instr *intr)
{
nir_builder *b = &state->b;
b->cursor = nir_after_instr(&intr->instr);
nir_ssa_def *pos = &intr->dest.ssa;
nir_ssa_def *scale = nir_channel(b, get_transform(state), 0);
nir_ssa_def *neg_scale = nir_channel(b, get_transform(state), 2);
/* Either y or 1-y for scale equal to 1 or -1 respectively. */
nir_ssa_def *flipped_y =
nir_fadd(b, nir_fmax(b, neg_scale, nir_imm_float(b, 0.0)),
nir_fmul(b, nir_channel(b, pos, 1), scale));
nir_ssa_def *flipped_pos = nir_vec2(b, nir_channel(b, pos, 0), flipped_y);
nir_ssa_def_rewrite_uses_after(&intr->dest.ssa, nir_src_for_ssa(flipped_pos),
flipped_pos->parent_instr);
}
/* turns 'fddy(p)' into 'fddy(fmul(p, transform.x))' */
static void
lower_fddy(lower_wpos_ytransform_state *state, nir_alu_instr *fddy)
{
nir_builder *b = &state->b;
nir_ssa_def *p, *pt, *trans;
b->cursor = nir_before_instr(&fddy->instr);
p = nir_ssa_for_alu_src(b, fddy, 0);
trans = get_transform(state);
pt = nir_fmul(b, p, nir_channel(b, trans, 0));
nir_instr_rewrite_src(&fddy->instr,
&fddy->src[0].src,
nir_src_for_ssa(pt));
for (unsigned i = 0; i < 4; i++)
fddy->src[0].swizzle[i] = MIN2(i, pt->num_components - 1);
}
