static void lower_load_store(nir_builder *b, nir_intrinsic_instr *intrin, glsl_type_size_align_func size_align) { b->cursor = nir_before_instr(&intrin->instr); nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); nir_variable *var = nir_deref_instr_get_variable(deref); nir_ssa_def *offset = nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align), var->data.location); unsigned align, UNUSED size; size_align(deref->type, &size, &align); if (intrin->intrinsic == nir_intrinsic_load_deref) { nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch); load->num_components = intrin->num_components; load->src[0] = nir_src_for_ssa(offset); nir_intrinsic_set_align(load, align, 0); nir_ssa_dest_init(&load->instr, &load->dest, intrin->dest.ssa.num_components, intrin->dest.ssa.bit_size, NULL); nir_builder_instr_insert(b, &load->instr); nir_ssa_def *value = &load->dest.ssa; if (glsl_type_is_boolean(deref->type)) value = nir_b2i32(b, value); nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(&load->dest.ssa)); } else { assert(intrin->intrinsic == nir_intrinsic_store_deref); assert(intrin->src[1].is_ssa); nir_ssa_def *value = intrin->src[1].ssa; if (glsl_type_is_boolean(deref->type)) value = nir_i2b(b, value); nir_intrinsic_instr *store = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch); store->num_components = intrin->num_components; store->src[0] = nir_src_for_ssa(value); store->src[1] = nir_src_for_ssa(offset); nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin)); nir_intrinsic_set_align(store, align, 0); nir_builder_instr_insert(b, &store->instr); } nir_instr_remove(&intrin->instr); nir_deref_instr_remove_if_unused(deref); }
static nir_ssa_def * get_texture_size(nir_builder *b, nir_tex_instr *tex) { b->cursor = nir_before_instr(&tex->instr); /* RECT textures should not be array: */ assert(!tex->is_array); nir_tex_instr *txs; txs = nir_tex_instr_create(b->shader, 1); txs->op = nir_texop_txs; txs->sampler_dim = GLSL_SAMPLER_DIM_RECT; txs->texture_index = tex->texture_index; txs->dest_type = nir_type_int; /* only single src, the lod: */ txs->src[0].src = nir_src_for_ssa(nir_imm_int(b, 0)); txs->src[0].src_type = nir_tex_src_lod; nir_ssa_dest_init(&txs->instr, &txs->dest, 2, 32, NULL); nir_builder_instr_insert(b, &txs->instr); return nir_i2f(b, &txs->dest.ssa); }
static nir_ssa_def * sample_plane(nir_builder *b, nir_tex_instr *tex, int plane) { assert(tex->dest.is_ssa); assert(nir_tex_instr_dest_size(tex) == 4); assert(nir_alu_type_get_base_type(tex->dest_type) == nir_type_float); assert(tex->op == nir_texop_tex); assert(tex->coord_components == 2); nir_tex_instr *plane_tex = nir_tex_instr_create(b->shader, tex->num_srcs + 1); for (unsigned i = 0; i < tex->num_srcs; i++) { nir_src_copy(&plane_tex->src[i].src, &tex->src[i].src, plane_tex); plane_tex->src[i].src_type = tex->src[i].src_type; } plane_tex->src[tex->num_srcs].src = nir_src_for_ssa(nir_imm_int(b, plane)); plane_tex->src[tex->num_srcs].src_type = nir_tex_src_plane; plane_tex->op = nir_texop_tex; plane_tex->sampler_dim = GLSL_SAMPLER_DIM_2D; plane_tex->dest_type = nir_type_float; plane_tex->coord_components = 2; plane_tex->texture_index = tex->texture_index; plane_tex->sampler_index = tex->sampler_index; nir_ssa_dest_init(&plane_tex->instr, &plane_tex->dest, 4, 32, NULL); nir_builder_instr_insert(b, &plane_tex->instr); return &plane_tex->dest.ssa; }
static bool repair_ssa_def(nir_ssa_def *def, void *void_state) { struct repair_ssa_state *state = void_state; bool is_valid = true; nir_foreach_use(def, src) { if (!nir_block_dominates(def->parent_instr->block, get_src_block(src))) { is_valid = false; break; } } if (is_valid) return true; struct nir_phi_builder *pb = prep_build_phi(state); BITSET_SET(state->def_set, def->parent_instr->block->index); struct nir_phi_builder_value *val = nir_phi_builder_add_value(pb, def->num_components, def->bit_size, state->def_set); nir_phi_builder_value_set_block_def(val, def->parent_instr->block, def); nir_foreach_use_safe(def, src) { nir_block *src_block = get_src_block(src); if (!nir_block_dominates(def->parent_instr->block, src_block)) { nir_instr_rewrite_src(src->parent_instr, src, nir_src_for_ssa( nir_phi_builder_value_get_block_def(val, src_block))); } }
static nir_ssa_def * sample_plane(nir_builder *b, nir_tex_instr *tex, int plane) { assert(tex->dest.is_ssa); assert(nir_tex_instr_dest_size(tex) == 4); assert(nir_alu_type_get_base_type(tex->dest_type) == nir_type_float); assert(tex->op == nir_texop_tex); assert(tex->coord_components == 2); nir_tex_instr *plane_tex = nir_tex_instr_create(b->shader, 2); nir_src_copy(&plane_tex->src[0].src, &tex->src[0].src, plane_tex); plane_tex->src[0].src_type = nir_tex_src_coord; plane_tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, plane)); plane_tex->src[1].src_type = nir_tex_src_plane; plane_tex->op = nir_texop_tex; plane_tex->sampler_dim = 2; plane_tex->dest_type = nir_type_float; plane_tex->coord_components = 2; plane_tex->texture_index = tex->texture_index; plane_tex->texture = (nir_deref_var *) nir_copy_deref(plane_tex, &tex->texture->deref); plane_tex->sampler_index = tex->sampler_index; plane_tex->sampler = (nir_deref_var *) nir_copy_deref(plane_tex, &tex->sampler->deref); nir_ssa_dest_init(&plane_tex->instr, &plane_tex->dest, 4, 32, NULL); nir_builder_instr_insert(b, &plane_tex->instr); return &plane_tex->dest.ssa; }
static void lower_load_const_instr_scalar(nir_load_const_instr *lower) { if (lower->def.num_components == 1) return; nir_builder b; nir_builder_init(&b, nir_cf_node_get_function(&lower->instr.block->cf_node)); b.cursor = nir_before_instr(&lower->instr); /* Emit the individual loads. */ nir_ssa_def *loads[4]; for (unsigned i = 0; i < lower->def.num_components; i++) { nir_load_const_instr *load_comp = nir_load_const_instr_create(b.shader, 1); load_comp->value.u[0] = lower->value.u[i]; nir_builder_instr_insert(&b, &load_comp->instr); loads[i] = &load_comp->def; } /* Batch things back together into a vector. */ nir_ssa_def *vec = nir_vec(&b, loads, lower->def.num_components); /* Replace the old load with a reference to our reconstructed vector. */ nir_ssa_def_rewrite_uses(&lower->def, nir_src_for_ssa(vec)); nir_instr_remove(&lower->instr); }
static nir_ssa_def * get_texture_size(nir_builder *b, nir_tex_instr *tex) { b->cursor = nir_before_instr(&tex->instr); nir_tex_instr *txs; txs = nir_tex_instr_create(b->shader, 1); txs->op = nir_texop_txs; txs->sampler_dim = tex->sampler_dim; txs->is_array = tex->is_array; txs->is_shadow = tex->is_shadow; txs->is_new_style_shadow = tex->is_new_style_shadow; txs->texture_index = tex->texture_index; txs->texture = nir_deref_var_clone(tex->texture, txs); txs->sampler_index = tex->sampler_index; txs->sampler = nir_deref_var_clone(tex->sampler, txs); txs->dest_type = nir_type_int; /* only single src, the lod: */ txs->src[0].src = nir_src_for_ssa(nir_imm_int(b, 0)); txs->src[0].src_type = nir_tex_src_lod; nir_ssa_dest_init(&txs->instr, &txs->dest, tex->coord_components, 32, NULL); nir_builder_instr_insert(b, &txs->instr); return nir_i2f(b, &txs->dest.ssa); }
static nir_ssa_def * get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain, struct vtn_type **type, unsigned *chain_idx) { /* Push constants have no explicit binding */ if (chain->var->mode == vtn_variable_mode_push_constant) { *chain_idx = 0; *type = chain->var->type; return NULL; } nir_ssa_def *array_index; if (glsl_type_is_array(chain->var->type->type)) { assert(chain->length > 0); array_index = vtn_access_link_as_ssa(b, chain->link[0], 1); *chain_idx = 1; *type = chain->var->type->array_element; } else { array_index = nir_imm_int(&b->nb, 0); *chain_idx = 0; *type = chain->var->type; } nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, nir_intrinsic_vulkan_resource_index); instr->src[0] = nir_src_for_ssa(array_index); nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set); nir_intrinsic_set_binding(instr, chain->var->binding); nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); nir_builder_instr_insert(&b->nb, &instr->instr); return &instr->dest.ssa; }
static void lower_implicit_lod(nir_builder *b, nir_tex_instr *tex) { assert(tex->op == nir_texop_tex || tex->op == nir_texop_txb); assert(nir_tex_instr_src_index(tex, nir_tex_src_lod) < 0); assert(nir_tex_instr_src_index(tex, nir_tex_src_ddx) < 0); assert(nir_tex_instr_src_index(tex, nir_tex_src_ddy) < 0); b->cursor = nir_before_instr(&tex->instr); nir_ssa_def *lod = get_texture_lod(b, tex); int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias); if (bias_idx >= 0) { /* If we have a bias, add it in */ lod = nir_fadd(b, lod, nir_ssa_for_src(b, tex->src[bias_idx].src, 1)); nir_tex_instr_remove_src(tex, bias_idx); } int min_lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod); if (min_lod_idx >= 0) { /* If we have a minimum LOD, clamp LOD accordingly */ lod = nir_fmax(b, lod, nir_ssa_for_src(b, tex->src[min_lod_idx].src, 1)); nir_tex_instr_remove_src(tex, min_lod_idx); } nir_tex_instr_add_src(tex, nir_tex_src_lod, nir_src_for_ssa(lod)); tex->op = nir_texop_txl; }
static bool constant_fold_deref(nir_instr *instr, nir_deref_var *deref) { bool progress = false; for (nir_deref *tail = deref->deref.child; tail; tail = tail->child) { if (tail->deref_type != nir_deref_type_array) continue; nir_deref_array *arr = nir_deref_as_array(tail); if (arr->deref_array_type == nir_deref_array_type_indirect && arr->indirect.is_ssa && arr->indirect.ssa->parent_instr->type == nir_instr_type_load_const) { nir_load_const_instr *indirect = nir_instr_as_load_const(arr->indirect.ssa->parent_instr); arr->base_offset += indirect->value.u[0]; /* Clear out the source */ nir_instr_rewrite_src(instr, &arr->indirect, nir_src_for_ssa(NULL)); arr->deref_array_type = nir_deref_array_type_direct; progress = true; } } return progress; }
static void vtn_build_subgroup_instr(struct vtn_builder *b, nir_intrinsic_op nir_op, struct vtn_ssa_value *dst, struct vtn_ssa_value *src0, nir_ssa_def *index, unsigned const_idx0, unsigned const_idx1) { /* Some of the subgroup operations take an index. SPIR-V allows this to be * any integer type. To make things simpler for drivers, we only support * 32-bit indices. */ if (index && index->bit_size != 32) index = nir_u2u32(&b->nb, index); vtn_assert(dst->type == src0->type); if (!glsl_type_is_vector_or_scalar(dst->type)) { for (unsigned i = 0; i < glsl_get_length(dst->type); i++) { vtn_build_subgroup_instr(b, nir_op, dst->elems[i], src0->elems[i], index, const_idx0, const_idx1); } return; } nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->nb.shader, nir_op); nir_ssa_dest_init_for_type(&intrin->instr, &intrin->dest, dst->type, NULL); intrin->num_components = intrin->dest.ssa.num_components; intrin->src[0] = nir_src_for_ssa(src0->def); if (index) intrin->src[1] = nir_src_for_ssa(index); intrin->const_index[0] = const_idx0; intrin->const_index[1] = const_idx1; nir_builder_instr_insert(&b->nb, &intrin->instr); dst->def = &intrin->dest.ssa; }
static bool remove_phis_block(nir_block *block, void *state) { bool *progress = state; nir_foreach_instr_safe(block, instr) { if (instr->type != nir_instr_type_phi) break; nir_phi_instr *phi = nir_instr_as_phi(instr); nir_ssa_def *def = NULL; bool srcs_same = true; nir_foreach_phi_src(phi, src) { assert(src->src.is_ssa); /* For phi nodes at the beginning of loops, we may encounter some * sources from backedges that point back to the destination of the * same phi, i.e. something like: * * a = phi(a, b, ...) * * We can safely ignore these sources, since if all of the normal * sources point to the same definition, then that definition must * still dominate the phi node, and the phi will still always take * the value of that definition. */ if (src->src.ssa == &phi->dest.ssa) continue; if (def == NULL) { def = src->src.ssa; } else { if (src->src.ssa != def) { srcs_same = false; break; } } } if (!srcs_same) continue; /* We must have found at least one definition, since there must be at * least one forward edge. */ assert(def != NULL); assert(phi->dest.is_ssa); nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def)); nir_instr_remove(instr); *progress = true; }
static void build_color_shaders(struct nir_shader **out_vs, struct nir_shader **out_fs, uint32_t frag_output) { nir_builder vs_b; nir_builder fs_b; nir_builder_init_simple_shader(&vs_b, NULL, MESA_SHADER_VERTEX, NULL); nir_builder_init_simple_shader(&fs_b, NULL, MESA_SHADER_FRAGMENT, NULL); vs_b.shader->info.name = ralloc_strdup(vs_b.shader, "meta_clear_color_vs"); fs_b.shader->info.name = ralloc_strdup(fs_b.shader, "meta_clear_color_fs"); const struct glsl_type *position_type = glsl_vec4_type(); const struct glsl_type *color_type = glsl_vec4_type(); nir_variable *vs_out_pos = nir_variable_create(vs_b.shader, nir_var_shader_out, position_type, "gl_Position"); vs_out_pos->data.location = VARYING_SLOT_POS; nir_intrinsic_instr *in_color_load = nir_intrinsic_instr_create(fs_b.shader, nir_intrinsic_load_push_constant); nir_intrinsic_set_base(in_color_load, 0); nir_intrinsic_set_range(in_color_load, 16); in_color_load->src[0] = nir_src_for_ssa(nir_imm_int(&fs_b, 0)); in_color_load->num_components = 4; nir_ssa_dest_init(&in_color_load->instr, &in_color_load->dest, 4, 32, "clear color"); nir_builder_instr_insert(&fs_b, &in_color_load->instr); nir_variable *fs_out_color = nir_variable_create(fs_b.shader, nir_var_shader_out, color_type, "f_color"); fs_out_color->data.location = FRAG_RESULT_DATA0 + frag_output; nir_store_var(&fs_b, fs_out_color, &in_color_load->dest.ssa, 0xf); nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&vs_b); nir_store_var(&vs_b, vs_out_pos, outvec, 0xf); const struct glsl_type *layer_type = glsl_int_type(); nir_variable *vs_out_layer = nir_variable_create(vs_b.shader, nir_var_shader_out, layer_type, "v_layer"); vs_out_layer->data.location = VARYING_SLOT_LAYER; vs_out_layer->data.interpolation = INTERP_MODE_FLAT; nir_ssa_def *inst_id = nir_load_system_value(&vs_b, nir_intrinsic_load_instance_id, 0); nir_ssa_def *base_instance = nir_load_system_value(&vs_b, nir_intrinsic_load_base_instance, 0); nir_ssa_def *layer_id = nir_iadd(&vs_b, inst_id, base_instance); nir_store_var(&vs_b, vs_out_layer, layer_id, 0x1); *out_vs = vs_b.shader; *out_fs = fs_b.shader; }
static bool lower_offset(nir_builder *b, nir_tex_instr *tex) { int offset_index = nir_tex_instr_src_index(tex, nir_tex_src_offset); if (offset_index < 0) return false; int coord_index = nir_tex_instr_src_index(tex, nir_tex_src_coord); assert(coord_index >= 0); assert(tex->src[offset_index].src.is_ssa); assert(tex->src[coord_index].src.is_ssa); nir_ssa_def *offset = tex->src[offset_index].src.ssa; nir_ssa_def *coord = tex->src[coord_index].src.ssa; b->cursor = nir_before_instr(&tex->instr); nir_ssa_def *offset_coord; if (nir_tex_instr_src_type(tex, coord_index) == nir_type_float) { if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) { offset_coord = nir_fadd(b, coord, nir_i2f32(b, offset)); } else { nir_ssa_def *txs = get_texture_size(b, tex); nir_ssa_def *scale = nir_frcp(b, txs); offset_coord = nir_fadd(b, coord, nir_fmul(b, nir_i2f32(b, offset), scale)); } } else { offset_coord = nir_iadd(b, coord, offset); } if (tex->is_array) { /* The offset is not applied to the array index */ if (tex->coord_components == 2) { offset_coord = nir_vec2(b, nir_channel(b, offset_coord, 0), nir_channel(b, coord, 1)); } else if (tex->coord_components == 3) { offset_coord = nir_vec3(b, nir_channel(b, offset_coord, 0), nir_channel(b, offset_coord, 1), nir_channel(b, coord, 2)); } else { unreachable("Invalid number of components"); } } nir_instr_rewrite_src(&tex->instr, &tex->src[coord_index].src, nir_src_for_ssa(offset_coord)); nir_tex_instr_remove_src(tex, offset_index); return true; }
static void lower_sampler(nir_tex_instr *instr, const struct gl_shader_program *shader_program, gl_shader_stage stage, nir_builder *builder) { if (instr->sampler == NULL) return; instr->sampler_index = 0; unsigned location = instr->sampler->var->data.location; unsigned array_elements = 1; nir_ssa_def *indirect = NULL; builder->cursor = nir_before_instr(&instr->instr); calc_sampler_offsets(&instr->sampler->deref, instr, &array_elements, &indirect, builder, &location); if (indirect) { /* First, we have to resize the array of texture sources */ nir_tex_src *new_srcs = rzalloc_array(instr, nir_tex_src, instr->num_srcs + 1); for (unsigned i = 0; i < instr->num_srcs; i++) { new_srcs[i].src_type = instr->src[i].src_type; nir_instr_move_src(&instr->instr, &new_srcs[i].src, &instr->src[i].src); } ralloc_free(instr->src); instr->src = new_srcs; /* Now we can go ahead and move the source over to being a * first-class texture source. */ instr->src[instr->num_srcs].src_type = nir_tex_src_sampler_offset; instr->num_srcs++; nir_instr_rewrite_src(&instr->instr, &instr->src[instr->num_srcs - 1].src, nir_src_for_ssa(indirect)); instr->sampler_array_size = array_elements; } if (location > shader_program->NumUniformStorage - 1 || !shader_program->UniformStorage[location].opaque[stage].active) { assert(!"cannot return a sampler"); return; } instr->sampler_index += shader_program->UniformStorage[location].opaque[stage].index; instr->sampler = NULL; }
static void opt_constant_if(nir_if *if_stmt, bool condition) { /* First, we need to remove any phi nodes after the if by rewriting uses to * point to the correct source. */ nir_block *after = nir_cf_node_as_block(nir_cf_node_next(&if_stmt->cf_node)); nir_block *last_block = nir_cf_node_as_block(condition ? nir_if_last_then_node(if_stmt) : nir_if_last_else_node(if_stmt)); nir_foreach_instr_safe(after, instr) { if (instr->type != nir_instr_type_phi) break; nir_phi_instr *phi = nir_instr_as_phi(instr); nir_ssa_def *def = NULL; nir_foreach_phi_src(phi, phi_src) { if (phi_src->pred != last_block) continue; assert(phi_src->src.is_ssa); def = phi_src->src.ssa; } assert(def); assert(phi->dest.is_ssa); nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def)); nir_instr_remove(instr); } /* The control flow list we're about to paste in may include a jump at the * end, and in that case we have to delete the rest of the control flow * list after the if since it's unreachable and the validator will balk if * we don't. */ if (!exec_list_is_empty(&last_block->instr_list)) { nir_instr *last_instr = nir_block_last_instr(last_block); if (last_instr->type == nir_instr_type_jump) remove_after_cf_node(&if_stmt->cf_node); } /* Finally, actually paste in the then or else branch and delete the if. */ struct exec_list *cf_list = condition ? &if_stmt->then_list : &if_stmt->else_list; nir_cf_list list; nir_cf_extract(&list, nir_before_cf_list(cf_list), nir_after_cf_list(cf_list)); nir_cf_reinsert(&list, nir_after_cf_node(&if_stmt->cf_node)); nir_cf_node_remove(&if_stmt->cf_node); }
static void _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load, nir_ssa_def *index, nir_ssa_def *offset, struct vtn_ssa_value **inout, const struct glsl_type *type) { nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op); instr->num_components = glsl_get_vector_elements(type); int src = 0; if (!load) { nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1); instr->src[src++] = nir_src_for_ssa((*inout)->def); } /* We set the base and size for push constant load to the entire push * constant block for now. */ if (op == nir_intrinsic_load_push_constant) { nir_intrinsic_set_base(instr, 0); nir_intrinsic_set_range(instr, 128); } if (index) instr->src[src++] = nir_src_for_ssa(index); instr->src[src++] = nir_src_for_ssa(offset); if (load) { nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components, glsl_get_bit_size(glsl_get_base_type(type)), NULL); (*inout)->def = &instr->dest.ssa; } nir_builder_instr_insert(&b->nb, &instr->instr); if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL) (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0)); }
static nir_ssa_def * get_texture_size(nir_builder *b, nir_tex_instr *tex) { b->cursor = nir_before_instr(&tex->instr); nir_tex_instr *txs; unsigned num_srcs = 1; /* One for the LOD */ for (unsigned i = 0; i < tex->num_srcs; i++) { if (tex->src[i].src_type == nir_tex_src_texture_deref || tex->src[i].src_type == nir_tex_src_sampler_deref || tex->src[i].src_type == nir_tex_src_texture_offset || tex->src[i].src_type == nir_tex_src_sampler_offset || tex->src[i].src_type == nir_tex_src_texture_handle || tex->src[i].src_type == nir_tex_src_sampler_handle) num_srcs++; } txs = nir_tex_instr_create(b->shader, num_srcs); txs->op = nir_texop_txs; txs->sampler_dim = tex->sampler_dim; txs->is_array = tex->is_array; txs->is_shadow = tex->is_shadow; txs->is_new_style_shadow = tex->is_new_style_shadow; txs->texture_index = tex->texture_index; txs->sampler_index = tex->sampler_index; txs->dest_type = nir_type_int; unsigned idx = 0; for (unsigned i = 0; i < tex->num_srcs; i++) { if (tex->src[i].src_type == nir_tex_src_texture_deref || tex->src[i].src_type == nir_tex_src_sampler_deref || tex->src[i].src_type == nir_tex_src_texture_offset || tex->src[i].src_type == nir_tex_src_sampler_offset || tex->src[i].src_type == nir_tex_src_texture_handle || tex->src[i].src_type == nir_tex_src_sampler_handle) { nir_src_copy(&txs->src[idx].src, &tex->src[i].src, txs); txs->src[idx].src_type = tex->src[i].src_type; idx++; } } /* Add in an LOD because some back-ends require it */ txs->src[idx].src = nir_src_for_ssa(nir_imm_int(b, 0)); txs->src[idx].src_type = nir_tex_src_lod; nir_ssa_dest_init(&txs->instr, &txs->dest, nir_tex_instr_dest_size(txs), 32, NULL); nir_builder_instr_insert(b, &txs->instr); return nir_i2f32(b, &txs->dest.ssa); }
static void lower_reduction(nir_alu_instr *instr, nir_op chan_op, nir_op merge_op, void *mem_ctx) { unsigned num_components = nir_op_infos[instr->op].input_sizes[0]; nir_ssa_def *last = NULL; for (unsigned i = 0; i < num_components; i++) { nir_alu_instr *chan = nir_alu_instr_create(mem_ctx, chan_op); nir_alu_ssa_dest_init(chan, 1); nir_alu_src_copy(&chan->src[0], &instr->src[0], mem_ctx); chan->src[0].swizzle[0] = chan->src[0].swizzle[i]; if (nir_op_infos[chan_op].num_inputs > 1) { assert(nir_op_infos[chan_op].num_inputs == 2); nir_alu_src_copy(&chan->src[1], &instr->src[1], mem_ctx); chan->src[1].swizzle[0] = chan->src[1].swizzle[i]; } nir_instr_insert_before(&instr->instr, &chan->instr); if (i == 0) { last = &chan->dest.dest.ssa; } else { nir_alu_instr *merge = nir_alu_instr_create(mem_ctx, merge_op); nir_alu_ssa_dest_init(merge, 1); merge->dest.write_mask = 1; merge->src[0].src = nir_src_for_ssa(last); merge->src[1].src = nir_src_for_ssa(&chan->dest.dest.ssa); nir_instr_insert_before(&instr->instr, &merge->instr); last = &merge->dest.dest.ssa; } } assert(instr->dest.write_mask == 1); nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(last), mem_ctx); nir_instr_remove(&instr->instr); }
static bool constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx) { nir_const_value src[4]; if (!instr->dest.dest.is_ssa) return false; for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { if (!instr->src[i].src.is_ssa) return false; nir_instr *src_instr = instr->src[i].src.ssa->parent_instr; if (src_instr->type != nir_instr_type_load_const) return false; nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr); for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i); j++) { src[i].u[j] = load_const->value.u[instr->src[i].swizzle[j]]; } /* We shouldn't have any source modifiers in the optimization loop. */ assert(!instr->src[i].abs && !instr->src[i].negate); } /* We shouldn't have any saturate modifiers in the optimization loop. */ assert(!instr->dest.saturate); nir_const_value dest = nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components, src); nir_load_const_instr *new_instr = nir_load_const_instr_create(mem_ctx, instr->dest.dest.ssa.num_components); new_instr->value = dest; nir_instr_insert_before(&instr->instr, &new_instr->instr); nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(&new_instr->def), mem_ctx); nir_instr_remove(&instr->instr); ralloc_free(instr); return true; }
static nir_src get_deref_reg_src(nir_deref_instr *deref, struct locals_to_regs_state *state) { nir_builder *b = &state->builder; nir_src src; src.is_ssa = false; src.reg.reg = get_reg_for_deref(deref, state); src.reg.base_offset = 0; src.reg.indirect = NULL; /* It is possible for a user to create a shader that has an array with a * single element and then proceed to access it indirectly. Indirectly * accessing a non-array register is not allowed in NIR. In order to * handle this case we just convert it to a direct reference. */ if (src.reg.reg->num_array_elems == 0) return src; unsigned inner_array_size = 1; for (const nir_deref_instr *d = deref; d; d = nir_deref_instr_parent(d)) { if (d->deref_type != nir_deref_type_array) continue; if (nir_src_is_const(d->arr.index) && !src.reg.indirect) { src.reg.base_offset += nir_src_as_uint(d->arr.index) * inner_array_size; } else { if (src.reg.indirect) { assert(src.reg.base_offset == 0); } else { src.reg.indirect = ralloc(b->shader, nir_src); *src.reg.indirect = nir_src_for_ssa(nir_imm_int(b, src.reg.base_offset)); src.reg.base_offset = 0; } assert(src.reg.indirect->is_ssa); nir_ssa_def *index = nir_i2i(b, nir_ssa_for_src(b, d->arr.index, 1), 32); src.reg.indirect->ssa = nir_iadd(b, src.reg.indirect->ssa, nir_imul(b, index, nir_imm_int(b, inner_array_size))); } inner_array_size *= glsl_get_length(nir_deref_instr_parent(d)->type); } return src; }
static void handle_glsl450_interpolation(struct vtn_builder *b, enum GLSLstd450 opcode, const uint32_t *w, unsigned count) { const struct glsl_type *dest_type = vtn_value(b, w[1], vtn_value_type_type)->type->type; struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); val->ssa = vtn_create_ssa_value(b, dest_type); nir_intrinsic_op op; switch (opcode) { case GLSLstd450InterpolateAtCentroid: op = nir_intrinsic_interp_var_at_centroid; break; case GLSLstd450InterpolateAtSample: op = nir_intrinsic_interp_var_at_sample; break; case GLSLstd450InterpolateAtOffset: op = nir_intrinsic_interp_var_at_offset; break; default: unreachable("Invalid opcode"); } nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->nb.shader, op); nir_deref_var *deref = vtn_nir_deref(b, w[5]); intrin->variables[0] = nir_deref_as_var(nir_copy_deref(intrin, &deref->deref)); switch (opcode) { case GLSLstd450InterpolateAtCentroid: break; case GLSLstd450InterpolateAtSample: case GLSLstd450InterpolateAtOffset: intrin->src[0] = nir_src_for_ssa(vtn_ssa_value(b, w[6])->def); break; default: unreachable("Invalid opcode"); } intrin->num_components = glsl_get_vector_elements(dest_type); nir_ssa_dest_init(&intrin->instr, &intrin->dest, glsl_get_vector_elements(dest_type), glsl_get_bit_size(dest_type), NULL); val->ssa->def = &intrin->dest.ssa; nir_builder_instr_insert(&b->nb, &intrin->instr); }
static nir_shader * build_resolve_fragment_shader(struct radv_device *dev, bool is_integer, int samples) { nir_builder b; char name[64]; const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2); const struct glsl_type *vec4 = glsl_vec4_type(); const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_MS, false, false, GLSL_TYPE_FLOAT); snprintf(name, 64, "meta_resolve_fs-%d-%s", samples, is_integer ? "int" : "float"); nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL); b.shader->info.name = ralloc_strdup(b.shader, name); nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex"); input_img->data.descriptor_set = 0; input_img->data.binding = 0; nir_variable *fs_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "fs_pos_in"); fs_pos_in->data.location = VARYING_SLOT_POS; nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); color_out->data.location = FRAG_RESULT_DATA0; nir_ssa_def *pos_in = nir_load_var(&b, fs_pos_in); nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); nir_intrinsic_set_base(src_offset, 0); nir_intrinsic_set_range(src_offset, 8); src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0)); src_offset->num_components = 2; nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, 2, 32, "src_offset"); nir_builder_instr_insert(&b, &src_offset->instr); nir_ssa_def *pos_int = nir_f2i32(&b, pos_in); nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, &src_offset->dest.ssa), 0x3); nir_variable *color = nir_local_variable_create(b.impl, glsl_vec4_type(), "color"); radv_meta_build_resolve_shader_core(&b, is_integer, samples, input_img, color, img_coord); nir_ssa_def *outval = nir_load_var(&b, color); nir_store_var(&b, color_out, outval, 0xf); return b.shader; }
static void emit_load_store(nir_builder *b, nir_intrinsic_instr *orig_instr, nir_deref_var *deref, nir_deref *tail, nir_ssa_def **dest, nir_ssa_def *src) { for (; tail->child; tail = tail->child) { if (tail->child->deref_type != nir_deref_type_array) continue; nir_deref_array *arr = nir_deref_as_array(tail->child); if (arr->deref_array_type != nir_deref_array_type_indirect) continue; int length = glsl_get_length(tail->type); emit_indirect_load_store(b, orig_instr, deref, tail, -arr->base_offset, length - arr->base_offset, dest, src); return; } assert(tail && tail->child == NULL); /* We reached the end of the deref chain. Emit the instruction */ if (src == NULL) { /* This is a load instruction */ nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_var); load->num_components = orig_instr->num_components; load->variables[0] = nir_deref_as_var(nir_copy_deref(load, &deref->deref)); unsigned bit_size = orig_instr->dest.ssa.bit_size; nir_ssa_dest_init(&load->instr, &load->dest, load->num_components, bit_size, NULL); nir_builder_instr_insert(b, &load->instr); *dest = &load->dest.ssa; } else { /* This is a store instruction */ nir_intrinsic_instr *store = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_var); store->num_components = orig_instr->num_components; nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(orig_instr)); store->variables[0] = nir_deref_as_var(nir_copy_deref(store, &deref->deref)); store->src[0] = nir_src_for_ssa(src); nir_builder_instr_insert(b, &store->instr); } }
static void saturate_src(nir_builder *b, nir_tex_instr *tex, unsigned sat_mask) { b->cursor = nir_before_instr(&tex->instr); /* Walk through the sources saturating the requested arguments. */ for (unsigned i = 0; i < tex->num_srcs; i++) { if (tex->src[i].src_type != nir_tex_src_coord) continue; nir_ssa_def *src = nir_ssa_for_src(b, tex->src[i].src, tex->coord_components); /* split src into components: */ nir_ssa_def *comp[4]; for (unsigned j = 0; j < tex->coord_components; j++) comp[j] = nir_channel(b, src, j); /* clamp requested components, array index does not get clamped: */ unsigned ncomp = tex->coord_components; if (tex->is_array) ncomp--; for (unsigned j = 0; j < ncomp; j++) { if ((1 << j) & sat_mask) { if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) { /* non-normalized texture coords, so clamp to texture * size rather than [0.0, 1.0] */ nir_ssa_def *txs = get_texture_size(b, tex); comp[j] = nir_fmax(b, comp[j], nir_imm_float(b, 0.0)); comp[j] = nir_fmin(b, comp[j], nir_channel(b, txs, j)); } else { comp[j] = nir_fsat(b, comp[j]); } } } /* and move the result back into a single vecN: */ src = nir_vec(b, comp, tex->coord_components); nir_instr_rewrite_src(&tex->instr, &tex->src[i].src, nir_src_for_ssa(src)); } }
/* Multiply interp_var_at_offset's offset by transform.x to flip it. */ static void lower_interp_var_at_offset(lower_wpos_ytransform_state *state, nir_intrinsic_instr *interp) { nir_builder *b = &state->b; nir_ssa_def *offset; nir_ssa_def *flip_y; b->cursor = nir_before_instr(&interp->instr); offset = nir_ssa_for_src(b, interp->src[0], 2); flip_y = nir_fmul(b, nir_channel(b, offset, 1), nir_channel(b, get_transform(state), 0)); nir_instr_rewrite_src(&interp->instr, &interp->src[0], nir_src_for_ssa(nir_vec2(b, nir_channel(b, offset, 0), flip_y))); }
nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if ((mode == nir_var_shader_in && is_input(intrin)) || (mode == nir_var_shader_out && is_output(intrin))) { nir_src *offset = nir_get_io_offset_src(intrin); nir_const_value *const_offset = nir_src_as_const_value(*offset); if (const_offset) { intrin->const_index[0] += const_offset->u32[0]; b->cursor = nir_before_instr(&intrin->instr); nir_instr_rewrite_src(&intrin->instr, offset, nir_src_for_ssa(nir_imm_int(b, 0))); } } }
nir_alu_instr * nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search, const nir_search_value *replace, void *mem_ctx) { uint8_t swizzle[4] = { 0, 0, 0, 0 }; for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) swizzle[i] = i; assert(instr->dest.dest.is_ssa); struct match_state state; state.variables_seen = 0; if (!match_expression(search, instr, instr->dest.dest.ssa.num_components, swizzle, &state)) return NULL; /* Inserting a mov may be unnecessary. However, it's much easier to * simply let copy propagation clean this up than to try to go through * and rewrite swizzles ourselves. */ nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov); mov->dest.write_mask = instr->dest.write_mask; nir_ssa_dest_init(&mov->instr, &mov->dest.dest, instr->dest.dest.ssa.num_components, NULL); mov->src[0] = construct_value(replace, nir_op_infos[instr->op].output_type, instr->dest.dest.ssa.num_components, &state, &instr->instr, mem_ctx); nir_instr_insert_before(&instr->instr, &mov->instr); nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(&mov->dest.dest.ssa), mem_ctx); /* We know this one has no more uses because we just rewrote them all, * so we can remove it. The rest of the matched expression, however, we * don't know so much about. We'll just let dead code clean them up. */ nir_instr_remove(&instr->instr); return mov; }
static void lower_load_sample_pos(lower_wpos_ytransform_state *state, nir_intrinsic_instr *intr) { nir_builder *b = &state->b; b->cursor = nir_after_instr(&intr->instr); nir_ssa_def *pos = &intr->dest.ssa; nir_ssa_def *scale = nir_channel(b, get_transform(state), 0); nir_ssa_def *neg_scale = nir_channel(b, get_transform(state), 2); /* Either y or 1-y for scale equal to 1 or -1 respectively. */ nir_ssa_def *flipped_y = nir_fadd(b, nir_fmax(b, neg_scale, nir_imm_float(b, 0.0)), nir_fmul(b, nir_channel(b, pos, 1), scale)); nir_ssa_def *flipped_pos = nir_vec2(b, nir_channel(b, pos, 0), flipped_y); nir_ssa_def_rewrite_uses_after(&intr->dest.ssa, nir_src_for_ssa(flipped_pos), flipped_pos->parent_instr); }
/* turns 'fddy(p)' into 'fddy(fmul(p, transform.x))' */ static void lower_fddy(lower_wpos_ytransform_state *state, nir_alu_instr *fddy) { nir_builder *b = &state->b; nir_ssa_def *p, *pt, *trans; b->cursor = nir_before_instr(&fddy->instr); p = nir_ssa_for_alu_src(b, fddy, 0); trans = get_transform(state); pt = nir_fmul(b, p, nir_channel(b, trans, 0)); nir_instr_rewrite_src(&fddy->instr, &fddy->src[0].src, nir_src_for_ssa(pt)); for (unsigned i = 0; i < 4; i++) fddy->src[0].swizzle[i] = MIN2(i, pt->num_components - 1); }