static void lower_load_store(nir_builder *b, nir_intrinsic_instr *intrin, glsl_type_size_align_func size_align) { b->cursor = nir_before_instr(&intrin->instr); nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); nir_variable *var = nir_deref_instr_get_variable(deref); nir_ssa_def *offset = nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align), var->data.location); unsigned align, UNUSED size; size_align(deref->type, &size, &align); if (intrin->intrinsic == nir_intrinsic_load_deref) { nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch); load->num_components = intrin->num_components; load->src[0] = nir_src_for_ssa(offset); nir_intrinsic_set_align(load, align, 0); nir_ssa_dest_init(&load->instr, &load->dest, intrin->dest.ssa.num_components, intrin->dest.ssa.bit_size, NULL); nir_builder_instr_insert(b, &load->instr); nir_ssa_def *value = &load->dest.ssa; if (glsl_type_is_boolean(deref->type)) value = nir_b2i32(b, value); nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(&load->dest.ssa)); } else { assert(intrin->intrinsic == nir_intrinsic_store_deref); assert(intrin->src[1].is_ssa); nir_ssa_def *value = intrin->src[1].ssa; if (glsl_type_is_boolean(deref->type)) value = nir_i2b(b, value); nir_intrinsic_instr *store = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch); store->num_components = intrin->num_components; store->src[0] = nir_src_for_ssa(value); store->src[1] = nir_src_for_ssa(offset); nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin)); nir_intrinsic_set_align(store, align, 0); nir_builder_instr_insert(b, &store->instr); } nir_instr_remove(&intrin->instr); nir_deref_instr_remove_if_unused(deref); }
static void emit_load_store(nir_builder *b, nir_intrinsic_instr *orig_instr, nir_deref_var *deref, nir_deref *tail, nir_ssa_def **dest, nir_ssa_def *src) { for (; tail->child; tail = tail->child) { if (tail->child->deref_type != nir_deref_type_array) continue; nir_deref_array *arr = nir_deref_as_array(tail->child); if (arr->deref_array_type != nir_deref_array_type_indirect) continue; int length = glsl_get_length(tail->type); emit_indirect_load_store(b, orig_instr, deref, tail, -arr->base_offset, length - arr->base_offset, dest, src); return; } assert(tail && tail->child == NULL); /* We reached the end of the deref chain. Emit the instruction */ if (src == NULL) { /* This is a load instruction */ nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_var); load->num_components = orig_instr->num_components; load->variables[0] = nir_deref_as_var(nir_copy_deref(load, &deref->deref)); unsigned bit_size = orig_instr->dest.ssa.bit_size; nir_ssa_dest_init(&load->instr, &load->dest, load->num_components, bit_size, NULL); nir_builder_instr_insert(b, &load->instr); *dest = &load->dest.ssa; } else { /* This is a store instruction */ nir_intrinsic_instr *store = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_var); store->num_components = orig_instr->num_components; nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(orig_instr)); store->variables[0] = nir_deref_as_var(nir_copy_deref(store, &deref->deref)); store->src[0] = nir_src_for_ssa(src); nir_builder_instr_insert(b, &store->instr); } }
static bool lower_locals_to_regs_block(nir_block *block, struct locals_to_regs_state *state) { nir_builder *b = &state->builder; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (deref->mode != nir_var_function_temp) continue; b->cursor = nir_before_instr(&intrin->instr); nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov); mov->src[0].src = get_deref_reg_src(deref, state); mov->dest.write_mask = (1 << intrin->num_components) - 1; if (intrin->dest.is_ssa) { nir_ssa_dest_init(&mov->instr, &mov->dest.dest, intrin->num_components, intrin->dest.ssa.bit_size, NULL); nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(&mov->dest.dest.ssa)); } else { nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr); } nir_builder_instr_insert(b, &mov->instr); nir_instr_remove(&intrin->instr); state->progress = true; break; } case nir_intrinsic_store_deref: { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (deref->mode != nir_var_function_temp) continue; b->cursor = nir_before_instr(&intrin->instr); nir_src reg_src = get_deref_reg_src(deref, state); nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov); nir_src_copy(&mov->src[0].src, &intrin->src[1], mov); mov->dest.write_mask = nir_intrinsic_write_mask(intrin); mov->dest.dest.is_ssa = false; mov->dest.dest.reg.reg = reg_src.reg.reg; mov->dest.dest.reg.base_offset = reg_src.reg.base_offset; mov->dest.dest.reg.indirect = reg_src.reg.indirect; nir_builder_instr_insert(b, &mov->instr); nir_instr_remove(&intrin->instr); state->progress = true; break; } case nir_intrinsic_copy_deref: unreachable("There should be no copies whatsoever at this point"); break; default: continue; } } return true; }
nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_var: { if (intrin->variables[0]->var->data.mode != nir_var_local) continue; nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov); mov->src[0].src = get_deref_reg_src(intrin->variables[0], &intrin->instr, state); mov->dest.write_mask = (1 << intrin->num_components) - 1; if (intrin->dest.is_ssa) { nir_ssa_dest_init(&mov->instr, &mov->dest.dest, intrin->num_components, intrin->dest.ssa.bit_size, NULL); nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(&mov->dest.dest.ssa)); } else { nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr); } nir_instr_insert_before(&intrin->instr, &mov->instr); nir_instr_remove(&intrin->instr); state->progress = true; break; } case nir_intrinsic_store_var: { if (intrin->variables[0]->var->data.mode != nir_var_local) continue; nir_src reg_src = get_deref_reg_src(intrin->variables[0], &intrin->instr, state); nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov); nir_src_copy(&mov->src[0].src, &intrin->src[0], mov); mov->dest.write_mask = nir_intrinsic_write_mask(intrin); mov->dest.dest.is_ssa = false; mov->dest.dest.reg.reg = reg_src.reg.reg; mov->dest.dest.reg.base_offset = reg_src.reg.base_offset; mov->dest.dest.reg.indirect = reg_src.reg.indirect; nir_instr_insert_before(&intrin->instr, &mov->instr); nir_instr_remove(&intrin->instr); state->progress = true; break; } case nir_intrinsic_copy_var: unreachable("There should be no copies whatsoever at this point"); break; default: continue; } }
static void validate_intrinsic_instr(nir_intrinsic_instr *instr, validate_state *state) { unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs; for (unsigned i = 0; i < num_srcs; i++) { unsigned components_read = nir_intrinsic_infos[instr->intrinsic].src_components[i]; if (components_read == 0) components_read = instr->num_components; validate_assert(state, components_read > 0); if (instr->src[i].is_ssa) { validate_assert(state, components_read <= instr->src[i].ssa->num_components); } else if (!instr->src[i].reg.reg->is_packed) { validate_assert(state, components_read <= instr->src[i].reg.reg->num_components); } validate_src(&instr->src[i], state); } unsigned num_vars = nir_intrinsic_infos[instr->intrinsic].num_variables; for (unsigned i = 0; i < num_vars; i++) { validate_deref_var(instr, instr->variables[i], state); } if (nir_intrinsic_infos[instr->intrinsic].has_dest) { unsigned components_written = nir_intrinsic_infos[instr->intrinsic].dest_components; if (components_written == 0) components_written = instr->num_components; validate_assert(state, components_written > 0); if (instr->dest.is_ssa) { validate_assert(state, components_written <= instr->dest.ssa.num_components); } else if (!instr->dest.reg.reg->is_packed) { validate_assert(state, components_written <= instr->dest.reg.reg->num_components); } validate_dest(&instr->dest, state); } switch (instr->intrinsic) { case nir_intrinsic_load_var: { const struct glsl_type *type = nir_deref_tail(&instr->variables[0]->deref)->type; validate_assert(state, glsl_type_is_vector_or_scalar(type) || (instr->variables[0]->var->data.mode == nir_var_uniform && glsl_get_base_type(type) == GLSL_TYPE_SUBROUTINE)); validate_assert(state, instr->num_components == glsl_get_vector_elements(type)); break; } case nir_intrinsic_store_var: { const struct glsl_type *type = nir_deref_tail(&instr->variables[0]->deref)->type; validate_assert(state, glsl_type_is_vector_or_scalar(type) || (instr->variables[0]->var->data.mode == nir_var_uniform && glsl_get_base_type(type) == GLSL_TYPE_SUBROUTINE)); validate_assert(state, instr->num_components == glsl_get_vector_elements(type)); validate_assert(state, instr->variables[0]->var->data.mode != nir_var_shader_in && instr->variables[0]->var->data.mode != nir_var_uniform && instr->variables[0]->var->data.mode != nir_var_shader_storage); validate_assert(state, (nir_intrinsic_write_mask(instr) & ~((1 << instr->num_components) - 1)) == 0); break; } case nir_intrinsic_copy_var: validate_assert(state, nir_deref_tail(&instr->variables[0]->deref)->type == nir_deref_tail(&instr->variables[1]->deref)->type); validate_assert(state, instr->variables[0]->var->data.mode != nir_var_shader_in && instr->variables[0]->var->data.mode != nir_var_uniform && instr->variables[0]->var->data.mode != nir_var_shader_storage); break; default: break; } }