static bool
constant_fold_intrinsic_instr(nir_intrinsic_instr *instr)
{
   bool progress = false;

   if (instr->intrinsic == nir_intrinsic_discard_if &&
       nir_src_is_const(instr->src[0])) {
      if (nir_src_as_bool(instr->src[0])) {
         /* This method of getting a nir_shader * from a nir_instr is
          * admittedly gross, but given the rarity of hitting this case I think
          * it's preferable to plumbing an otherwise unused nir_shader *
          * parameter through four functions to get here.
          */
         nir_cf_node *cf_node = &instr->instr.block->cf_node;
         nir_function_impl *impl = nir_cf_node_get_function(cf_node);
         nir_shader *shader = impl->function->shader;

         nir_intrinsic_instr *discard =
            nir_intrinsic_instr_create(shader, nir_intrinsic_discard);
         nir_instr_insert_before(&instr->instr, &discard->instr);
         nir_instr_remove(&instr->instr);
         progress = true;
      } else {
         /* We're not discarding, just delete the instruction */
         nir_instr_remove(&instr->instr);
         progress = true;
      }
   }

   return progress;
}
static void
lower_load_const_instr_scalar(nir_load_const_instr *lower)
{
   if (lower->def.num_components == 1)
      return;

   nir_builder b;
   nir_builder_init(&b, nir_cf_node_get_function(&lower->instr.block->cf_node));
   b.cursor = nir_before_instr(&lower->instr);

   /* Emit the individual loads. */
   nir_ssa_def *loads[4];
   for (unsigned i = 0; i < lower->def.num_components; i++) {
      nir_load_const_instr *load_comp = nir_load_const_instr_create(b.shader, 1);
      load_comp->value.u[0] = lower->value.u[i];
      nir_builder_instr_insert(&b, &load_comp->instr);
      loads[i] = &load_comp->def;
   }

   /* Batch things back together into a vector. */
   nir_ssa_def *vec = nir_vec(&b, loads, lower->def.num_components);

   /* Replace the old load with a reference to our reconstructed vector. */
   nir_ssa_def_rewrite_uses(&lower->def, nir_src_for_ssa(vec));
   nir_instr_remove(&lower->instr);
}
Esempio n. 3
0
static bool
lower_vec_to_movs_block(nir_block *block, void *void_state)
{
   struct vec_to_movs_state *state = void_state;
   nir_function_impl *impl = state->impl;
   nir_shader *shader = impl->overload->function->shader;

   nir_foreach_instr_safe(block, instr) {
      if (instr->type != nir_instr_type_alu)
         continue;

      nir_alu_instr *vec = nir_instr_as_alu(instr);

      switch (vec->op) {
      case nir_op_vec2:
      case nir_op_vec3:
      case nir_op_vec4:
         break;
      default:
         continue; /* The loop */
      }

      /* Since we insert multiple MOVs, we have to be non-SSA. */
      assert(!vec->dest.dest.is_ssa);

      unsigned finished_write_mask = 0;

      /* First, emit a MOV for all the src channels that are in the
       * destination reg, in case other values we're populating in the dest
       * might overwrite them.
       */
      for (unsigned i = 0, src_idx = 0; i < 4; i++) {
         if (!(vec->dest.write_mask & (1 << i)))
            continue;

         if (src_matches_dest_reg(&vec->dest.dest, &vec->src[src_idx].src)) {
            finished_write_mask |= insert_mov(vec, i, src_idx, shader);
            break;
         }
         src_idx++;
      }

      /* Now, emit MOVs for all the other src channels. */
      for (unsigned i = 0, src_idx = 0; i < 4; i++) {
         if (!(vec->dest.write_mask & (1 << i)))
            continue;

         if (!(finished_write_mask & (1 << i)))
            finished_write_mask |= insert_mov(vec, i, src_idx, shader);

         src_idx++;
      }

      nir_instr_remove(&vec->instr);
      ralloc_free(vec);
      state->progress = true;
   }

   return true;
}
Esempio n. 4
0
static bool
split_var_copies_block(nir_block *block, void *void_state)
{
   struct split_var_copies_state *state = void_state;

   nir_foreach_instr_safe(block, instr) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
      if (intrinsic->intrinsic != nir_intrinsic_copy_var)
         continue;

      nir_deref *dest_head = &intrinsic->variables[0]->deref;
      nir_deref *src_head = &intrinsic->variables[1]->deref;
      nir_deref *dest_tail = get_deref_tail(dest_head);
      nir_deref *src_tail = get_deref_tail(src_head);

      switch (glsl_get_base_type(src_tail->type)) {
      case GLSL_TYPE_ARRAY:
      case GLSL_TYPE_STRUCT:
         split_var_copy_instr(intrinsic, dest_head, src_head,
                              dest_tail, src_tail, state);
         nir_instr_remove(&intrinsic->instr);
         ralloc_steal(state->dead_ctx, instr);
         break;
      case GLSL_TYPE_FLOAT:
      case GLSL_TYPE_INT:
      case GLSL_TYPE_UINT:
      case GLSL_TYPE_BOOL:
         if (glsl_type_is_matrix(src_tail->type)) {
            split_var_copy_instr(intrinsic, dest_head, src_head,
                                 dest_tail, src_tail, state);
            nir_instr_remove(&intrinsic->instr);
            ralloc_steal(state->dead_ctx, instr);
         }
         break;
      default:
         unreachable("Invalid type");
         break;
      }
   }

   return true;
}
static void
lower_load_store(nir_builder *b,
                 nir_intrinsic_instr *intrin,
                 glsl_type_size_align_func size_align)
{
   b->cursor = nir_before_instr(&intrin->instr);

   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
   nir_variable *var = nir_deref_instr_get_variable(deref);

   nir_ssa_def *offset =
      nir_iadd_imm(b, nir_build_deref_offset(b, deref, size_align),
                      var->data.location);

   unsigned align, UNUSED size;
   size_align(deref->type, &size, &align);

   if (intrin->intrinsic == nir_intrinsic_load_deref) {
      nir_intrinsic_instr *load =
         nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_scratch);
      load->num_components = intrin->num_components;
      load->src[0] = nir_src_for_ssa(offset);
      nir_intrinsic_set_align(load, align, 0);
      nir_ssa_dest_init(&load->instr, &load->dest,
                        intrin->dest.ssa.num_components,
                        intrin->dest.ssa.bit_size, NULL);
      nir_builder_instr_insert(b, &load->instr);

      nir_ssa_def *value = &load->dest.ssa;
      if (glsl_type_is_boolean(deref->type))
         value = nir_b2i32(b, value);

      nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
                               nir_src_for_ssa(&load->dest.ssa));
   } else {
      assert(intrin->intrinsic == nir_intrinsic_store_deref);

      assert(intrin->src[1].is_ssa);
      nir_ssa_def *value = intrin->src[1].ssa;
      if (glsl_type_is_boolean(deref->type))
         value = nir_i2b(b, value);

      nir_intrinsic_instr *store =
         nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_scratch);
      store->num_components = intrin->num_components;
      store->src[0] = nir_src_for_ssa(value);
      store->src[1] = nir_src_for_ssa(offset);
      nir_intrinsic_set_write_mask(store, nir_intrinsic_write_mask(intrin));
      nir_intrinsic_set_align(store, align, 0);
      nir_builder_instr_insert(b, &store->instr);
   }

   nir_instr_remove(&intrin->instr);
   nir_deref_instr_remove_if_unused(deref);
}
Esempio n. 6
0
static bool
remove_phis_block(nir_block *block, void *state)
{
   bool *progress = state;

   nir_foreach_instr_safe(block, instr) {
      if (instr->type != nir_instr_type_phi)
         break;

      nir_phi_instr *phi = nir_instr_as_phi(instr);

      nir_ssa_def *def = NULL;
      bool srcs_same = true;

      nir_foreach_phi_src(phi, src) {
         assert(src->src.is_ssa);

         /* For phi nodes at the beginning of loops, we may encounter some
          * sources from backedges that point back to the destination of the
          * same phi, i.e. something like:
          *
          * a = phi(a, b, ...)
          *
          * We can safely ignore these sources, since if all of the normal
          * sources point to the same definition, then that definition must
          * still dominate the phi node, and the phi will still always take
          * the value of that definition.
          */
         if (src->src.ssa == &phi->dest.ssa)
            continue;
         
         if (def == NULL) {
            def  = src->src.ssa;
         } else {
            if (src->src.ssa != def) {
               srcs_same = false;
               break;
            }
         }
      }

      if (!srcs_same)
         continue;

      /* We must have found at least one definition, since there must be at
       * least one forward edge.
       */
      assert(def != NULL);

      assert(phi->dest.is_ssa);
      nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
      nir_instr_remove(instr);

      *progress = true;
   }
Esempio n. 7
0
static void
opt_constant_if(nir_if *if_stmt, bool condition)
{
   /* First, we need to remove any phi nodes after the if by rewriting uses to
    * point to the correct source.
    */
   nir_block *after = nir_cf_node_as_block(nir_cf_node_next(&if_stmt->cf_node));
   nir_block *last_block =
      nir_cf_node_as_block(condition ? nir_if_last_then_node(if_stmt)
                                     : nir_if_last_else_node(if_stmt));

   nir_foreach_instr_safe(after, instr) {
      if (instr->type != nir_instr_type_phi)
         break;

      nir_phi_instr *phi = nir_instr_as_phi(instr);
      nir_ssa_def *def = NULL;
      nir_foreach_phi_src(phi, phi_src) {
         if (phi_src->pred != last_block)
            continue;

         assert(phi_src->src.is_ssa);
         def = phi_src->src.ssa;
      }

      assert(def);
      assert(phi->dest.is_ssa);
      nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
      nir_instr_remove(instr);
   }

   /* The control flow list we're about to paste in may include a jump at the
    * end, and in that case we have to delete the rest of the control flow
    * list after the if since it's unreachable and the validator will balk if
    * we don't.
    */

   if (!exec_list_is_empty(&last_block->instr_list)) {
      nir_instr *last_instr = nir_block_last_instr(last_block);
      if (last_instr->type == nir_instr_type_jump)
         remove_after_cf_node(&if_stmt->cf_node);
   }

   /* Finally, actually paste in the then or else branch and delete the if. */
   struct exec_list *cf_list = condition ? &if_stmt->then_list
                                         : &if_stmt->else_list;

   nir_cf_list list;
   nir_cf_extract(&list, nir_before_cf_list(cf_list),
                  nir_after_cf_list(cf_list));
   nir_cf_reinsert(&list, nir_after_cf_node(&if_stmt->cf_node));
   nir_cf_node_remove(&if_stmt->cf_node);
}
Esempio n. 8
0
    nir_foreach_instr_safe(block, instr) {
        if (instr->type != nir_instr_type_intrinsic)
            continue;

        nir_intrinsic_instr *copy = nir_instr_as_intrinsic(instr);
        if (copy->intrinsic != nir_intrinsic_copy_var)
            continue;

        nir_lower_var_copy_instr(copy, mem_ctx);

        nir_instr_remove(&copy->instr);
        ralloc_free(copy);
    }
Esempio n. 9
0
static bool
delete_block_cb(nir_block *block, void *_state)
{
   bool *progress = (bool *) _state;

   nir_foreach_instr_safe(block, instr) {
      if (!instr->pass_flags) {
         nir_instr_remove(instr);
         *progress = true;
      }
   }

   return true;
}
static bool
constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx)
{
   nir_const_value src[4];

   if (!instr->dest.dest.is_ssa)
      return false;

   for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
      if (!instr->src[i].src.is_ssa)
         return false;

      nir_instr *src_instr = instr->src[i].src.ssa->parent_instr;

      if (src_instr->type != nir_instr_type_load_const)
         return false;
      nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr);

      for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i);
           j++) {
         src[i].u[j] = load_const->value.u[instr->src[i].swizzle[j]];
      }

      /* We shouldn't have any source modifiers in the optimization loop. */
      assert(!instr->src[i].abs && !instr->src[i].negate);
   }

   /* We shouldn't have any saturate modifiers in the optimization loop. */
   assert(!instr->dest.saturate);

   nir_const_value dest =
      nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components,
                            src);

   nir_load_const_instr *new_instr =
      nir_load_const_instr_create(mem_ctx,
                                  instr->dest.dest.ssa.num_components);

   new_instr->value = dest;

   nir_instr_insert_before(&instr->instr, &new_instr->instr);

   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(&new_instr->def),
                            mem_ctx);

   nir_instr_remove(&instr->instr);
   ralloc_free(instr);

   return true;
}
Esempio n. 11
0
nir_alu_instr *
nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search,
                  const nir_search_value *replace, void *mem_ctx)
{
    uint8_t swizzle[4] = { 0, 0, 0, 0 };

    for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i)
        swizzle[i] = i;

    assert(instr->dest.dest.is_ssa);

    struct match_state state;
    state.variables_seen = 0;

    if (!match_expression(search, instr, instr->dest.dest.ssa.num_components,
                          swizzle, &state))
        return NULL;

    /* Inserting a mov may be unnecessary.  However, it's much easier to
     * simply let copy propagation clean this up than to try to go through
     * and rewrite swizzles ourselves.
     */
    nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov);
    mov->dest.write_mask = instr->dest.write_mask;
    nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
                      instr->dest.dest.ssa.num_components, NULL);

    mov->src[0] = construct_value(replace, nir_op_infos[instr->op].output_type,
                                  instr->dest.dest.ssa.num_components, &state,
                                  &instr->instr, mem_ctx);
    nir_instr_insert_before(&instr->instr, &mov->instr);

    nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                             nir_src_for_ssa(&mov->dest.dest.ssa), mem_ctx);

    /* We know this one has no more uses because we just rewrote them all,
     * so we can remove it.  The rest of the matched expression, however, we
     * don't know so much about.  We'll just let dead code clean them up.
     */
    nir_instr_remove(&instr->instr);

    return mov;
}
static bool
lower_indirect_block(nir_block *block, nir_builder *b,
                     nir_variable_mode modes)
{
   bool progress = false;

   nir_foreach_instr_safe(instr, block) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
      if (intrin->intrinsic != nir_intrinsic_load_var &&
          intrin->intrinsic != nir_intrinsic_store_var)
         continue;

      if (!deref_has_indirect(intrin->variables[0]))
         continue;

      /* Only lower variables whose mode is in the mask */
      if (!(modes & intrin->variables[0]->var->data.mode))
         continue;

      b->cursor = nir_before_instr(&intrin->instr);

      if (intrin->intrinsic == nir_intrinsic_load_var) {
         nir_ssa_def *result;
         emit_load_store(b, intrin, intrin->variables[0],
                         &intrin->variables[0]->deref, &result, NULL);
         nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(result));
      } else {
         assert(intrin->src[0].is_ssa);
         emit_load_store(b, intrin, intrin->variables[0],
                         &intrin->variables[0]->deref, NULL, intrin->src[0].ssa);
      }
      nir_instr_remove(&intrin->instr);
      progress = true;
   }

   return progress;
}
Esempio n. 13
0
static bool
cse_block(nir_block *block, struct set *instr_set)
{
   bool progress = false;

   nir_foreach_instr_safe(instr, block) {
      if (nir_instr_set_add_or_rewrite(instr_set, instr)) {
         progress = true;
         nir_instr_remove(instr);
      }
   }

   for (unsigned i = 0; i < block->num_dom_children; i++) {
      nir_block *child = block->dom_children[i];
      progress |= cse_block(child, instr_set);
   }

   nir_foreach_instr(instr, block)
     nir_instr_set_remove(instr_set, instr);

   return progress;
}
Esempio n. 14
0
static void
lower_reduction(nir_alu_instr *instr, nir_op chan_op, nir_op merge_op,
                void *mem_ctx)
{
   unsigned num_components = nir_op_infos[instr->op].input_sizes[0];

   nir_ssa_def *last = NULL;
   for (unsigned i = 0; i < num_components; i++) {
      nir_alu_instr *chan = nir_alu_instr_create(mem_ctx, chan_op);
      nir_alu_ssa_dest_init(chan, 1);
      nir_alu_src_copy(&chan->src[0], &instr->src[0], mem_ctx);
      chan->src[0].swizzle[0] = chan->src[0].swizzle[i];
      if (nir_op_infos[chan_op].num_inputs > 1) {
         assert(nir_op_infos[chan_op].num_inputs == 2);
         nir_alu_src_copy(&chan->src[1], &instr->src[1], mem_ctx);
         chan->src[1].swizzle[0] = chan->src[1].swizzle[i];
      }

      nir_instr_insert_before(&instr->instr, &chan->instr);

      if (i == 0) {
         last = &chan->dest.dest.ssa;
      } else {
         nir_alu_instr *merge = nir_alu_instr_create(mem_ctx, merge_op);
         nir_alu_ssa_dest_init(merge, 1);
         merge->dest.write_mask = 1;
         merge->src[0].src = nir_src_for_ssa(last);
         merge->src[1].src = nir_src_for_ssa(&chan->dest.dest.ssa);
         nir_instr_insert_before(&instr->instr, &merge->instr);
         last = &merge->dest.dest.ssa;
      }
   }

   assert(instr->dest.write_mask == 1);
   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(last),
                            mem_ctx);
   nir_instr_remove(&instr->instr);
}
Esempio n. 15
0
static bool
nir_lower_io_block(nir_block *block, void *void_state)
{
   struct lower_io_state *state = void_state;

   nir_foreach_instr_safe(block, instr) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);

      switch (intrin->intrinsic) {
      case nir_intrinsic_load_var: {
         nir_variable_mode mode = intrin->variables[0]->var->data.mode;
         if (mode != nir_var_shader_in && mode != nir_var_uniform)
            continue;

         bool has_indirect = deref_has_indirect(intrin->variables[0]);

         /* Figure out the opcode */
         nir_intrinsic_op load_op;
         switch (mode) {
         case nir_var_shader_in:
            load_op = has_indirect ? nir_intrinsic_load_input_indirect :
                                     nir_intrinsic_load_input;
            break;
         case nir_var_uniform:
            load_op = has_indirect ? nir_intrinsic_load_uniform_indirect :
                                     nir_intrinsic_load_uniform;
            break;
         default:
            unreachable("Unknown variable mode");
         }

         nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->mem_ctx,
                                                                load_op);
         load->num_components = intrin->num_components;

         nir_src indirect;
         unsigned offset = get_io_offset(intrin->variables[0],
                                         &intrin->instr, &indirect, state);
         offset += intrin->variables[0]->var->data.driver_location;

         load->const_index[0] = offset;

         if (has_indirect)
            load->src[0] = indirect;

         if (intrin->dest.is_ssa) {
            nir_ssa_dest_init(&load->instr, &load->dest,
                              intrin->num_components, NULL);
            nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
                                     nir_src_for_ssa(&load->dest.ssa),
                                     state->mem_ctx);
         } else {
            nir_dest_copy(&load->dest, &intrin->dest, state->mem_ctx);
         }

         nir_instr_insert_before(&intrin->instr, &load->instr);
         nir_instr_remove(&intrin->instr);
         break;
      }

      case nir_intrinsic_store_var: {
         if (intrin->variables[0]->var->data.mode != nir_var_shader_out)
            continue;

         bool has_indirect = deref_has_indirect(intrin->variables[0]);

         nir_intrinsic_op store_op;
         if (has_indirect) {
            store_op = nir_intrinsic_store_output_indirect;
         } else {
            store_op = nir_intrinsic_store_output;
         }

         nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
                                                                 store_op);
         store->num_components = intrin->num_components;

         nir_src indirect;
         unsigned offset = get_io_offset(intrin->variables[0],
                                         &intrin->instr, &indirect, state);
         offset += intrin->variables[0]->var->data.driver_location;

         store->const_index[0] = offset;

         nir_src_copy(&store->src[0], &intrin->src[0], state->mem_ctx);

         if (has_indirect)
            store->src[1] = indirect;

         nir_instr_insert_before(&intrin->instr, &store->instr);
         nir_instr_remove(&intrin->instr);
         break;
      }

      default:
         break;
      }
   }

   return true;
}
Esempio n. 16
0
static void
lower_alu_instr_scalar(nir_alu_instr *instr, void *mem_ctx)
{
   unsigned num_src = nir_op_infos[instr->op].num_inputs;
   unsigned i, chan;

   assert(instr->dest.dest.is_ssa);
   assert(instr->dest.write_mask != 0);

#define LOWER_REDUCTION(name, chan, merge) \
   case name##2: \
   case name##3: \
   case name##4: \
      lower_reduction(instr, chan, merge, mem_ctx); \
      break;

   switch (instr->op) {
   case nir_op_vec4:
   case nir_op_vec3:
   case nir_op_vec2:
      /* We don't need to scalarize these ops, they're the ones generated to
       * group up outputs into a value that can be SSAed.
       */
      return;

      LOWER_REDUCTION(nir_op_fdot, nir_op_fmul, nir_op_fadd);
      LOWER_REDUCTION(nir_op_ball_fequal, nir_op_feq, nir_op_iand);
      LOWER_REDUCTION(nir_op_ball_iequal, nir_op_ieq, nir_op_iand);
      LOWER_REDUCTION(nir_op_bany_fnequal, nir_op_fne, nir_op_ior);
      LOWER_REDUCTION(nir_op_bany_inequal, nir_op_ine, nir_op_ior);
      LOWER_REDUCTION(nir_op_fall_equal, nir_op_seq, nir_op_fand);
      LOWER_REDUCTION(nir_op_fany_nequal, nir_op_sne, nir_op_for);
      LOWER_REDUCTION(nir_op_ball, nir_op_imov, nir_op_iand);
      LOWER_REDUCTION(nir_op_bany, nir_op_imov, nir_op_ior);
      LOWER_REDUCTION(nir_op_fall, nir_op_fmov, nir_op_fand);
      LOWER_REDUCTION(nir_op_fany, nir_op_fmov, nir_op_for);

   default:
      break;
   }

   if (instr->dest.dest.ssa.num_components == 1)
      return;

   unsigned num_components = instr->dest.dest.ssa.num_components;
   static const nir_op nir_op_map[] = {nir_op_vec2, nir_op_vec3, nir_op_vec4};
   nir_alu_instr *vec_instr =
      nir_alu_instr_create(mem_ctx, nir_op_map[num_components - 2]);
   nir_alu_ssa_dest_init(vec_instr, num_components);

   for (chan = 0; chan < 4; chan++) {
      if (!(instr->dest.write_mask & (1 << chan)))
         continue;

      nir_alu_instr *lower = nir_alu_instr_create(mem_ctx, instr->op);
      for (i = 0; i < num_src; i++) {
         /* We only handle same-size-as-dest (input_sizes[] == 0) or scalar
          * args (input_sizes[] == 1).
          */
         assert(nir_op_infos[instr->op].input_sizes[i] < 2);
         unsigned src_chan = (nir_op_infos[instr->op].input_sizes[i] == 1 ?
                              0 : chan);

         nir_alu_src_copy(&lower->src[i], &instr->src[i], mem_ctx);
         for (int j = 0; j < 4; j++)
            lower->src[i].swizzle[j] = instr->src[i].swizzle[src_chan];
      }

      nir_alu_ssa_dest_init(lower, 1);
      lower->dest.saturate = instr->dest.saturate;
      vec_instr->src[chan].src = nir_src_for_ssa(&lower->dest.dest.ssa);

      nir_instr_insert_before(&instr->instr, &lower->instr);
   }

   nir_instr_insert_before(&instr->instr, &vec_instr->instr);

   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                            nir_src_for_ssa(&vec_instr->dest.dest.ssa),
                            mem_ctx);

   nir_instr_remove(&instr->instr);
}
static bool
lower_locals_to_regs_block(nir_block *block,
                           struct locals_to_regs_state *state)
{
   nir_builder *b = &state->builder;

   nir_foreach_instr_safe(instr, block) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);

      switch (intrin->intrinsic) {
      case nir_intrinsic_load_deref: {
         nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
         if (deref->mode != nir_var_function_temp)
            continue;

         b->cursor = nir_before_instr(&intrin->instr);

         nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
         mov->src[0].src = get_deref_reg_src(deref, state);
         mov->dest.write_mask = (1 << intrin->num_components) - 1;
         if (intrin->dest.is_ssa) {
            nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
                              intrin->num_components,
                              intrin->dest.ssa.bit_size, NULL);
            nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
                                     nir_src_for_ssa(&mov->dest.dest.ssa));
         } else {
            nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
         }
         nir_builder_instr_insert(b, &mov->instr);

         nir_instr_remove(&intrin->instr);
         state->progress = true;
         break;
      }

      case nir_intrinsic_store_deref: {
         nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
         if (deref->mode != nir_var_function_temp)
            continue;

         b->cursor = nir_before_instr(&intrin->instr);

         nir_src reg_src = get_deref_reg_src(deref, state);

         nir_alu_instr *mov = nir_alu_instr_create(b->shader, nir_op_imov);
         nir_src_copy(&mov->src[0].src, &intrin->src[1], mov);
         mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
         mov->dest.dest.is_ssa = false;
         mov->dest.dest.reg.reg = reg_src.reg.reg;
         mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
         mov->dest.dest.reg.indirect = reg_src.reg.indirect;

         nir_builder_instr_insert(b, &mov->instr);

         nir_instr_remove(&intrin->instr);
         state->progress = true;
         break;
      }

      case nir_intrinsic_copy_deref:
         unreachable("There should be no copies whatsoever at this point");
         break;

      default:
         continue;
      }
   }

   return true;
}
Esempio n. 18
0
static void
lower_instr(nir_intrinsic_instr *instr,
            lower_atomic_state *state)
{
   nir_intrinsic_op op;
   switch (instr->intrinsic) {
   case nir_intrinsic_atomic_counter_read_var:
      op = nir_intrinsic_atomic_counter_read;
      break;

   case nir_intrinsic_atomic_counter_inc_var:
      op = nir_intrinsic_atomic_counter_inc;
      break;

   case nir_intrinsic_atomic_counter_dec_var:
      op = nir_intrinsic_atomic_counter_dec;
      break;

   default:
      return;
   }

   if (instr->variables[0]->var->data.mode != nir_var_uniform &&
       instr->variables[0]->var->data.mode != nir_var_shader_storage)
      return; /* atomics passed as function arguments can't be lowered */

   void *mem_ctx = ralloc_parent(instr);
   unsigned uniform_loc = instr->variables[0]->var->data.location;

   nir_intrinsic_instr *new_instr = nir_intrinsic_instr_create(mem_ctx, op);
   new_instr->const_index[0] =
      state->shader_program->UniformStorage[uniform_loc].opaque[state->shader->stage].index;

   nir_load_const_instr *offset_const = nir_load_const_instr_create(mem_ctx, 1);
   offset_const->value.u[0] = instr->variables[0]->var->data.atomic.offset;

   nir_instr_insert_before(&instr->instr, &offset_const->instr);

   nir_ssa_def *offset_def = &offset_const->def;

   nir_deref *tail = &instr->variables[0]->deref;
   while (tail->child != NULL) {
      assert(tail->child->deref_type == nir_deref_type_array);
      nir_deref_array *deref_array = nir_deref_as_array(tail->child);
      tail = tail->child;

      unsigned child_array_elements = tail->child != NULL ?
         glsl_get_aoa_size(tail->type) : 1;

      offset_const->value.u[0] += deref_array->base_offset *
         child_array_elements * ATOMIC_COUNTER_SIZE;

      if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
         nir_load_const_instr *atomic_counter_size =
               nir_load_const_instr_create(mem_ctx, 1);
         atomic_counter_size->value.u[0] = child_array_elements * ATOMIC_COUNTER_SIZE;
         nir_instr_insert_before(&instr->instr, &atomic_counter_size->instr);

         nir_alu_instr *mul = nir_alu_instr_create(mem_ctx, nir_op_imul);
         nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, NULL);
         mul->dest.write_mask = 0x1;
         nir_src_copy(&mul->src[0].src, &deref_array->indirect, mul);
         mul->src[1].src.is_ssa = true;
         mul->src[1].src.ssa = &atomic_counter_size->def;
         nir_instr_insert_before(&instr->instr, &mul->instr);

         nir_alu_instr *add = nir_alu_instr_create(mem_ctx, nir_op_iadd);
         nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, NULL);
         add->dest.write_mask = 0x1;
         add->src[0].src.is_ssa = true;
         add->src[0].src.ssa = &mul->dest.dest.ssa;
         add->src[1].src.is_ssa = true;
         add->src[1].src.ssa = offset_def;
         nir_instr_insert_before(&instr->instr, &add->instr);

         offset_def = &add->dest.dest.ssa;
      }
   }

   new_instr->src[0].is_ssa = true;
   new_instr->src[0].ssa = offset_def;

   if (instr->dest.is_ssa) {
      nir_ssa_dest_init(&new_instr->instr, &new_instr->dest,
                        instr->dest.ssa.num_components, NULL);
      nir_ssa_def_rewrite_uses(&instr->dest.ssa,
                               nir_src_for_ssa(&new_instr->dest.ssa));
   } else {
      nir_dest_copy(&new_instr->dest, &instr->dest, mem_ctx);
   }

   nir_instr_insert_before(&instr->instr, &new_instr->instr);
   nir_instr_remove(&instr->instr);
}
Esempio n. 19
0
   nir_foreach_instr_safe(instr, block) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);

      switch (intrin->intrinsic) {
      case nir_intrinsic_load_var: {
         if (intrin->variables[0]->var->data.mode != nir_var_local)
            continue;

         nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
         mov->src[0].src = get_deref_reg_src(intrin->variables[0],
                                             &intrin->instr, state);
         mov->dest.write_mask = (1 << intrin->num_components) - 1;
         if (intrin->dest.is_ssa) {
            nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
                              intrin->num_components,
                              intrin->dest.ssa.bit_size, NULL);
            nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
                                     nir_src_for_ssa(&mov->dest.dest.ssa));
         } else {
            nir_dest_copy(&mov->dest.dest, &intrin->dest, &mov->instr);
         }
         nir_instr_insert_before(&intrin->instr, &mov->instr);

         nir_instr_remove(&intrin->instr);
         state->progress = true;
         break;
      }

      case nir_intrinsic_store_var: {
         if (intrin->variables[0]->var->data.mode != nir_var_local)
            continue;

         nir_src reg_src = get_deref_reg_src(intrin->variables[0],
                                             &intrin->instr, state);

         nir_alu_instr *mov = nir_alu_instr_create(state->shader, nir_op_imov);
         nir_src_copy(&mov->src[0].src, &intrin->src[0], mov);
         mov->dest.write_mask = nir_intrinsic_write_mask(intrin);
         mov->dest.dest.is_ssa = false;
         mov->dest.dest.reg.reg = reg_src.reg.reg;
         mov->dest.dest.reg.base_offset = reg_src.reg.base_offset;
         mov->dest.dest.reg.indirect = reg_src.reg.indirect;

         nir_instr_insert_before(&intrin->instr, &mov->instr);

         nir_instr_remove(&intrin->instr);
         state->progress = true;
         break;
      }

      case nir_intrinsic_copy_var:
         unreachable("There should be no copies whatsoever at this point");
         break;

      default:
         continue;
      }
   }
Esempio n. 20
0
static bool
constant_fold_alu_instr(nir_alu_instr *instr, void *mem_ctx)
{
   nir_const_value src[NIR_MAX_VEC_COMPONENTS];

   if (!instr->dest.dest.is_ssa)
      return false;

   /* In the case that any outputs/inputs have unsized types, then we need to
    * guess the bit-size. In this case, the validator ensures that all
    * bit-sizes match so we can just take the bit-size from first
    * output/input with an unsized type. If all the outputs/inputs are sized
    * then we don't need to guess the bit-size at all because the code we
    * generate for constant opcodes in this case already knows the sizes of
    * the types involved and does not need the provided bit-size for anything
    * (although it still requires to receive a valid bit-size).
    */
   unsigned bit_size = 0;
   if (!nir_alu_type_get_type_size(nir_op_infos[instr->op].output_type))
      bit_size = instr->dest.dest.ssa.bit_size;

   for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
      if (!instr->src[i].src.is_ssa)
         return false;

      if (bit_size == 0 &&
          !nir_alu_type_get_type_size(nir_op_infos[instr->op].input_sizes[i])) {
         bit_size = instr->src[i].src.ssa->bit_size;
      }

      nir_instr *src_instr = instr->src[i].src.ssa->parent_instr;

      if (src_instr->type != nir_instr_type_load_const)
         return false;
      nir_load_const_instr* load_const = nir_instr_as_load_const(src_instr);

      for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(instr, i);
           j++) {
         switch(load_const->def.bit_size) {
         case 64:
            src[i].u64[j] = load_const->value.u64[instr->src[i].swizzle[j]];
            break;
         case 32:
            src[i].u32[j] = load_const->value.u32[instr->src[i].swizzle[j]];
            break;
         case 16:
            src[i].u16[j] = load_const->value.u16[instr->src[i].swizzle[j]];
            break;
         case 8:
            src[i].u8[j] = load_const->value.u8[instr->src[i].swizzle[j]];
            break;
         default:
            unreachable("Invalid bit size");
         }
      }

      /* We shouldn't have any source modifiers in the optimization loop. */
      assert(!instr->src[i].abs && !instr->src[i].negate);
   }

   if (bit_size == 0)
      bit_size = 32;

   /* We shouldn't have any saturate modifiers in the optimization loop. */
   assert(!instr->dest.saturate);

   nir_const_value dest =
      nir_eval_const_opcode(instr->op, instr->dest.dest.ssa.num_components,
                            bit_size, src);

   nir_load_const_instr *new_instr =
      nir_load_const_instr_create(mem_ctx,
                                  instr->dest.dest.ssa.num_components,
                                  instr->dest.dest.ssa.bit_size);

   new_instr->value = dest;

   nir_instr_insert_before(&instr->instr, &new_instr->instr);

   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                            nir_src_for_ssa(&new_instr->def));

   nir_instr_remove(&instr->instr);
   ralloc_free(instr);

   return true;
}
Esempio n. 21
0
static void
vc4_nir_lower_txf_ms_instr(struct vc4_compile *c, nir_builder *b,
                           nir_tex_instr *txf_ms)
{
    if (txf_ms->op != nir_texop_txf_ms)
        return;

    b->cursor = nir_before_instr(&txf_ms->instr);

    nir_tex_instr *txf = nir_tex_instr_create(c->s, 1);
    txf->op = nir_texop_txf;
    txf->sampler = txf_ms->sampler;
    txf->sampler_index = txf_ms->sampler_index;
    txf->coord_components = txf_ms->coord_components;
    txf->is_shadow = txf_ms->is_shadow;
    txf->is_new_style_shadow = txf_ms->is_new_style_shadow;

    nir_ssa_def *coord = NULL, *sample_index = NULL;
    for (int i = 0; i < txf_ms->num_srcs; i++) {
        assert(txf_ms->src[i].src.is_ssa);

        switch (txf_ms->src[i].src_type) {
        case nir_tex_src_coord:
            coord = txf_ms->src[i].src.ssa;
            break;
        case nir_tex_src_ms_index:
            sample_index = txf_ms->src[i].src.ssa;
            break;
        default:
            unreachable("Unknown txf_ms src\n");
        }
    }
    assert(coord);
    assert(sample_index);

    nir_ssa_def *x = nir_channel(b, coord, 0);
    nir_ssa_def *y = nir_channel(b, coord, 1);

    uint32_t tile_w = 32;
    uint32_t tile_h = 32;
    uint32_t tile_w_shift = 5;
    uint32_t tile_h_shift = 5;
    uint32_t tile_size = (tile_h * tile_w *
                          VC4_MAX_SAMPLES * sizeof(uint32_t));
    unsigned unit = txf_ms->sampler_index;
    uint32_t w = align(c->key->tex[unit].msaa_width, tile_w);
    uint32_t w_tiles = w / tile_w;

    nir_ssa_def *x_tile = nir_ushr(b, x, nir_imm_int(b, tile_w_shift));
    nir_ssa_def *y_tile = nir_ushr(b, y, nir_imm_int(b, tile_h_shift));
    nir_ssa_def *tile_addr = nir_iadd(b,
                                      nir_imul(b, x_tile,
                                              nir_imm_int(b, tile_size)),
                                      nir_imul(b, y_tile,
                                              nir_imm_int(b, (w_tiles *
                                                      tile_size))));
    nir_ssa_def *x_subspan = nir_iand(b, x,
                                      nir_imm_int(b, (tile_w - 1) & ~1));
    nir_ssa_def *y_subspan = nir_iand(b, y,
                                      nir_imm_int(b, (tile_h - 1) & ~1));
    nir_ssa_def *subspan_addr = nir_iadd(b,
                                         nir_imul(b, x_subspan,
                                                 nir_imm_int(b, 2 * VC4_MAX_SAMPLES * sizeof(uint32_t))),
                                         nir_imul(b, y_subspan,
                                                 nir_imm_int(b,
                                                         tile_w *
                                                         VC4_MAX_SAMPLES *
                                                         sizeof(uint32_t))));

    nir_ssa_def *pixel_addr = nir_ior(b,
                                      nir_iand(b,
                                              nir_ishl(b, x,
                                                      nir_imm_int(b, 2)),
                                              nir_imm_int(b, (1 << 2))),
                                      nir_iand(b,
                                              nir_ishl(b, y,
                                                      nir_imm_int(b, 3)),
                                              nir_imm_int(b, (1 << 3))));

    nir_ssa_def *sample_addr = nir_ishl(b, sample_index, nir_imm_int(b, 4));

    nir_ssa_def *addr = nir_iadd(b,
                                 nir_ior(b, sample_addr, pixel_addr),
                                 nir_iadd(b, subspan_addr, tile_addr));

    txf->src[0].src_type = nir_tex_src_coord;
    txf->src[0].src = nir_src_for_ssa(nir_vec2(b, addr, nir_imm_int(b, 0)));
    nir_ssa_dest_init(&txf->instr, &txf->dest, 4, NULL);
    nir_builder_instr_insert(b, &txf->instr);
    nir_ssa_def_rewrite_uses(&txf_ms->dest.ssa,
                             nir_src_for_ssa(&txf->dest.ssa));
    nir_instr_remove(&txf_ms->instr);
}
Esempio n. 22
0
static bool
remove_phis_block(nir_block *block, nir_builder *b)
{
   bool progress = false;

   nir_foreach_instr_safe(instr, block) {
      if (instr->type != nir_instr_type_phi)
         break;

      nir_phi_instr *phi = nir_instr_as_phi(instr);

      nir_ssa_def *def = NULL;
      nir_alu_instr *mov = NULL;
      bool srcs_same = true;

      nir_foreach_phi_src(src, phi) {
         assert(src->src.is_ssa);

         /* For phi nodes at the beginning of loops, we may encounter some
          * sources from backedges that point back to the destination of the
          * same phi, i.e. something like:
          *
          * a = phi(a, b, ...)
          *
          * We can safely ignore these sources, since if all of the normal
          * sources point to the same definition, then that definition must
          * still dominate the phi node, and the phi will still always take
          * the value of that definition.
          */
         if (src->src.ssa == &phi->dest.ssa)
            continue;
         
         if (def == NULL) {
            def  = src->src.ssa;
            mov = get_parent_mov(def);
         } else {
            if (src->src.ssa != def && !matching_mov(mov, src->src.ssa)) {
               srcs_same = false;
               break;
            }
         }
      }

      if (!srcs_same)
         continue;

      /* We must have found at least one definition, since there must be at
       * least one forward edge.
       */
      assert(def != NULL);

      if (mov) {
         /* If the sources were all movs from the same source with the same
          * swizzle, then we can't just pick a random move because it may not
          * dominate the phi node. Instead, we need to emit our own move after
          * the phi which uses the shared source, and rewrite uses of the phi
          * to use the move instead. This is ok, because while the movs may
          * not all dominate the phi node, their shared source does.
          */

         b->cursor = nir_after_phis(block);
         def = nir_mov_alu(b, mov->src[0], def->num_components);
      }

      assert(phi->dest.is_ssa);
      nir_ssa_def_rewrite_uses(&phi->dest.ssa, nir_src_for_ssa(def));
      nir_instr_remove(instr);

      progress = true;
   }
Esempio n. 23
0
static void
lower_instr(nir_intrinsic_instr *instr, nir_function_impl *impl)
{
   nir_intrinsic_op op;
   switch (instr->intrinsic) {
   case nir_intrinsic_atomic_counter_read_var:
      op = nir_intrinsic_atomic_counter_read;
      break;

   case nir_intrinsic_atomic_counter_inc_var:
      op = nir_intrinsic_atomic_counter_inc;
      break;

   case nir_intrinsic_atomic_counter_dec_var:
      op = nir_intrinsic_atomic_counter_dec;
      break;

   default:
      return;
   }

   if (instr->variables[0]->var->data.mode != nir_var_uniform)
      return; /* atomics passed as function arguments can't be lowered */

   void *mem_ctx = ralloc_parent(instr);

   nir_intrinsic_instr *new_instr = nir_intrinsic_instr_create(mem_ctx, op);
   new_instr->const_index[0] =
      (int) instr->variables[0]->var->data.atomic.buffer_index;

   nir_load_const_instr *offset_const = nir_load_const_instr_create(mem_ctx, 1);
   offset_const->value.u[0] = instr->variables[0]->var->data.atomic.offset;

   nir_instr_insert_before(&instr->instr, &offset_const->instr);

   nir_ssa_def *offset_def = &offset_const->def;

   if (instr->variables[0]->deref.child != NULL) {
      assert(instr->variables[0]->deref.child->deref_type ==
             nir_deref_type_array);
      nir_deref_array *deref_array =
         nir_deref_as_array(instr->variables[0]->deref.child);
      assert(deref_array->deref.child == NULL);

      offset_const->value.u[0] +=
         deref_array->base_offset * ATOMIC_COUNTER_SIZE;

      if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
         nir_load_const_instr *atomic_counter_size =
               nir_load_const_instr_create(mem_ctx, 1);
         atomic_counter_size->value.u[0] = ATOMIC_COUNTER_SIZE;
         nir_instr_insert_before(&instr->instr, &atomic_counter_size->instr);

         nir_alu_instr *mul = nir_alu_instr_create(mem_ctx, nir_op_imul);
         nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, NULL);
         mul->dest.write_mask = 0x1;
         nir_src_copy(&mul->src[0].src, &deref_array->indirect, mem_ctx);
         mul->src[1].src.is_ssa = true;
         mul->src[1].src.ssa = &atomic_counter_size->def;
         nir_instr_insert_before(&instr->instr, &mul->instr);

         nir_alu_instr *add = nir_alu_instr_create(mem_ctx, nir_op_iadd);
         nir_ssa_dest_init(&add->instr, &add->dest.dest, 1, NULL);
         add->dest.write_mask = 0x1;
         add->src[0].src.is_ssa = true;
         add->src[0].src.ssa = &mul->dest.dest.ssa;
         add->src[1].src.is_ssa = true;
         add->src[1].src.ssa = &offset_const->def;
         nir_instr_insert_before(&instr->instr, &add->instr);

         offset_def = &add->dest.dest.ssa;
      }
   }

   new_instr->src[0].is_ssa = true;
   new_instr->src[0].ssa = offset_def;;

   if (instr->dest.is_ssa) {
      nir_ssa_dest_init(&new_instr->instr, &new_instr->dest,
                        instr->dest.ssa.num_components, NULL);
      nir_ssa_def_rewrite_uses(&instr->dest.ssa,
                               nir_src_for_ssa(&new_instr->dest.ssa),
                               mem_ctx);
   } else {
      nir_dest_copy(&new_instr->dest, &instr->dest, mem_ctx);
   }

   nir_instr_insert_before(&instr->instr, &new_instr->instr);
   nir_instr_remove(&instr->instr);
}