static void
emit_mfence_after_loop (struct loop *loop)
{
VEC (edge, heap) *exits = get_loop_exit_edges (loop);
edge exit;
tree call;
block_stmt_iterator bsi;
unsigned i;
for (i = 0; VEC_iterate (edge, exits, i, exit); i++)
{
call = build_function_call_expr (FENCE_FOLLOWING_MOVNT, NULL_TREE);
if (!single_pred_p (exit->dest)
/* If possible, we prefer not to insert the fence on other paths
in cfg. */
&& !(exit->flags & EDGE_ABNORMAL))
split_loop_exit_edge (exit);
bsi = bsi_after_labels (exit->dest);
bsi_insert_before (&bsi, call, BSI_NEW_STMT);
mark_virtual_ops_for_renaming (call);
}
VEC_free (edge, heap, exits);
update_ssa (TODO_update_ssa_only_virtuals);
}
static void
tree_gen_one_value_profiler (histogram_value value, unsigned tag, unsigned base)
{
tree stmt = value->hvalue.stmt;
block_stmt_iterator bsi = bsi_for_stmt (stmt);
tree ref = tree_coverage_counter_ref (tag, base), ref_ptr;
tree args, call, val;
ref_ptr = force_gimple_operand_bsi (&bsi,
build_addr (ref, current_function_decl),
true, NULL_TREE);
val = prepare_instrumented_value (&bsi, value);
args = tree_cons (NULL_TREE, ref_ptr,
tree_cons (NULL_TREE, val,
NULL_TREE));
call = build_function_call_expr (tree_one_value_profiler_fn, args);
bsi_insert_before (&bsi, call, BSI_SAME_STMT);
}
static void
issue_prefetch_ref (struct mem_ref *ref, unsigned unroll_factor, unsigned ahead)
{
HOST_WIDE_INT delta;
tree addr, addr_base, prefetch, write_p, local;
block_stmt_iterator bsi;
unsigned n_prefetches, ap;
bool nontemporal = ref->reuse_distance >= L2_CACHE_SIZE_BYTES;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Issued%s prefetch for %p.\n",
nontemporal ? " nontemporal" : "",
(void *) ref);
bsi = bsi_for_stmt (ref->stmt);
n_prefetches = ((unroll_factor + ref->prefetch_mod - 1)
/ ref->prefetch_mod);
addr_base = build_fold_addr_expr_with_type (ref->mem, ptr_type_node);
addr_base = force_gimple_operand_bsi (&bsi, unshare_expr (addr_base),
true, NULL, true, BSI_SAME_STMT);
write_p = ref->write_p ? integer_one_node : integer_zero_node;
local = build_int_cst (integer_type_node, nontemporal ? 0 : 3);
for (ap = 0; ap < n_prefetches; ap++)
{
/* Determine the address to prefetch. */
delta = (ahead + ap * ref->prefetch_mod) * ref->group->step;
addr = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
addr_base, size_int (delta));
addr = force_gimple_operand_bsi (&bsi, unshare_expr (addr), true, NULL,
true, BSI_SAME_STMT);
/* Create the prefetch instruction. */
prefetch = build_call_expr (built_in_decls[BUILT_IN_PREFETCH],
3, addr, write_p, local);
bsi_insert_before (&bsi, prefetch, BSI_SAME_STMT);
}
}
static void
issue_prefetch_ref (struct mem_ref *ref, unsigned unroll_factor, unsigned ahead)
{
HOST_WIDE_INT delta;
tree addr, addr_base, prefetch, params, write_p;
block_stmt_iterator bsi;
unsigned n_prefetches, ap;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Issued prefetch for %p.\n", (void *) ref);
bsi = bsi_for_stmt (ref->stmt);
n_prefetches = ((unroll_factor + ref->prefetch_mod - 1)
/ ref->prefetch_mod);
addr_base = build_fold_addr_expr_with_type (ref->mem, ptr_type_node);
addr_base = force_gimple_operand_bsi (&bsi, unshare_expr (addr_base), true, NULL);
for (ap = 0; ap < n_prefetches; ap++)
{
/* Determine the address to prefetch. */
delta = (ahead + ap * ref->prefetch_mod) * ref->group->step;
addr = fold_build2 (PLUS_EXPR, ptr_type_node,
addr_base, build_int_cst (ptr_type_node, delta));
addr = force_gimple_operand_bsi (&bsi, unshare_expr (addr), true, NULL);
/* Create the prefetch instruction. */
write_p = ref->write_p ? integer_one_node : integer_zero_node;
params = tree_cons (NULL_TREE, addr,
tree_cons (NULL_TREE, write_p, NULL_TREE));
prefetch = build_function_call_expr (built_in_decls[BUILT_IN_PREFETCH],
params);
bsi_insert_before (&bsi, prefetch, BSI_SAME_STMT);
}
}
static tree
rewrite_bittest (block_stmt_iterator *bsi)
{
tree stmt, lhs, rhs, var, name, use_stmt, stmt1, stmt2, t;
use_operand_p use;
stmt = bsi_stmt (*bsi);
lhs = GENERIC_TREE_OPERAND (stmt, 0);
rhs = GENERIC_TREE_OPERAND (stmt, 1);
/* Verify that the single use of lhs is a comparison against zero. */
if (TREE_CODE (lhs) != SSA_NAME
|| !single_imm_use (lhs, &use, &use_stmt)
|| TREE_CODE (use_stmt) != COND_EXPR)
return stmt;
t = COND_EXPR_COND (use_stmt);
if (TREE_OPERAND (t, 0) != lhs
|| (TREE_CODE (t) != NE_EXPR
&& TREE_CODE (t) != EQ_EXPR)
|| !integer_zerop (TREE_OPERAND (t, 1)))
return stmt;
/* Get at the operands of the shift. The rhs is TMP1 & 1. */
stmt1 = SSA_NAME_DEF_STMT (TREE_OPERAND (rhs, 0));
if (TREE_CODE (stmt1) != GIMPLE_MODIFY_STMT)
return stmt;
/* There is a conversion in between possibly inserted by fold. */
t = GIMPLE_STMT_OPERAND (stmt1, 1);
if (TREE_CODE (t) == NOP_EXPR
|| TREE_CODE (t) == CONVERT_EXPR)
{
t = TREE_OPERAND (t, 0);
if (TREE_CODE (t) != SSA_NAME
|| !has_single_use (t))
return stmt;
stmt1 = SSA_NAME_DEF_STMT (t);
if (TREE_CODE (stmt1) != GIMPLE_MODIFY_STMT)
return stmt;
t = GIMPLE_STMT_OPERAND (stmt1, 1);
}
/* Verify that B is loop invariant but A is not. Verify that with
all the stmt walking we are still in the same loop. */
if (TREE_CODE (t) == RSHIFT_EXPR
&& loop_containing_stmt (stmt1) == loop_containing_stmt (stmt)
&& outermost_invariant_loop_expr (TREE_OPERAND (t, 1),
loop_containing_stmt (stmt1)) != NULL
&& outermost_invariant_loop_expr (TREE_OPERAND (t, 0),
loop_containing_stmt (stmt1)) == NULL)
{
tree a = TREE_OPERAND (t, 0);
tree b = TREE_OPERAND (t, 1);
/* 1 << B */
var = create_tmp_var (TREE_TYPE (a), "shifttmp");
add_referenced_var (var);
t = fold_build2 (LSHIFT_EXPR, TREE_TYPE (a),
build_int_cst (TREE_TYPE (a), 1), b);
stmt1 = build_gimple_modify_stmt (var, t);
name = make_ssa_name (var, stmt1);
GIMPLE_STMT_OPERAND (stmt1, 0) = name;
/* A & (1 << B) */
t = fold_build2 (BIT_AND_EXPR, TREE_TYPE (a), a, name);
stmt2 = build_gimple_modify_stmt (var, t);
name = make_ssa_name (var, stmt2);
GIMPLE_STMT_OPERAND (stmt2, 0) = name;
/* Replace the SSA_NAME we compare against zero. Adjust
the type of zero accordingly. */
SET_USE (use, name);
TREE_OPERAND (COND_EXPR_COND (use_stmt), 1)
= build_int_cst_type (TREE_TYPE (name), 0);
bsi_insert_before (bsi, stmt1, BSI_SAME_STMT);
bsi_replace (bsi, stmt2, true);
return stmt1;
}
return stmt;
}