/* Return true if we should ignore the basic block for purposes of tracing. */
static bool
ignore_bb_p (basic_block bb)
{
if (bb->index < NUM_FIXED_BLOCKS)
return true;
if (!maybe_hot_bb_p (bb))
return true;
return false;
}
/* Return true if we should ignore the basic block for purposes of tracing. */
static bool
ignore_bb_p (basic_block bb)
{
if (bb->index < 0)
return true;
if (!maybe_hot_bb_p (bb))
return true;
return false;
}
static inline int
coalesce_cost_edge (edge e)
{
if (e->flags & EDGE_ABNORMAL)
return MUST_COALESCE_COST;
return coalesce_cost (EDGE_FREQUENCY (e),
maybe_hot_bb_p (e->src),
EDGE_CRITICAL_P (e));
}
static bool
rtl_value_profile_transformations (void)
{
rtx insn, next;
int changed = false;
for (insn = get_insns (); insn; insn = next)
{
next = NEXT_INSN (insn);
if (!INSN_P (insn))
continue;
/* Scan for insn carrying a histogram. */
if (!find_reg_note (insn, REG_VALUE_PROFILE, 0))
continue;
/* Ignore cold areas -- we are growing a code. */
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
continue;
if (dump_file)
{
fprintf (dump_file, "Trying transformations on insn %d\n",
INSN_UID (insn));
print_rtl_single (dump_file, insn);
}
/* Transformations: */
if (flag_value_profile_transformations
&& (mod_subtract_transform (insn)
|| divmod_fixed_value_transform (insn)
|| mod_pow2_value_transform (insn)))
changed = true;
#ifdef HAVE_prefetch
if (flag_speculative_prefetching
&& speculative_prefetching_transform (insn))
changed = true;
#endif
}
if (changed)
{
commit_edge_insertions ();
allocate_reg_info (max_reg_num (), FALSE, FALSE);
}
return changed;
}
static bool
copy_bb_p (basic_block bb, int code_may_grow)
{
int size = 0;
int max_size = uncond_jump_length;
rtx insn;
int n_succ;
edge e;
if (!bb->frequency)
return false;
if (!bb->pred || !bb->pred->pred_next)
return false;
if (!cfg_layout_can_duplicate_bb_p (bb))
return false;
/* Avoid duplicating blocks which have many successors (PR/13430). */
n_succ = 0;
for (e = bb->succ; e; e = e->succ_next)
{
n_succ++;
if (n_succ > 8)
return false;
}
if (code_may_grow && maybe_hot_bb_p (bb))
max_size *= 8;
for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
insn = NEXT_INSN (insn))
{
if (INSN_P (insn))
size += get_attr_length (insn);
}
if (size <= max_size)
return true;
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Block %d can't be copied because its size = %d.\n",
bb->index, size);
}
return false;
}
static inline int
coalesce_cost_bb (basic_block bb)
{
return coalesce_cost (bb->frequency, maybe_hot_bb_p (bb), false);
}
/* Unswitch single LOOP. COND_CHECKED holds list of conditions we already
unswitched on and are therefore known to be true in this LOOP. NUM is
number of unswitchings done; do not allow it to grow too much, it is too
easy to create example on that the code would grow exponentially. */
static void
unswitch_single_loop (struct loops *loops, struct loop *loop,
rtx cond_checked, int num)
{
basic_block *bbs;
struct loop *nloop;
unsigned i;
rtx cond, rcond = NULL_RTX, conds, rconds, acond, cinsn;
int repeat;
edge e;
/* Do not unswitch too much. */
if (num > PARAM_VALUE (PARAM_MAX_UNSWITCH_LEVEL))
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching anymore, hit max level\n");
return;
}
/* Only unswitch innermost loops. */
if (loop->inner)
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching, not innermost loop\n");
return;
}
/* We must be able to duplicate loop body. */
if (!can_duplicate_loop_p (loop))
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching, can't duplicate loop\n");
return;
}
/* The loop should not be too large, to limit code growth. */
if (num_loop_insns (loop) > PARAM_VALUE (PARAM_MAX_UNSWITCH_INSNS))
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching, loop too big\n");
return;
}
/* Do not unswitch in cold areas. */
if (!maybe_hot_bb_p (loop->header))
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching, not hot area\n");
return;
}
/* Nor if the loop usually does not roll. */
if (expected_loop_iterations (loop) < 1)
{
if (dump_file)
fprintf (dump_file, ";; Not unswitching, loop iterations < 1\n");
return;
}
do
{
repeat = 0;
cinsn = NULL_RTX;
/* Find a bb to unswitch on. */
bbs = get_loop_body (loop);
iv_analysis_loop_init (loop);
for (i = 0; i < loop->num_nodes; i++)
if ((cond = may_unswitch_on (bbs[i], loop, &cinsn)))
break;
if (i == loop->num_nodes)
{
free (bbs);
return;
}
if (cond != const0_rtx
&& cond != const_true_rtx)
{
rcond = reversed_condition (cond);
if (rcond)
rcond = canon_condition (rcond);
/* Check whether the result can be predicted. */
for (acond = cond_checked; acond; acond = XEXP (acond, 1))
simplify_using_condition (XEXP (acond, 0), &cond, NULL);
}
if (cond == const_true_rtx)
{
/* Remove false path. */
e = FALLTHRU_EDGE (bbs[i]);
remove_path (loops, e);
free (bbs);
repeat = 1;
}
else if (cond == const0_rtx)
{
/* Remove true path. */
e = BRANCH_EDGE (bbs[i]);
remove_path (loops, e);
free (bbs);
repeat = 1;
}
} while (repeat);
/* We found the condition we can unswitch on. */
conds = alloc_EXPR_LIST (0, cond, cond_checked);
if (rcond)
rconds = alloc_EXPR_LIST (0, rcond, cond_checked);
else
rconds = cond_checked;
if (dump_file)
fprintf (dump_file, ";; Unswitching loop\n");
/* Unswitch the loop on this condition. */
nloop = unswitch_loop (loops, loop, bbs[i], cond, cinsn);
gcc_assert (nloop);
/* Invoke itself on modified loops. */
unswitch_single_loop (loops, nloop, rconds, num + 1);
unswitch_single_loop (loops, loop, conds, num + 1);
free_EXPR_LIST_node (conds);
if (rcond)
free_EXPR_LIST_node (rconds);
free (bbs);
}
static bool
speculative_prefetching_transform (rtx insn)
{
rtx histogram, value;
gcov_type val, count, all;
edge e;
rtx mem, address;
int write;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
return false;
if (!find_mem_reference (insn, &mem, &write))
return false;
address = XEXP (mem, 0);
if (side_effects_p (address))
return false;
if (CONSTANT_P (address))
return false;
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_CONST_DELTA))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
/* Skip last value referenced. */
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
/* With that few executions we do not really have a reason to optimize the
statement, and more importantly, the data about differences of addresses
are spoiled by the first item that had no previous value to compare
with. */
if (all < 4)
return false;
/* We require that count be at least half of all; this means
that for the transformation to fire the value must be constant
at least 50% of time (and 75% gives the guarantee of usage). */
if (!rtx_equal_p (address, value) || 2 * count < all)
return false;
/* If the difference is too small, it does not make too much sense to
prefetch, as the memory is probably already in cache. */
if (val >= NOPREFETCH_RANGE_MIN && val <= NOPREFETCH_RANGE_MAX)
return false;
if (dump_file)
fprintf (dump_file, "Speculative prefetching for insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
insert_insn_on_edge (gen_speculative_prefetch (address, val, write), e);
return true;
}
static bool
loop_prefetch_arrays (struct loop *loop)
{
struct mem_ref_group *refs;
unsigned ahead, ninsns, time, unroll_factor;
HOST_WIDE_INT est_niter;
struct tree_niter_desc desc;
bool unrolled = false, no_other_refs;
if (!maybe_hot_bb_p (loop->header))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " ignored (cold area)\n");
return false;
}
/* Step 1: gather the memory references. */
refs = gather_memory_references (loop, &no_other_refs);
/* Step 2: estimate the reuse effects. */
prune_by_reuse (refs);
if (!anything_to_prefetch_p (refs))
goto fail;
determine_loop_nest_reuse (loop, refs, no_other_refs);
/* Step 3: determine the ahead and unroll factor. */
/* FIXME: the time should be weighted by the probabilities of the blocks in
the loop body. */
time = tree_num_loop_insns (loop, &eni_time_weights);
ahead = (PREFETCH_LATENCY + time - 1) / time;
est_niter = estimated_loop_iterations_int (loop, false);
/* The prefetches will run for AHEAD iterations of the original loop. Unless
the loop rolls at least AHEAD times, prefetching the references does not
make sense. */
if (est_niter >= 0 && est_niter <= (HOST_WIDE_INT) ahead)
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Not prefetching -- loop estimated to roll only %d times\n",
(int) est_niter);
goto fail;
}
mark_nontemporal_stores (loop, refs);
ninsns = tree_num_loop_insns (loop, &eni_size_weights);
unroll_factor = determine_unroll_factor (loop, refs, ninsns, &desc,
est_niter);
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Ahead %d, unroll factor %d\n", ahead, unroll_factor);
/* Step 4: what to prefetch? */
if (!schedule_prefetches (refs, unroll_factor, ahead))
goto fail;
/* Step 5: unroll the loop. TODO -- peeling of first and last few
iterations so that we do not issue superfluous prefetches. */
if (unroll_factor != 1)
{
tree_unroll_loop (loop, unroll_factor,
single_dom_exit (loop), &desc);
unrolled = true;
}
/* Step 6: issue the prefetches. */
issue_prefetches (refs, unroll_factor, ahead);
fail:
release_mem_refs (refs);
return unrolled;
}