/* Checks whether all N blocks in BBS array can be copied. */
bool
can_copy_bbs_p (basic_block *bbs, unsigned n)
{
unsigned i;
edge e;
int ret = true;
for (i = 0; i < n; i++)
bbs[i]->rbi->duplicated = 1;
for (i = 0; i < n; i++)
{
/* In case we should redirect abnormal edge during duplication, fail. */
for (e = bbs[i]->succ; e; e = e->succ_next)
if ((e->flags & EDGE_ABNORMAL)
&& e->dest->rbi->duplicated)
{
ret = false;
goto end;
}
if (!cfg_layout_can_duplicate_bb_p (bbs[i]))
{
ret = false;
break;
}
}
end:
for (i = 0; i < n; i++)
bbs[i]->rbi->duplicated = 0;
return ret;
}
static bool
copy_bb_p (basic_block bb, int code_may_grow)
{
int size = 0;
int max_size = uncond_jump_length;
rtx insn;
int n_succ;
edge e;
if (!bb->frequency)
return false;
if (!bb->pred || !bb->pred->pred_next)
return false;
if (!cfg_layout_can_duplicate_bb_p (bb))
return false;
/* Avoid duplicating blocks which have many successors (PR/13430). */
n_succ = 0;
for (e = bb->succ; e; e = e->succ_next)
{
n_succ++;
if (n_succ > 8)
return false;
}
if (code_may_grow && maybe_hot_bb_p (bb))
max_size *= 8;
for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
insn = NEXT_INSN (insn))
{
if (INSN_P (insn))
size += get_attr_length (insn);
}
if (size <= max_size)
return true;
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Block %d can't be copied because its size = %d.\n",
bb->index, size);
}
return false;
}
basic_block
cfg_layout_duplicate_bb (basic_block bb, edge e)
{
rtx insn;
edge s, n;
basic_block new_bb;
gcov_type new_count = e ? e->count : 0;
if (bb->count < new_count)
new_count = bb->count;
if (!bb->pred)
abort ();
#ifdef ENABLE_CHECKING
if (!cfg_layout_can_duplicate_bb_p (bb))
abort ();
#endif
insn = duplicate_insn_chain (BB_HEAD (bb), BB_END (bb));
new_bb = create_basic_block (insn,
insn ? get_last_insn () : NULL,
EXIT_BLOCK_PTR->prev_bb);
if (bb->rbi->header)
{
insn = bb->rbi->header;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
insn = duplicate_insn_chain (bb->rbi->header, insn);
if (insn)
new_bb->rbi->header = unlink_insn_chain (insn, get_last_insn ());
}
if (bb->rbi->footer)
{
insn = bb->rbi->footer;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
insn = duplicate_insn_chain (bb->rbi->footer, insn);
if (insn)
new_bb->rbi->footer = unlink_insn_chain (insn, get_last_insn ());
}
if (bb->global_live_at_start)
{
new_bb->global_live_at_start = OBSTACK_ALLOC_REG_SET (&flow_obstack);
new_bb->global_live_at_end = OBSTACK_ALLOC_REG_SET (&flow_obstack);
COPY_REG_SET (new_bb->global_live_at_start, bb->global_live_at_start);
COPY_REG_SET (new_bb->global_live_at_end, bb->global_live_at_end);
}
new_bb->loop_depth = bb->loop_depth;
new_bb->flags = bb->flags;
for (s = bb->succ; s; s = s->succ_next)
{
/* Since we are creating edges from a new block to successors
of another block (which therefore are known to be disjoint), there
is no need to actually check for duplicated edges. */
n = unchecked_make_edge (new_bb, s->dest, s->flags);
n->probability = s->probability;
if (e && bb->count)
{
/* Take care for overflows! */
n->count = s->count * (new_count * 10000 / bb->count) / 10000;
s->count -= n->count;
}
else
n->count = s->count;
n->aux = s->aux;
}
if (e)
{
new_bb->count = new_count;
bb->count -= new_count;
new_bb->frequency = EDGE_FREQUENCY (e);
bb->frequency -= EDGE_FREQUENCY (e);
redirect_edge_and_branch_force (e, new_bb);
if (bb->count < 0)
bb->count = 0;
if (bb->frequency < 0)
bb->frequency = 0;
}
else
{
new_bb->count = bb->count;
new_bb->frequency = bb->frequency;
}
new_bb->rbi->original = bb;
bb->rbi->copy = new_bb;
return new_bb;
}
static void
tail_duplicate (void)
{
fibnode_t *blocks = xcalloc (last_basic_block, sizeof (fibnode_t));
basic_block *trace = xmalloc (sizeof (basic_block) * n_basic_blocks);
int *counts = xmalloc (sizeof (int) * last_basic_block);
int ninsns = 0, nduplicated = 0;
gcov_type weighted_insns = 0, traced_insns = 0;
fibheap_t heap = fibheap_new ();
gcov_type cover_insns;
int max_dup_insns;
basic_block bb;
if (profile_info && flag_branch_probabilities)
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK);
else
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY);
probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff;
branch_ratio_cutoff =
(REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO));
FOR_EACH_BB (bb)
{
int n = count_insns (bb);
if (!ignore_bb_p (bb))
blocks[bb->index] = fibheap_insert (heap, -bb->frequency,
bb);
counts [bb->index] = n;
ninsns += n;
weighted_insns += n * bb->frequency;
}
if (profile_info && flag_branch_probabilities)
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK);
else
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE);
cover_insns = (weighted_insns * cover_insns + 50) / 100;
max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100;
while (traced_insns < cover_insns && nduplicated < max_dup_insns
&& !fibheap_empty (heap))
{
basic_block bb = fibheap_extract_min (heap);
int n, pos;
if (!bb)
break;
blocks[bb->index] = NULL;
if (ignore_bb_p (bb))
continue;
if (seen (bb))
abort ();
n = find_trace (bb, trace);
bb = trace[0];
traced_insns += bb->frequency * counts [bb->index];
if (blocks[bb->index])
{
fibheap_delete_node (heap, blocks[bb->index]);
blocks[bb->index] = NULL;
}
for (pos = 1; pos < n; pos++)
{
basic_block bb2 = trace[pos];
if (blocks[bb2->index])
{
fibheap_delete_node (heap, blocks[bb2->index]);
blocks[bb2->index] = NULL;
}
traced_insns += bb2->frequency * counts [bb2->index];
if (bb2->pred && bb2->pred->pred_next
&& cfg_layout_can_duplicate_bb_p (bb2))
{
edge e = bb2->pred;
basic_block old = bb2;
while (e->src != bb)
e = e->pred_next;
nduplicated += counts [bb2->index];
bb2 = cfg_layout_duplicate_bb (bb2, e);
/* Reconsider the original copy of block we've duplicated.
Removing the most common predecessor may make it to be
head. */
blocks[old->index] =
fibheap_insert (heap, -old->frequency, old);
if (rtl_dump_file)
fprintf (rtl_dump_file, "Duplicated %i as %i [%i]\n",
old->index, bb2->index, bb2->frequency);
}
bb->rbi->next = bb2;
bb2->rbi->visited = 1;
bb = bb2;
/* In case the trace became infrequent, stop duplicating. */
if (ignore_bb_p (bb))
break;
}
if (rtl_dump_file)
fprintf (rtl_dump_file, " covered now %.1f\n\n",
traced_insns * 100.0 / weighted_insns);
}
if (rtl_dump_file)
fprintf (rtl_dump_file, "Duplicated %i insns (%i%%)\n", nduplicated,
nduplicated * 100 / ninsns);
free (blocks);
free (trace);
free (counts);
fibheap_delete (heap);
}
