static fibheapkey_t
bb_to_key (basic_block bb)
{
edge e;
int priority = 0;
/* Do not start in probably never executed blocks. */
if (probably_never_executed_bb_p (bb))
return BB_FREQ_MAX;
/* Prefer blocks whose predecessor is an end of some trace
or whose predecessor edge is EDGE_DFS_BACK. */
for (e = bb->pred; e; e = e->pred_next)
{
if ((e->src != ENTRY_BLOCK_PTR && bbd[e->src->index].end_of_trace >= 0)
|| (e->flags & EDGE_DFS_BACK))
{
int edge_freq = EDGE_FREQUENCY (e);
if (edge_freq > priority)
priority = edge_freq;
}
}
if (priority)
/* The block with priority should have significantly lower key. */
return -(100 * BB_FREQ_MAX + 100 * priority + bb->frequency);
return -bb->frequency;
}
static inline int
coalesce_cost_edge (edge e)
{
if (e->flags & EDGE_ABNORMAL)
return MUST_COALESCE_COST;
return coalesce_cost (EDGE_FREQUENCY (e),
maybe_hot_bb_p (e->src),
EDGE_CRITICAL_P (e));
}
/* Check the consistency of profile information. We can't do that
in verify_flow_info, as the counts may get invalid for incompletely
solved graphs, later eliminating of conditionals or roundoff errors.
It is still practical to have them reported for debugging of simple
testcases. */
static void
check_bb_profile (basic_block bb, FILE * file, int indent, int flags)
{
edge e;
int sum = 0;
gcov_type lsum;
edge_iterator ei;
struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl);
char *s_indent = (char *) alloca ((size_t) indent + 1);
memset ((void *) s_indent, ' ', (size_t) indent);
s_indent[indent] = '\0';
if (profile_status_for_function (fun) == PROFILE_ABSENT)
return;
if (bb != EXIT_BLOCK_PTR_FOR_FUNCTION (fun))
{
FOR_EACH_EDGE (e, ei, bb->succs)
sum += e->probability;
if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100)
fprintf (file, "%s%sInvalid sum of outgoing probabilities %.1f%%\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
sum * 100.0 / REG_BR_PROB_BASE);
lsum = 0;
FOR_EACH_EDGE (e, ei, bb->succs)
lsum += e->count;
if (EDGE_COUNT (bb->succs)
&& (lsum - bb->count > 100 || lsum - bb->count < -100))
fprintf (file, "%s%sInvalid sum of outgoing counts %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
(int) lsum, (int) bb->count);
}
if (bb != ENTRY_BLOCK_PTR_FOR_FUNCTION (fun))
{
sum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
sum += EDGE_FREQUENCY (e);
if (abs (sum - bb->frequency) > 100)
fprintf (file,
"%s%sInvalid sum of incoming frequencies %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
sum, bb->frequency);
lsum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
lsum += e->count;
if (lsum - bb->count > 100 || lsum - bb->count < -100)
fprintf (file, "%s%sInvalid sum of incoming counts %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
(int) lsum, (int) bb->count);
}
}
static void
layout_superblocks (void)
{
basic_block end = single_succ (ENTRY_BLOCK_PTR);
basic_block bb = end->next_bb;
while (bb != EXIT_BLOCK_PTR)
{
edge_iterator ei;
edge e, best = NULL;
while (end->aux)
end = end->aux;
FOR_EACH_EDGE (e, ei, end->succs)
if (e->dest != EXIT_BLOCK_PTR
&& e->dest != single_succ (ENTRY_BLOCK_PTR)
&& !e->dest->il.rtl->visited
&& (!best || EDGE_FREQUENCY (e) > EDGE_FREQUENCY (best)))
best = e;
if (best)
{
end->aux = best->dest;
best->dest->il.rtl->visited = 1;
}
else
for (; bb != EXIT_BLOCK_PTR; bb = bb->next_bb)
{
if (!bb->il.rtl->visited)
{
end->aux = bb;
bb->il.rtl->visited = 1;
break;
}
}
}
}
static void
layout_superblocks (void)
{
basic_block end = ENTRY_BLOCK_PTR->succ->dest;
basic_block bb = ENTRY_BLOCK_PTR->succ->dest->next_bb;
while (bb != EXIT_BLOCK_PTR)
{
edge e, best = NULL;
while (end->rbi->next)
end = end->rbi->next;
for (e = end->succ; e; e = e->succ_next)
if (e->dest != EXIT_BLOCK_PTR
&& e->dest != ENTRY_BLOCK_PTR->succ->dest
&& !e->dest->rbi->visited
&& (!best || EDGE_FREQUENCY (e) > EDGE_FREQUENCY (best)))
best = e;
if (best)
{
end->rbi->next = best->dest;
best->dest->rbi->visited = 1;
}
else
for (; bb != EXIT_BLOCK_PTR; bb = bb->next_bb)
{
if (!bb->rbi->visited)
{
end->rbi->next = bb;
bb->rbi->visited = 1;
break;
}
}
}
}
static edge
find_best_predecessor (basic_block bb)
{
edge e;
edge best = NULL;
for (e = bb->pred; e; e = e->pred_next)
if (!best || better_p (e, best))
best = e;
if (!best || ignore_bb_p (best->src))
return NULL;
if (EDGE_FREQUENCY (best) * REG_BR_PROB_BASE
< bb->frequency * branch_ratio_cutoff)
return NULL;
return best;
}
static edge
find_best_predecessor (basic_block bb)
{
edge e;
edge best = NULL;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, bb->preds)
if (!best || better_p (e, best))
best = e;
if (!best || ignore_bb_p (best->src))
return NULL;
if (EDGE_FREQUENCY (best) * REG_BR_PROB_BASE
< bb->frequency * branch_ratio_cutoff)
return NULL;
return best;
}
static basic_block
make_forwarder_block (basic_block bb, int redirect_latch, int redirect_nonlatch, edge except, int conn_latch)
{
edge e, next_e, fallthru;
basic_block dummy;
rtx insn;
insn = PREV_INSN (first_insn_after_basic_block_note (bb));
/* For empty block split_block will return NULL. */
if (BB_END (bb) == insn)
emit_note_after (NOTE_INSN_DELETED, insn);
fallthru = split_block (bb, insn);
dummy = fallthru->src;
bb = fallthru->dest;
bb->aux = xmalloc (sizeof (int));
HEADER_BLOCK (dummy) = 0;
HEADER_BLOCK (bb) = 1;
/* Redirect back edges we want to keep. */
for (e = dummy->pred; e; e = next_e)
{
next_e = e->pred_next;
if (e == except
|| !((redirect_latch && LATCH_EDGE (e))
|| (redirect_nonlatch && !LATCH_EDGE (e))))
{
dummy->frequency -= EDGE_FREQUENCY (e);
dummy->count -= e->count;
if (dummy->frequency < 0)
dummy->frequency = 0;
if (dummy->count < 0)
dummy->count = 0;
redirect_edge_with_latch_update (e, bb);
}
}
alloc_aux_for_edge (fallthru, sizeof (int));
LATCH_EDGE (fallthru) = conn_latch;
return dummy;
}
static basic_block
rotate_loop (edge back_edge, struct trace *trace, int trace_n)
{
basic_block bb;
/* Information about the best end (end after rotation) of the loop. */
basic_block best_bb = NULL;
edge best_edge = NULL;
int best_freq = -1;
gcov_type best_count = -1;
/* The best edge is preferred when its destination is not visited yet
or is a start block of some trace. */
bool is_preferred = false;
/* Find the most frequent edge that goes out from current trace. */
bb = back_edge->dest;
do
{
edge e;
for (e = bb->succ; e; e = e->succ_next)
if (e->dest != EXIT_BLOCK_PTR
&& e->dest->rbi->visited != trace_n
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX))
{
if (is_preferred)
{
/* The best edge is preferred. */
if (!e->dest->rbi->visited
|| bbd[e->dest->index].start_of_trace >= 0)
{
/* The current edge E is also preferred. */
int freq = EDGE_FREQUENCY (e);
if (freq > best_freq || e->count > best_count)
{
best_freq = freq;
best_count = e->count;
best_edge = e;
best_bb = bb;
}
}
}
else
{
if (!e->dest->rbi->visited
|| bbd[e->dest->index].start_of_trace >= 0)
{
/* The current edge E is preferred. */
is_preferred = true;
best_freq = EDGE_FREQUENCY (e);
best_count = e->count;
best_edge = e;
best_bb = bb;
}
else
{
int freq = EDGE_FREQUENCY (e);
if (!best_edge || freq > best_freq || e->count > best_count)
{
best_freq = freq;
best_count = e->count;
best_edge = e;
best_bb = bb;
}
}
}
}
bb = bb->rbi->next;
}
while (bb != back_edge->dest);
if (best_bb)
{
/* Rotate the loop so that the BEST_EDGE goes out from the last block of
the trace. */
if (back_edge->dest == trace->first)
{
trace->first = best_bb->rbi->next;
}
else
{
basic_block prev_bb;
for (prev_bb = trace->first;
prev_bb->rbi->next != back_edge->dest;
prev_bb = prev_bb->rbi->next)
;
prev_bb->rbi->next = best_bb->rbi->next;
/* Try to get rid of uncond jump to cond jump. */
if (prev_bb->succ && !prev_bb->succ->succ_next)
{
basic_block header = prev_bb->succ->dest;
/* Duplicate HEADER if it is a small block containing cond jump
in the end. */
if (any_condjump_p (BB_END (header)) && copy_bb_p (header, 0))
{
copy_bb (header, prev_bb->succ, prev_bb, trace_n);
}
}
}
}
else
{
/* We have not found suitable loop tail so do no rotation. */
best_bb = back_edge->src;
}
best_bb->rbi->next = NULL;
return best_bb;
}
static void
eliminate_tail_call (struct tailcall *t)
{
tree param, rslt;
gimple stmt, call;
tree arg;
size_t idx;
basic_block bb, first;
edge e;
gimple phi;
gimple_stmt_iterator gsi;
gimple orig_stmt;
stmt = orig_stmt = gsi_stmt (t->call_gsi);
bb = gsi_bb (t->call_gsi);
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Eliminated tail recursion in bb %d : ",
bb->index);
print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
fprintf (dump_file, "\n");
}
gcc_assert (is_gimple_call (stmt));
first = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
/* Remove the code after call_gsi that will become unreachable. The
possibly unreachable code in other blocks is removed later in
cfg cleanup. */
gsi = t->call_gsi;
gsi_next (&gsi);
while (!gsi_end_p (gsi))
{
gimple t = gsi_stmt (gsi);
/* Do not remove the return statement, so that redirect_edge_and_branch
sees how the block ends. */
if (gimple_code (t) == GIMPLE_RETURN)
break;
gsi_remove (&gsi, true);
release_defs (t);
}
/* Number of executions of function has reduced by the tailcall. */
e = single_succ_edge (gsi_bb (t->call_gsi));
decrease_profile (EXIT_BLOCK_PTR_FOR_FN (cfun), e->count, EDGE_FREQUENCY (e));
decrease_profile (ENTRY_BLOCK_PTR_FOR_FN (cfun), e->count,
EDGE_FREQUENCY (e));
if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
decrease_profile (e->dest, e->count, EDGE_FREQUENCY (e));
/* Replace the call by a jump to the start of function. */
e = redirect_edge_and_branch (single_succ_edge (gsi_bb (t->call_gsi)),
first);
gcc_assert (e);
PENDING_STMT (e) = NULL;
/* Add phi node entries for arguments. The ordering of the phi nodes should
be the same as the ordering of the arguments. */
for (param = DECL_ARGUMENTS (current_function_decl),
idx = 0, gsi = gsi_start_phis (first);
param;
param = DECL_CHAIN (param), idx++)
{
if (!arg_needs_copy_p (param))
continue;
arg = gimple_call_arg (stmt, idx);
phi = gsi_stmt (gsi);
gcc_assert (param == SSA_NAME_VAR (PHI_RESULT (phi)));
add_phi_arg (phi, arg, e, gimple_location (stmt));
gsi_next (&gsi);
}
/* Update the values of accumulators. */
adjust_accumulator_values (t->call_gsi, t->mult, t->add, e);
call = gsi_stmt (t->call_gsi);
rslt = gimple_call_lhs (call);
if (rslt != NULL_TREE)
{
/* Result of the call will no longer be defined. So adjust the
SSA_NAME_DEF_STMT accordingly. */
SSA_NAME_DEF_STMT (rslt) = gimple_build_nop ();
}
gsi_remove (&t->call_gsi, true);
release_defs (call);
}
/* Takes care of merging natural loops with shared headers. */
static void
canonicalize_loop_headers (void)
{
basic_block header;
edge e;
/* Compute the dominators. */
calculate_dominance_info (CDI_DOMINATORS);
alloc_aux_for_blocks (sizeof (int));
alloc_aux_for_edges (sizeof (int));
/* Split blocks so that each loop has only single latch. */
FOR_EACH_BB (header)
{
int num_latches = 0;
int have_abnormal_edge = 0;
for (e = header->pred; e; e = e->pred_next)
{
basic_block latch = e->src;
if (e->flags & EDGE_ABNORMAL)
have_abnormal_edge = 1;
if (latch != ENTRY_BLOCK_PTR
&& dominated_by_p (CDI_DOMINATORS, latch, header))
{
num_latches++;
LATCH_EDGE (e) = 1;
}
}
if (have_abnormal_edge)
HEADER_BLOCK (header) = 0;
else
HEADER_BLOCK (header) = num_latches;
}
free_dominance_info (CDI_DOMINATORS);
if (HEADER_BLOCK (ENTRY_BLOCK_PTR->succ->dest))
{
basic_block bb;
/* We could not redirect edges freely here. On the other hand,
we can simply split the edge from entry block. */
bb = split_edge (ENTRY_BLOCK_PTR->succ);
alloc_aux_for_edge (bb->succ, sizeof (int));
LATCH_EDGE (bb->succ) = 0;
alloc_aux_for_block (bb, sizeof (int));
HEADER_BLOCK (bb) = 0;
}
FOR_EACH_BB (header)
{
int num_latch;
int want_join_latch;
int max_freq, is_heavy;
edge heavy;
if (!HEADER_BLOCK (header))
continue;
num_latch = HEADER_BLOCK (header);
want_join_latch = (num_latch > 1);
if (!want_join_latch)
continue;
/* Find a heavy edge. */
is_heavy = 1;
heavy = NULL;
max_freq = 0;
for (e = header->pred; e; e = e->pred_next)
if (LATCH_EDGE (e) &&
EDGE_FREQUENCY (e) > max_freq)
max_freq = EDGE_FREQUENCY (e);
for (e = header->pred; e; e = e->pred_next)
if (LATCH_EDGE (e) &&
EDGE_FREQUENCY (e) >= max_freq / HEAVY_EDGE_RATIO)
{
if (heavy)
{
is_heavy = 0;
break;
}
else
heavy = e;
}
if (is_heavy)
{
basic_block new_header =
make_forwarder_block (header, true, true, heavy, 0);
if (num_latch > 2)
make_forwarder_block (new_header, true, false, NULL, 1);
}
else
make_forwarder_block (header, true, false, NULL, 1);
}
free_aux_for_blocks ();
free_aux_for_edges ();
}
static void
connect_traces (int n_traces, struct trace *traces)
{
int i;
bool *connected;
int last_trace;
int freq_threshold;
gcov_type count_threshold;
freq_threshold = max_entry_frequency * DUPLICATION_THRESHOLD / 1000;
if (max_entry_count < INT_MAX / 1000)
count_threshold = max_entry_count * DUPLICATION_THRESHOLD / 1000;
else
count_threshold = max_entry_count / 1000 * DUPLICATION_THRESHOLD;
connected = xcalloc (n_traces, sizeof (bool));
last_trace = -1;
for (i = 0; i < n_traces; i++)
{
int t = i;
int t2;
edge e, best;
int best_len;
if (connected[t])
continue;
connected[t] = true;
/* Find the predecessor traces. */
for (t2 = t; t2 > 0;)
{
best = NULL;
best_len = 0;
for (e = traces[t2].first->pred; e; e = e->pred_next)
{
int si = e->src->index;
if (e->src != ENTRY_BLOCK_PTR
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX)
&& bbd[si].end_of_trace >= 0
&& !connected[bbd[si].end_of_trace]
&& (!best
|| e->probability > best->probability
|| (e->probability == best->probability
&& traces[bbd[si].end_of_trace].length > best_len)))
{
best = e;
best_len = traces[bbd[si].end_of_trace].length;
}
}
if (best)
{
best->src->rbi->next = best->dest;
t2 = bbd[best->src->index].end_of_trace;
connected[t2] = true;
if (rtl_dump_file)
{
fprintf (rtl_dump_file, "Connection: %d %d\n",
best->src->index, best->dest->index);
}
}
else
break;
}
if (last_trace >= 0)
traces[last_trace].last->rbi->next = traces[t2].first;
last_trace = t;
/* Find the successor traces. */
while (1)
{
/* Find the continuation of the chain. */
best = NULL;
best_len = 0;
for (e = traces[t].last->succ; e; e = e->succ_next)
{
int di = e->dest->index;
if (e->dest != EXIT_BLOCK_PTR
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX)
&& bbd[di].start_of_trace >= 0
&& !connected[bbd[di].start_of_trace]
&& (!best
|| e->probability > best->probability
|| (e->probability == best->probability
&& traces[bbd[di].start_of_trace].length > best_len)))
{
best = e;
best_len = traces[bbd[di].start_of_trace].length;
}
}
if (best)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file, "Connection: %d %d\n",
best->src->index, best->dest->index);
}
t = bbd[best->dest->index].start_of_trace;
traces[last_trace].last->rbi->next = traces[t].first;
connected[t] = true;
last_trace = t;
}
else
{
/* Try to connect the traces by duplication of 1 block. */
edge e2;
basic_block next_bb = NULL;
bool try_copy = false;
for (e = traces[t].last->succ; e; e = e->succ_next)
if (e->dest != EXIT_BLOCK_PTR
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX)
&& (!best || e->probability > best->probability))
{
edge best2 = NULL;
int best2_len = 0;
/* If the destination is a start of a trace which is only
one block long, then no need to search the successor
blocks of the trace. Accept it. */
if (bbd[e->dest->index].start_of_trace >= 0
&& traces[bbd[e->dest->index].start_of_trace].length
== 1)
{
best = e;
try_copy = true;
continue;
}
for (e2 = e->dest->succ; e2; e2 = e2->succ_next)
{
int di = e2->dest->index;
if (e2->dest == EXIT_BLOCK_PTR
|| ((e2->flags & EDGE_CAN_FALLTHRU)
&& !(e2->flags & EDGE_COMPLEX)
&& bbd[di].start_of_trace >= 0
&& !connected[bbd[di].start_of_trace]
&& (EDGE_FREQUENCY (e2) >= freq_threshold)
&& (e2->count >= count_threshold)
&& (!best2
|| e2->probability > best2->probability
|| (e2->probability == best2->probability
&& traces[bbd[di].start_of_trace].length
> best2_len))))
{
best = e;
best2 = e2;
if (e2->dest != EXIT_BLOCK_PTR)
best2_len = traces[bbd[di].start_of_trace].length;
else
best2_len = INT_MAX;
next_bb = e2->dest;
try_copy = true;
}
}
}
/* Copy tiny blocks always; copy larger blocks only when the
edge is traversed frequently enough. */
if (try_copy
&& copy_bb_p (best->dest,
!optimize_size
&& EDGE_FREQUENCY (best) >= freq_threshold
&& best->count >= count_threshold))
{
basic_block new_bb;
if (rtl_dump_file)
{
fprintf (rtl_dump_file, "Connection: %d %d ",
traces[t].last->index, best->dest->index);
if (!next_bb)
fputc ('\n', rtl_dump_file);
else if (next_bb == EXIT_BLOCK_PTR)
fprintf (rtl_dump_file, "exit\n");
else
fprintf (rtl_dump_file, "%d\n", next_bb->index);
}
new_bb = copy_bb (best->dest, best, traces[t].last, t);
traces[t].last = new_bb;
if (next_bb && next_bb != EXIT_BLOCK_PTR)
{
t = bbd[next_bb->index].start_of_trace;
traces[last_trace].last->rbi->next = traces[t].first;
connected[t] = true;
last_trace = t;
}
else
break; /* Stop finding the successor traces. */
}
else
break; /* Stop finding the successor traces. */
}
}
}
if (rtl_dump_file)
{
basic_block bb;
fprintf (rtl_dump_file, "Final order:\n");
for (bb = traces[0].first; bb; bb = bb->rbi->next)
fprintf (rtl_dump_file, "%d ", bb->index);
fprintf (rtl_dump_file, "\n");
fflush (rtl_dump_file);
}
FREE (connected);
}
static void
find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
struct trace *traces, int *n_traces, int round,
fibheap_t *heap)
{
/* Heap for discarded basic blocks which are possible starting points for
the next round. */
fibheap_t new_heap = fibheap_new ();
while (!fibheap_empty (*heap))
{
basic_block bb;
struct trace *trace;
edge best_edge, e;
fibheapkey_t key;
bb = fibheap_extract_min (*heap);
bbd[bb->index].heap = NULL;
bbd[bb->index].node = NULL;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Getting bb %d\n", bb->index);
/* If the BB's frequency is too low send BB to the next round. */
if (round < N_ROUNDS - 1
&& (bb->frequency < exec_th || bb->count < count_th
|| probably_never_executed_bb_p (bb)))
{
int key = bb_to_key (bb);
bbd[bb->index].heap = new_heap;
bbd[bb->index].node = fibheap_insert (new_heap, key, bb);
if (rtl_dump_file)
fprintf (rtl_dump_file,
" Possible start point of next round: %d (key: %d)\n",
bb->index, key);
continue;
}
trace = traces + *n_traces;
trace->first = bb;
trace->round = round;
trace->length = 0;
(*n_traces)++;
do
{
int prob, freq;
/* The probability and frequency of the best edge. */
int best_prob = INT_MIN / 2;
int best_freq = INT_MIN / 2;
best_edge = NULL;
mark_bb_visited (bb, *n_traces);
trace->length++;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Basic block %d was visited in trace %d\n",
bb->index, *n_traces - 1);
/* Select the successor that will be placed after BB. */
for (e = bb->succ; e; e = e->succ_next)
{
#ifdef ENABLE_CHECKING
if (e->flags & EDGE_FAKE)
abort ();
#endif
if (e->dest == EXIT_BLOCK_PTR)
continue;
if (e->dest->rbi->visited
&& e->dest->rbi->visited != *n_traces)
continue;
prob = e->probability;
freq = EDGE_FREQUENCY (e);
/* Edge that cannot be fallthru or improbable or infrequent
successor (ie. it is unsuitable successor). */
if (!(e->flags & EDGE_CAN_FALLTHRU) || (e->flags & EDGE_COMPLEX)
|| prob < branch_th || freq < exec_th || e->count < count_th)
continue;
if (better_edge_p (bb, e, prob, freq, best_prob, best_freq))
{
best_edge = e;
best_prob = prob;
best_freq = freq;
}
}
/* If the best destination has multiple predecessors, and can be
duplicated cheaper than a jump, don't allow it to be added
to a trace. We'll duplicate it when connecting traces. */
if (best_edge && best_edge->dest->pred->pred_next
&& copy_bb_p (best_edge->dest, 0))
best_edge = NULL;
/* Add all non-selected successors to the heaps. */
for (e = bb->succ; e; e = e->succ_next)
{
if (e == best_edge
|| e->dest == EXIT_BLOCK_PTR
|| e->dest->rbi->visited)
continue;
key = bb_to_key (e->dest);
if (bbd[e->dest->index].heap)
{
/* E->DEST is already in some heap. */
if (key != bbd[e->dest->index].node->key)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Changing key for bb %d from %ld to %ld.\n",
e->dest->index,
(long) bbd[e->dest->index].node->key,
key);
}
fibheap_replace_key (bbd[e->dest->index].heap,
bbd[e->dest->index].node, key);
}
}
else
{
fibheap_t which_heap = *heap;
prob = e->probability;
freq = EDGE_FREQUENCY (e);
if (!(e->flags & EDGE_CAN_FALLTHRU)
|| (e->flags & EDGE_COMPLEX)
|| prob < branch_th || freq < exec_th
|| e->count < count_th)
{
if (round < N_ROUNDS - 1)
which_heap = new_heap;
}
bbd[e->dest->index].heap = which_heap;
bbd[e->dest->index].node = fibheap_insert (which_heap,
key, e->dest);
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
" Possible start of %s round: %d (key: %ld)\n",
(which_heap == new_heap) ? "next" : "this",
e->dest->index, (long) key);
}
}
}
if (best_edge) /* Suitable successor was found. */
{
if (best_edge->dest->rbi->visited == *n_traces)
{
/* We do nothing with one basic block loops. */
if (best_edge->dest != bb)
{
if (EDGE_FREQUENCY (best_edge)
> 4 * best_edge->dest->frequency / 5)
{
/* The loop has at least 4 iterations. If the loop
header is not the first block of the function
we can rotate the loop. */
if (best_edge->dest != ENTRY_BLOCK_PTR->next_bb)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Rotating loop %d - %d\n",
best_edge->dest->index, bb->index);
}
bb->rbi->next = best_edge->dest;
bb = rotate_loop (best_edge, trace, *n_traces);
}
}
else
{
/* The loop has less than 4 iterations. */
/* Check whether there is another edge from BB. */
edge another_edge;
for (another_edge = bb->succ;
another_edge;
another_edge = another_edge->succ_next)
if (another_edge != best_edge)
break;
if (!another_edge && copy_bb_p (best_edge->dest,
!optimize_size))
{
bb = copy_bb (best_edge->dest, best_edge, bb,
*n_traces);
}
}
}
/* Terminate the trace. */
break;
}
else
{
/* Check for a situation
A
/|
B |
\|
C
where
EDGE_FREQUENCY (AB) + EDGE_FREQUENCY (BC)
>= EDGE_FREQUENCY (AC).
(i.e. 2 * B->frequency >= EDGE_FREQUENCY (AC) )
Best ordering is then A B C.
This situation is created for example by:
if (A) B;
C;
*/
for (e = bb->succ; e; e = e->succ_next)
if (e != best_edge
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX)
&& !e->dest->rbi->visited
&& !e->dest->pred->pred_next
&& e->dest->succ
&& (e->dest->succ->flags & EDGE_CAN_FALLTHRU)
&& !(e->dest->succ->flags & EDGE_COMPLEX)
&& !e->dest->succ->succ_next
&& e->dest->succ->dest == best_edge->dest
&& 2 * e->dest->frequency >= EDGE_FREQUENCY (best_edge))
{
best_edge = e;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Selecting BB %d\n",
best_edge->dest->index);
break;
}
bb->rbi->next = best_edge->dest;
bb = best_edge->dest;
}
}
}
while (best_edge);
trace->last = bb;
bbd[trace->first->index].start_of_trace = *n_traces - 1;
bbd[trace->last->index].end_of_trace = *n_traces - 1;
/* The trace is terminated so we have to recount the keys in heap
(some block can have a lower key because now one of its predecessors
is an end of the trace). */
for (e = bb->succ; e; e = e->succ_next)
{
if (e->dest == EXIT_BLOCK_PTR
|| e->dest->rbi->visited)
continue;
if (bbd[e->dest->index].heap)
{
key = bb_to_key (e->dest);
if (key != bbd[e->dest->index].node->key)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Changing key for bb %d from %ld to %ld.\n",
e->dest->index,
(long) bbd[e->dest->index].node->key, key);
}
fibheap_replace_key (bbd[e->dest->index].heap,
bbd[e->dest->index].node,
key);
}
}
}
}
fibheap_delete (*heap);
/* "Return" the new heap. */
*heap = new_heap;
}
static basic_block
expand_gimple_tailcall (basic_block bb, tree stmt, bool *can_fallthru)
{
rtx last2, last;
edge e;
edge_iterator ei;
int probability;
gcov_type count;
last2 = last = get_last_insn ();
expand_expr_stmt (stmt);
for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
if (CALL_P (last) && SIBLING_CALL_P (last))
goto found;
maybe_dump_rtl_for_tree_stmt (stmt, last2);
*can_fallthru = true;
return NULL;
found:
/* ??? Wouldn't it be better to just reset any pending stack adjust?
Any instructions emitted here are about to be deleted. */
do_pending_stack_adjust ();
/* Remove any non-eh, non-abnormal edges that don't go to exit. */
/* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
EH or abnormal edges, we shouldn't have created a tail call in
the first place. So it seems to me we should just be removing
all edges here, or redirecting the existing fallthru edge to
the exit block. */
probability = 0;
count = 0;
for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
{
if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
{
if (e->dest != EXIT_BLOCK_PTR)
{
e->dest->count -= e->count;
e->dest->frequency -= EDGE_FREQUENCY (e);
if (e->dest->count < 0)
e->dest->count = 0;
if (e->dest->frequency < 0)
e->dest->frequency = 0;
}
count += e->count;
probability += e->probability;
remove_edge (e);
}
else
ei_next (&ei);
}
/* This is somewhat ugly: the call_expr expander often emits instructions
after the sibcall (to perform the function return). These confuse the
find_sub_basic_blocks code, so we need to get rid of these. */
last = NEXT_INSN (last);
gcc_assert (BARRIER_P (last));
*can_fallthru = false;
while (NEXT_INSN (last))
{
/* For instance an sqrt builtin expander expands if with
sibcall in the then and label for `else`. */
if (LABEL_P (NEXT_INSN (last)))
{
*can_fallthru = true;
break;
}
delete_insn (NEXT_INSN (last));
}
e = make_edge (bb, EXIT_BLOCK_PTR, EDGE_ABNORMAL | EDGE_SIBCALL);
e->probability += probability;
e->count += count;
BB_END (bb) = last;
update_bb_for_insn (bb);
if (NEXT_INSN (last))
{
bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
last = BB_END (bb);
if (BARRIER_P (last))
BB_END (bb) = PREV_INSN (last);
}
maybe_dump_rtl_for_tree_stmt (stmt, last2);
return bb;
}
static basic_block
expand_gimple_cond_expr (basic_block bb, tree stmt)
{
basic_block new_bb, dest;
edge new_edge;
edge true_edge;
edge false_edge;
tree pred = COND_EXPR_COND (stmt);
tree then_exp = COND_EXPR_THEN (stmt);
tree else_exp = COND_EXPR_ELSE (stmt);
rtx last2, last;
last2 = last = get_last_insn ();
extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
if (EXPR_LOCUS (stmt))
{
emit_line_note (*(EXPR_LOCUS (stmt)));
record_block_change (TREE_BLOCK (stmt));
}
/* These flags have no purpose in RTL land. */
true_edge->flags &= ~EDGE_TRUE_VALUE;
false_edge->flags &= ~EDGE_FALSE_VALUE;
/* We can either have a pure conditional jump with one fallthru edge or
two-way jump that needs to be decomposed into two basic blocks. */
if (TREE_CODE (then_exp) == GOTO_EXPR && IS_EMPTY_STMT (else_exp))
{
jumpif (pred, label_rtx (GOTO_DESTINATION (then_exp)));
add_reg_br_prob_note (dump_file, last, true_edge->probability);
maybe_dump_rtl_for_tree_stmt (stmt, last);
if (EXPR_LOCUS (then_exp))
emit_line_note (*(EXPR_LOCUS (then_exp)));
return NULL;
}
if (TREE_CODE (else_exp) == GOTO_EXPR && IS_EMPTY_STMT (then_exp))
{
jumpifnot (pred, label_rtx (GOTO_DESTINATION (else_exp)));
add_reg_br_prob_note (dump_file, last, false_edge->probability);
maybe_dump_rtl_for_tree_stmt (stmt, last);
if (EXPR_LOCUS (else_exp))
emit_line_note (*(EXPR_LOCUS (else_exp)));
return NULL;
}
gcc_assert (TREE_CODE (then_exp) == GOTO_EXPR
&& TREE_CODE (else_exp) == GOTO_EXPR);
jumpif (pred, label_rtx (GOTO_DESTINATION (then_exp)));
add_reg_br_prob_note (dump_file, last, true_edge->probability);
last = get_last_insn ();
expand_expr (else_exp, const0_rtx, VOIDmode, 0);
BB_END (bb) = last;
if (BARRIER_P (BB_END (bb)))
BB_END (bb) = PREV_INSN (BB_END (bb));
update_bb_for_insn (bb);
new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
dest = false_edge->dest;
redirect_edge_succ (false_edge, new_bb);
false_edge->flags |= EDGE_FALLTHRU;
new_bb->count = false_edge->count;
new_bb->frequency = EDGE_FREQUENCY (false_edge);
new_edge = make_edge (new_bb, dest, 0);
new_edge->probability = REG_BR_PROB_BASE;
new_edge->count = new_bb->count;
if (BARRIER_P (BB_END (new_bb)))
BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
update_bb_for_insn (new_bb);
maybe_dump_rtl_for_tree_stmt (stmt, last2);
if (EXPR_LOCUS (else_exp))
emit_line_note (*(EXPR_LOCUS (else_exp)));
return new_bb;
}
static void
construct_exit_block (void)
{
rtx head = get_last_insn ();
rtx end;
basic_block exit_block;
edge e, e2;
unsigned ix;
edge_iterator ei;
/* Make sure the locus is set to the end of the function, so that
epilogue line numbers and warnings are set properly. */
#ifdef USE_MAPPED_LOCATION
if (cfun->function_end_locus != UNKNOWN_LOCATION)
#else
if (cfun->function_end_locus.file)
#endif
input_location = cfun->function_end_locus;
/* The following insns belong to the top scope. */
record_block_change (DECL_INITIAL (current_function_decl));
/* Generate rtl for function exit. */
expand_function_end ();
end = get_last_insn ();
if (head == end)
return;
while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
head = NEXT_INSN (head);
exit_block = create_basic_block (NEXT_INSN (head), end,
EXIT_BLOCK_PTR->prev_bb);
exit_block->frequency = EXIT_BLOCK_PTR->frequency;
exit_block->count = EXIT_BLOCK_PTR->count;
ix = 0;
while (ix < EDGE_COUNT (EXIT_BLOCK_PTR->preds))
{
e = EDGE_I (EXIT_BLOCK_PTR->preds, ix);
if (!(e->flags & EDGE_ABNORMAL))
redirect_edge_succ (e, exit_block);
else
ix++;
}
e = make_edge (exit_block, EXIT_BLOCK_PTR, EDGE_FALLTHRU);
e->probability = REG_BR_PROB_BASE;
e->count = EXIT_BLOCK_PTR->count;
FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR->preds)
if (e2 != e)
{
e->count -= e2->count;
exit_block->count -= e2->count;
exit_block->frequency -= EDGE_FREQUENCY (e2);
}
if (e->count < 0)
e->count = 0;
if (exit_block->count < 0)
exit_block->count = 0;
if (exit_block->frequency < 0)
exit_block->frequency = 0;
update_bb_for_insn (exit_block);
}
basic_block
cfg_layout_duplicate_bb (basic_block bb, edge e)
{
rtx insn;
edge s, n;
basic_block new_bb;
gcov_type new_count = e ? e->count : 0;
if (bb->count < new_count)
new_count = bb->count;
if (!bb->pred)
abort ();
#ifdef ENABLE_CHECKING
if (!cfg_layout_can_duplicate_bb_p (bb))
abort ();
#endif
insn = duplicate_insn_chain (BB_HEAD (bb), BB_END (bb));
new_bb = create_basic_block (insn,
insn ? get_last_insn () : NULL,
EXIT_BLOCK_PTR->prev_bb);
if (bb->rbi->header)
{
insn = bb->rbi->header;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
insn = duplicate_insn_chain (bb->rbi->header, insn);
if (insn)
new_bb->rbi->header = unlink_insn_chain (insn, get_last_insn ());
}
if (bb->rbi->footer)
{
insn = bb->rbi->footer;
while (NEXT_INSN (insn))
insn = NEXT_INSN (insn);
insn = duplicate_insn_chain (bb->rbi->footer, insn);
if (insn)
new_bb->rbi->footer = unlink_insn_chain (insn, get_last_insn ());
}
if (bb->global_live_at_start)
{
new_bb->global_live_at_start = OBSTACK_ALLOC_REG_SET (&flow_obstack);
new_bb->global_live_at_end = OBSTACK_ALLOC_REG_SET (&flow_obstack);
COPY_REG_SET (new_bb->global_live_at_start, bb->global_live_at_start);
COPY_REG_SET (new_bb->global_live_at_end, bb->global_live_at_end);
}
new_bb->loop_depth = bb->loop_depth;
new_bb->flags = bb->flags;
for (s = bb->succ; s; s = s->succ_next)
{
/* Since we are creating edges from a new block to successors
of another block (which therefore are known to be disjoint), there
is no need to actually check for duplicated edges. */
n = unchecked_make_edge (new_bb, s->dest, s->flags);
n->probability = s->probability;
if (e && bb->count)
{
/* Take care for overflows! */
n->count = s->count * (new_count * 10000 / bb->count) / 10000;
s->count -= n->count;
}
else
n->count = s->count;
n->aux = s->aux;
}
if (e)
{
new_bb->count = new_count;
bb->count -= new_count;
new_bb->frequency = EDGE_FREQUENCY (e);
bb->frequency -= EDGE_FREQUENCY (e);
redirect_edge_and_branch_force (e, new_bb);
if (bb->count < 0)
bb->count = 0;
if (bb->frequency < 0)
bb->frequency = 0;
}
else
{
new_bb->count = bb->count;
new_bb->frequency = bb->frequency;
}
new_bb->rbi->original = bb;
bb->rbi->copy = new_bb;
return new_bb;
}
