/* Propagate the constant parameters found by ipcp_iterate_stage()
to the function's code. */
static void
ipcp_insert_stage (void)
{
struct cgraph_node *node, *node1 = NULL;
int i;
VEC (cgraph_edge_p, heap) * redirect_callers;
VEC (ipa_replace_map_p,gc)* replace_trees;
int node_callers, count;
tree parm_tree;
struct ipa_replace_map *replace_param;
fibheap_t heap;
long overall_size = 0, new_size = 0;
long max_new_size;
ipa_check_create_node_params ();
ipa_check_create_edge_args ();
if (dump_file)
fprintf (dump_file, "\nIPA insert stage:\n\n");
dead_nodes = BITMAP_ALLOC (NULL);
for (node = cgraph_nodes; node; node = node->next)
if (node->analyzed)
{
if (node->count > max_count)
max_count = node->count;
overall_size += node->local.inline_summary.self_size;
}
max_new_size = overall_size;
if (max_new_size < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS))
max_new_size = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS);
max_new_size = max_new_size * PARAM_VALUE (PARAM_IPCP_UNIT_GROWTH) / 100 + 1;
/* First collect all functions we proved to have constant arguments to
heap. */
heap = fibheap_new ();
for (node = cgraph_nodes; node; node = node->next)
{
struct ipa_node_params *info;
/* Propagation of the constant is forbidden in certain conditions. */
if (!node->analyzed || !ipcp_node_modifiable_p (node))
continue;
info = IPA_NODE_REF (node);
if (ipa_is_called_with_var_arguments (info))
continue;
if (ipcp_const_param_count (node))
node->aux = fibheap_insert (heap, ipcp_estimate_cloning_cost (node),
node);
}
/* Now clone in priority order until code size growth limits are met or
heap is emptied. */
while (!fibheap_empty (heap))
{
struct ipa_node_params *info;
int growth = 0;
bitmap args_to_skip;
struct cgraph_edge *cs;
node = (struct cgraph_node *)fibheap_extract_min (heap);
node->aux = NULL;
if (dump_file)
fprintf (dump_file, "considering function %s\n",
cgraph_node_name (node));
growth = ipcp_estimate_growth (node);
if (new_size + growth > max_new_size)
break;
if (growth
&& optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl)))
{
if (dump_file)
fprintf (dump_file, "Not versioning, cold code would grow");
continue;
}
info = IPA_NODE_REF (node);
count = ipa_get_param_count (info);
replace_trees = VEC_alloc (ipa_replace_map_p, gc, 1);
if (node->local.can_change_signature)
args_to_skip = BITMAP_GGC_ALLOC ();
else
args_to_skip = NULL;
for (i = 0; i < count; i++)
{
struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
parm_tree = ipa_get_param (info, i);
/* We can proactively remove obviously unused arguments. */
if (!ipa_is_param_used (info, i))
{
if (args_to_skip)
bitmap_set_bit (args_to_skip, i);
continue;
}
if (lat->type == IPA_CONST_VALUE)
{
replace_param =
ipcp_create_replace_map (parm_tree, lat);
if (replace_param == NULL)
break;
VEC_safe_push (ipa_replace_map_p, gc, replace_trees, replace_param);
if (args_to_skip)
bitmap_set_bit (args_to_skip, i);
}
}
if (i < count)
{
if (dump_file)
fprintf (dump_file, "Not versioning, some parameters couldn't be replaced");
continue;
}
new_size += growth;
/* Look if original function becomes dead after cloning. */
for (cs = node->callers; cs != NULL; cs = cs->next_caller)
if (cs->caller == node || ipcp_need_redirect_p (cs))
break;
if (!cs && cgraph_will_be_removed_from_program_if_no_direct_calls (node))
bitmap_set_bit (dead_nodes, node->uid);
/* Compute how many callers node has. */
node_callers = 0;
for (cs = node->callers; cs != NULL; cs = cs->next_caller)
node_callers++;
redirect_callers = VEC_alloc (cgraph_edge_p, heap, node_callers);
for (cs = node->callers; cs != NULL; cs = cs->next_caller)
if (!cs->indirect_inlining_edge)
VEC_quick_push (cgraph_edge_p, redirect_callers, cs);
/* Redirecting all the callers of the node to the
new versioned node. */
node1 =
cgraph_create_virtual_clone (node, redirect_callers, replace_trees,
args_to_skip, "constprop");
args_to_skip = NULL;
VEC_free (cgraph_edge_p, heap, redirect_callers);
replace_trees = NULL;
if (node1 == NULL)
continue;
ipcp_process_devirtualization_opportunities (node1);
if (dump_file)
fprintf (dump_file, "versioned function %s with growth %i, overall %i\n",
cgraph_node_name (node), (int)growth, (int)new_size);
ipcp_init_cloned_node (node, node1);
info = IPA_NODE_REF (node);
for (i = 0; i < count; i++)
{
struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
if (lat->type == IPA_CONST_VALUE)
ipcp_discover_new_direct_edges (node1, i, lat->constant);
}
if (dump_file)
dump_function_to_file (node1->decl, dump_file, dump_flags);
for (cs = node->callees; cs; cs = cs->next_callee)
if (cs->callee->aux)
{
fibheap_delete_node (heap, (fibnode_t) cs->callee->aux);
cs->callee->aux = fibheap_insert (heap,
ipcp_estimate_cloning_cost (cs->callee),
cs->callee);
}
}
while (!fibheap_empty (heap))
{
if (dump_file)
fprintf (dump_file, "skipping function %s\n",
cgraph_node_name (node));
node = (struct cgraph_node *) fibheap_extract_min (heap);
node->aux = NULL;
}
fibheap_delete (heap);
BITMAP_FREE (dead_nodes);
ipcp_update_callgraph ();
ipcp_update_profiling ();
}
static void
find_traces_1_round (int branch_th, int exec_th, gcov_type count_th,
struct trace *traces, int *n_traces, int round,
fibheap_t *heap)
{
/* Heap for discarded basic blocks which are possible starting points for
the next round. */
fibheap_t new_heap = fibheap_new ();
while (!fibheap_empty (*heap))
{
basic_block bb;
struct trace *trace;
edge best_edge, e;
fibheapkey_t key;
bb = fibheap_extract_min (*heap);
bbd[bb->index].heap = NULL;
bbd[bb->index].node = NULL;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Getting bb %d\n", bb->index);
/* If the BB's frequency is too low send BB to the next round. */
if (round < N_ROUNDS - 1
&& (bb->frequency < exec_th || bb->count < count_th
|| probably_never_executed_bb_p (bb)))
{
int key = bb_to_key (bb);
bbd[bb->index].heap = new_heap;
bbd[bb->index].node = fibheap_insert (new_heap, key, bb);
if (rtl_dump_file)
fprintf (rtl_dump_file,
" Possible start point of next round: %d (key: %d)\n",
bb->index, key);
continue;
}
trace = traces + *n_traces;
trace->first = bb;
trace->round = round;
trace->length = 0;
(*n_traces)++;
do
{
int prob, freq;
/* The probability and frequency of the best edge. */
int best_prob = INT_MIN / 2;
int best_freq = INT_MIN / 2;
best_edge = NULL;
mark_bb_visited (bb, *n_traces);
trace->length++;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Basic block %d was visited in trace %d\n",
bb->index, *n_traces - 1);
/* Select the successor that will be placed after BB. */
for (e = bb->succ; e; e = e->succ_next)
{
#ifdef ENABLE_CHECKING
if (e->flags & EDGE_FAKE)
abort ();
#endif
if (e->dest == EXIT_BLOCK_PTR)
continue;
if (e->dest->rbi->visited
&& e->dest->rbi->visited != *n_traces)
continue;
prob = e->probability;
freq = EDGE_FREQUENCY (e);
/* Edge that cannot be fallthru or improbable or infrequent
successor (ie. it is unsuitable successor). */
if (!(e->flags & EDGE_CAN_FALLTHRU) || (e->flags & EDGE_COMPLEX)
|| prob < branch_th || freq < exec_th || e->count < count_th)
continue;
if (better_edge_p (bb, e, prob, freq, best_prob, best_freq))
{
best_edge = e;
best_prob = prob;
best_freq = freq;
}
}
/* If the best destination has multiple predecessors, and can be
duplicated cheaper than a jump, don't allow it to be added
to a trace. We'll duplicate it when connecting traces. */
if (best_edge && best_edge->dest->pred->pred_next
&& copy_bb_p (best_edge->dest, 0))
best_edge = NULL;
/* Add all non-selected successors to the heaps. */
for (e = bb->succ; e; e = e->succ_next)
{
if (e == best_edge
|| e->dest == EXIT_BLOCK_PTR
|| e->dest->rbi->visited)
continue;
key = bb_to_key (e->dest);
if (bbd[e->dest->index].heap)
{
/* E->DEST is already in some heap. */
if (key != bbd[e->dest->index].node->key)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Changing key for bb %d from %ld to %ld.\n",
e->dest->index,
(long) bbd[e->dest->index].node->key,
key);
}
fibheap_replace_key (bbd[e->dest->index].heap,
bbd[e->dest->index].node, key);
}
}
else
{
fibheap_t which_heap = *heap;
prob = e->probability;
freq = EDGE_FREQUENCY (e);
if (!(e->flags & EDGE_CAN_FALLTHRU)
|| (e->flags & EDGE_COMPLEX)
|| prob < branch_th || freq < exec_th
|| e->count < count_th)
{
if (round < N_ROUNDS - 1)
which_heap = new_heap;
}
bbd[e->dest->index].heap = which_heap;
bbd[e->dest->index].node = fibheap_insert (which_heap,
key, e->dest);
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
" Possible start of %s round: %d (key: %ld)\n",
(which_heap == new_heap) ? "next" : "this",
e->dest->index, (long) key);
}
}
}
if (best_edge) /* Suitable successor was found. */
{
if (best_edge->dest->rbi->visited == *n_traces)
{
/* We do nothing with one basic block loops. */
if (best_edge->dest != bb)
{
if (EDGE_FREQUENCY (best_edge)
> 4 * best_edge->dest->frequency / 5)
{
/* The loop has at least 4 iterations. If the loop
header is not the first block of the function
we can rotate the loop. */
if (best_edge->dest != ENTRY_BLOCK_PTR->next_bb)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Rotating loop %d - %d\n",
best_edge->dest->index, bb->index);
}
bb->rbi->next = best_edge->dest;
bb = rotate_loop (best_edge, trace, *n_traces);
}
}
else
{
/* The loop has less than 4 iterations. */
/* Check whether there is another edge from BB. */
edge another_edge;
for (another_edge = bb->succ;
another_edge;
another_edge = another_edge->succ_next)
if (another_edge != best_edge)
break;
if (!another_edge && copy_bb_p (best_edge->dest,
!optimize_size))
{
bb = copy_bb (best_edge->dest, best_edge, bb,
*n_traces);
}
}
}
/* Terminate the trace. */
break;
}
else
{
/* Check for a situation
A
/|
B |
\|
C
where
EDGE_FREQUENCY (AB) + EDGE_FREQUENCY (BC)
>= EDGE_FREQUENCY (AC).
(i.e. 2 * B->frequency >= EDGE_FREQUENCY (AC) )
Best ordering is then A B C.
This situation is created for example by:
if (A) B;
C;
*/
for (e = bb->succ; e; e = e->succ_next)
if (e != best_edge
&& (e->flags & EDGE_CAN_FALLTHRU)
&& !(e->flags & EDGE_COMPLEX)
&& !e->dest->rbi->visited
&& !e->dest->pred->pred_next
&& e->dest->succ
&& (e->dest->succ->flags & EDGE_CAN_FALLTHRU)
&& !(e->dest->succ->flags & EDGE_COMPLEX)
&& !e->dest->succ->succ_next
&& e->dest->succ->dest == best_edge->dest
&& 2 * e->dest->frequency >= EDGE_FREQUENCY (best_edge))
{
best_edge = e;
if (rtl_dump_file)
fprintf (rtl_dump_file, "Selecting BB %d\n",
best_edge->dest->index);
break;
}
bb->rbi->next = best_edge->dest;
bb = best_edge->dest;
}
}
}
while (best_edge);
trace->last = bb;
bbd[trace->first->index].start_of_trace = *n_traces - 1;
bbd[trace->last->index].end_of_trace = *n_traces - 1;
/* The trace is terminated so we have to recount the keys in heap
(some block can have a lower key because now one of its predecessors
is an end of the trace). */
for (e = bb->succ; e; e = e->succ_next)
{
if (e->dest == EXIT_BLOCK_PTR
|| e->dest->rbi->visited)
continue;
if (bbd[e->dest->index].heap)
{
key = bb_to_key (e->dest);
if (key != bbd[e->dest->index].node->key)
{
if (rtl_dump_file)
{
fprintf (rtl_dump_file,
"Changing key for bb %d from %ld to %ld.\n",
e->dest->index,
(long) bbd[e->dest->index].node->key, key);
}
fibheap_replace_key (bbd[e->dest->index].heap,
bbd[e->dest->index].node,
key);
}
}
}
}
fibheap_delete (*heap);
/* "Return" the new heap. */
*heap = new_heap;
}
static void
tail_duplicate (void)
{
fibnode_t *blocks = XCNEWVEC (fibnode_t, last_basic_block);
basic_block *trace = XNEWVEC (basic_block, n_basic_blocks);
int *counts = XNEWVEC (int, last_basic_block);
int ninsns = 0, nduplicated = 0;
gcov_type weighted_insns = 0, traced_insns = 0;
fibheap_t heap = fibheap_new ();
gcov_type cover_insns;
int max_dup_insns;
basic_block bb;
if (profile_info && flag_branch_probabilities)
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK);
else
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY);
probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff;
branch_ratio_cutoff =
(REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO));
FOR_EACH_BB (bb)
{
int n = count_insns (bb);
if (!ignore_bb_p (bb))
blocks[bb->index] = fibheap_insert (heap, -bb->frequency,
bb);
counts [bb->index] = n;
ninsns += n;
weighted_insns += n * bb->frequency;
}
if (profile_info && flag_branch_probabilities)
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK);
else
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE);
cover_insns = (weighted_insns * cover_insns + 50) / 100;
max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100;
while (traced_insns < cover_insns && nduplicated < max_dup_insns
&& !fibheap_empty (heap))
{
basic_block bb = fibheap_extract_min (heap);
int n, pos;
if (!bb)
break;
blocks[bb->index] = NULL;
if (ignore_bb_p (bb))
continue;
gcc_assert (!seen (bb));
n = find_trace (bb, trace);
bb = trace[0];
traced_insns += bb->frequency * counts [bb->index];
if (blocks[bb->index])
{
fibheap_delete_node (heap, blocks[bb->index]);
blocks[bb->index] = NULL;
}
for (pos = 1; pos < n; pos++)
{
basic_block bb2 = trace[pos];
if (blocks[bb2->index])
{
fibheap_delete_node (heap, blocks[bb2->index]);
blocks[bb2->index] = NULL;
}
traced_insns += bb2->frequency * counts [bb2->index];
if (EDGE_COUNT (bb2->preds) > 1
&& can_duplicate_block_p (bb2))
{
edge e;
basic_block old = bb2;
e = find_edge (bb, bb2);
nduplicated += counts [bb2->index];
bb2 = duplicate_block (bb2, e, bb);
/* Reconsider the original copy of block we've duplicated.
Removing the most common predecessor may make it to be
head. */
blocks[old->index] =
fibheap_insert (heap, -old->frequency, old);
if (dump_file)
fprintf (dump_file, "Duplicated %i as %i [%i]\n",
old->index, bb2->index, bb2->frequency);
}
bb->aux = bb2;
bb2->il.rtl->visited = 1;
bb = bb2;
/* In case the trace became infrequent, stop duplicating. */
if (ignore_bb_p (bb))
break;
}
if (dump_file)
fprintf (dump_file, " covered now %.1f\n\n",
traced_insns * 100.0 / weighted_insns);
}
if (dump_file)
fprintf (dump_file, "Duplicated %i insns (%i%%)\n", nduplicated,
nduplicated * 100 / ninsns);
free (blocks);
free (trace);
free (counts);
fibheap_delete (heap);
}
static bool
tail_duplicate (void)
{
fibnode_t *blocks = XCNEWVEC (fibnode_t, last_basic_block);
basic_block *trace = XNEWVEC (basic_block, n_basic_blocks);
int *counts = XNEWVEC (int, last_basic_block);
int ninsns = 0, nduplicated = 0;
gcov_type weighted_insns = 0, traced_insns = 0;
fibheap_t heap = fibheap_new ();
gcov_type cover_insns;
int max_dup_insns;
basic_block bb;
bool changed = false;
/* Create an oversized sbitmap to reduce the chance that we need to
resize it. */
bb_seen = sbitmap_alloc (last_basic_block * 2);
bitmap_clear (bb_seen);
initialize_original_copy_tables ();
if (profile_info && flag_branch_probabilities)
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK);
else
probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY);
probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff;
branch_ratio_cutoff =
(REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO));
FOR_EACH_BB (bb)
{
int n = count_insns (bb);
if (!ignore_bb_p (bb))
blocks[bb->index] = fibheap_insert (heap, -bb->frequency,
bb);
counts [bb->index] = n;
ninsns += n;
weighted_insns += n * bb->frequency;
}
if (profile_info && flag_branch_probabilities)
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK);
else
cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE);
cover_insns = (weighted_insns * cover_insns + 50) / 100;
max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100;
while (traced_insns < cover_insns && nduplicated < max_dup_insns
&& !fibheap_empty (heap))
{
basic_block bb = (basic_block) fibheap_extract_min (heap);
int n, pos;
if (!bb)
break;
blocks[bb->index] = NULL;
if (ignore_bb_p (bb))
continue;
gcc_assert (!bb_seen_p (bb));
n = find_trace (bb, trace);
bb = trace[0];
traced_insns += bb->frequency * counts [bb->index];
if (blocks[bb->index])
{
fibheap_delete_node (heap, blocks[bb->index]);
blocks[bb->index] = NULL;
}
for (pos = 1; pos < n; pos++)
{
basic_block bb2 = trace[pos];
if (blocks[bb2->index])
{
fibheap_delete_node (heap, blocks[bb2->index]);
blocks[bb2->index] = NULL;
}
traced_insns += bb2->frequency * counts [bb2->index];
if (EDGE_COUNT (bb2->preds) > 1
&& can_duplicate_block_p (bb2)
/* We have the tendency to duplicate the loop header
of all do { } while loops. Do not do that - it is
not profitable and it might create a loop with multiple
entries or at least rotate the loop. */
&& (!current_loops
|| bb2->loop_father->header != bb2))
{
edge e;
basic_block copy;
nduplicated += counts [bb2->index];
e = find_edge (bb, bb2);
copy = duplicate_block (bb2, e, bb);
flush_pending_stmts (e);
add_phi_args_after_copy (©, 1, NULL);
/* Reconsider the original copy of block we've duplicated.
Removing the most common predecessor may make it to be
head. */
blocks[bb2->index] =
fibheap_insert (heap, -bb2->frequency, bb2);
if (dump_file)
fprintf (dump_file, "Duplicated %i as %i [%i]\n",
bb2->index, copy->index, copy->frequency);
bb2 = copy;
changed = true;
}
mark_bb_seen (bb2);
bb = bb2;
/* In case the trace became infrequent, stop duplicating. */
if (ignore_bb_p (bb))
break;
}
if (dump_file)
fprintf (dump_file, " covered now %.1f\n\n",
traced_insns * 100.0 / weighted_insns);
}
if (dump_file)
fprintf (dump_file, "Duplicated %i insns (%i%%)\n", nduplicated,
nduplicated * 100 / ninsns);
free_original_copy_tables ();
sbitmap_free (bb_seen);
free (blocks);
free (trace);
free (counts);
fibheap_delete (heap);
return changed;
}
int
dirfibheap_empty( dirfibheap_t self ) {
return fibheap_empty( self->heap );
}
