static bool
reachable_at_most_once (basic_block va_arg_bb, basic_block va_start_bb)
{
vec<edge> stack = vNULL;
edge e;
edge_iterator ei;
sbitmap visited;
bool ret;
if (va_arg_bb == va_start_bb)
return true;
if (! dominated_by_p (CDI_DOMINATORS, va_arg_bb, va_start_bb))
return false;
visited = sbitmap_alloc (last_basic_block_for_fn (cfun));
bitmap_clear (visited);
ret = true;
FOR_EACH_EDGE (e, ei, va_arg_bb->preds)
stack.safe_push (e);
while (! stack.is_empty ())
{
basic_block src;
e = stack.pop ();
src = e->src;
if (e->flags & EDGE_COMPLEX)
{
ret = false;
break;
}
if (src == va_start_bb)
continue;
/* va_arg_bb can be executed more times than va_start_bb. */
if (src == va_arg_bb)
{
ret = false;
break;
}
gcc_assert (src != ENTRY_BLOCK_PTR_FOR_FN (cfun));
if (! bitmap_bit_p (visited, src->index))
{
bitmap_set_bit (visited, src->index);
FOR_EACH_EDGE (e, ei, src->preds)
stack.safe_push (e);
}
}
stack.release ();
sbitmap_free (visited);
return ret;
}
static bool
reachable_at_most_once (basic_block va_arg_bb, basic_block va_start_bb)
{
VEC (edge, heap) *stack = NULL;
edge e;
edge_iterator ei;
sbitmap visited;
bool ret;
if (va_arg_bb == va_start_bb)
return true;
if (! dominated_by_p (CDI_DOMINATORS, va_arg_bb, va_start_bb))
return false;
visited = sbitmap_alloc (last_basic_block);
sbitmap_zero (visited);
ret = true;
FOR_EACH_EDGE (e, ei, va_arg_bb->preds)
VEC_safe_push (edge, heap, stack, e);
while (! VEC_empty (edge, stack))
{
basic_block src;
e = VEC_pop (edge, stack);
src = e->src;
if (e->flags & EDGE_COMPLEX)
{
ret = false;
break;
}
if (src == va_start_bb)
continue;
/* va_arg_bb can be executed more times than va_start_bb. */
if (src == va_arg_bb)
{
ret = false;
break;
}
gcc_assert (src != ENTRY_BLOCK_PTR);
if (! TEST_BIT (visited, src->index))
{
SET_BIT (visited, src->index);
FOR_EACH_EDGE (e, ei, src->preds)
VEC_safe_push (edge, heap, stack, e);
}
}
VEC_free (edge, heap, stack);
sbitmap_free (visited);
return ret;
}
/* Check the consistency of profile information. We can't do that
in verify_flow_info, as the counts may get invalid for incompletely
solved graphs, later eliminating of conditionals or roundoff errors.
It is still practical to have them reported for debugging of simple
testcases. */
static void
check_bb_profile (basic_block bb, FILE * file, int indent, int flags)
{
edge e;
int sum = 0;
gcov_type lsum;
edge_iterator ei;
struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl);
char *s_indent = (char *) alloca ((size_t) indent + 1);
memset ((void *) s_indent, ' ', (size_t) indent);
s_indent[indent] = '\0';
if (profile_status_for_function (fun) == PROFILE_ABSENT)
return;
if (bb != EXIT_BLOCK_PTR_FOR_FUNCTION (fun))
{
FOR_EACH_EDGE (e, ei, bb->succs)
sum += e->probability;
if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100)
fprintf (file, "%s%sInvalid sum of outgoing probabilities %.1f%%\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
sum * 100.0 / REG_BR_PROB_BASE);
lsum = 0;
FOR_EACH_EDGE (e, ei, bb->succs)
lsum += e->count;
if (EDGE_COUNT (bb->succs)
&& (lsum - bb->count > 100 || lsum - bb->count < -100))
fprintf (file, "%s%sInvalid sum of outgoing counts %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
(int) lsum, (int) bb->count);
}
if (bb != ENTRY_BLOCK_PTR_FOR_FUNCTION (fun))
{
sum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
sum += EDGE_FREQUENCY (e);
if (abs (sum - bb->frequency) > 100)
fprintf (file,
"%s%sInvalid sum of incoming frequencies %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
sum, bb->frequency);
lsum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
lsum += e->count;
if (lsum - bb->count > 100 || lsum - bb->count < -100)
fprintf (file, "%s%sInvalid sum of incoming counts %i, should be %i\n",
(flags & TDF_COMMENT) ? ";; " : "", s_indent,
(int) lsum, (int) bb->count);
}
}
void
alloc_aux_for_edges (int size)
{
static int initialized;
if (!initialized)
{
gcc_obstack_init (&edge_aux_obstack);
initialized = 1;
}
else
/* Check whether AUX data are still allocated. */
gcc_assert (!first_edge_aux_obj);
first_edge_aux_obj = obstack_alloc (&edge_aux_obstack, 0);
if (size)
{
basic_block bb;
FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR, next_bb)
{
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, bb->succs)
alloc_aux_for_edge (e, size);
}
}
void
gsi_commit_edge_inserts (void)
{
basic_block bb;
edge e;
edge_iterator ei;
gsi_commit_one_edge_insert (single_succ_edge (ENTRY_BLOCK_PTR), NULL);
FOR_EACH_BB (bb)
FOR_EACH_EDGE (e, ei, bb->succs)
gsi_commit_one_edge_insert (e, NULL);
}
void
clear_edges (void)
{
basic_block bb;
edge e;
edge_iterator ei;
FOR_EACH_BB (bb)
{
FOR_EACH_EDGE (e, ei, bb->succs)
free_edge (e);
VEC_truncate (edge, bb->succs, 0);
VEC_truncate (edge, bb->preds, 0);
}
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
free_edge (e);
VEC_truncate (edge, EXIT_BLOCK_PTR->preds, 0);
VEC_truncate (edge, ENTRY_BLOCK_PTR->succs, 0);
gcc_assert (!n_edges);
}
void
clear_edges (void)
{
basic_block bb;
edge e;
edge_iterator ei;
FOR_EACH_BB_FN (bb, cfun)
{
FOR_EACH_EDGE (e, ei, bb->succs)
free_edge (e);
vec_safe_truncate (bb->succs, 0);
vec_safe_truncate (bb->preds, 0);
}
void
clear_edges (struct function *fn)
{
basic_block bb;
edge e;
edge_iterator ei;
FOR_EACH_BB_FN (bb, fn)
{
FOR_EACH_EDGE (e, ei, bb->succs)
free_edge (fn, e);
vec_safe_truncate (bb->succs, 0);
vec_safe_truncate (bb->preds, 0);
}
static void
create_block_for_threading (basic_block bb, struct redirection_data *rd)
{
edge_iterator ei;
edge e;
/* We can use the generic block duplication code and simply remove
the stuff we do not need. */
rd->dup_block = duplicate_block (bb, NULL, NULL);
FOR_EACH_EDGE (e, ei, rd->dup_block->succs)
e->aux = NULL;
/* Zero out the profile, since the block is unreachable for now. */
rd->dup_block->frequency = 0;
rd->dup_block->count = 0;
}
static bool
afdo_set_bb_count (basic_block bb, const stmt_set &promoted)
{
gimple_stmt_iterator gsi;
edge e;
edge_iterator ei;
gcov_type max_count = 0;
bool has_annotated = false;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
count_info info;
gimple *stmt = gsi_stmt (gsi);
if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
continue;
if (afdo_source_profile->get_count_info (stmt, &info))
{
if (info.count > max_count)
max_count = info.count;
has_annotated = true;
if (info.targets.size () > 0
&& promoted.find (stmt) == promoted.end ())
afdo_vpt (&gsi, info.targets, false);
}
}
if (!has_annotated)
return false;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
afdo_source_profile->mark_annotated (gimple_location (gsi_stmt (gsi)));
for (gphi_iterator gpi = gsi_start_phis (bb);
!gsi_end_p (gpi);
gsi_next (&gpi))
{
gphi *phi = gpi.phi ();
size_t i;
for (i = 0; i < gimple_phi_num_args (phi); i++)
afdo_source_profile->mark_annotated (gimple_phi_arg_location (phi, i));
}
FOR_EACH_EDGE (e, ei, bb->succs)
afdo_source_profile->mark_annotated (e->goto_locus);
bb->count = profile_count::from_gcov_type (max_count).afdo ();
return true;
}
static void
flow_loops_cfg_dump (FILE *file)
{
basic_block bb;
if (!file)
return;
FOR_EACH_BB_FN (bb, cfun)
{
edge succ;
edge_iterator ei;
fprintf (file, ";; %d succs { ", bb->index);
FOR_EACH_EDGE (succ, ei, bb->succs)
fprintf (file, "%d ", succ->dest->index);
fprintf (file, "}\n");
}
/* Constructor for a dom walker.
If SKIP_UNREACHBLE_BLOCKS is true, then we need to set
EDGE_EXECUTABLE on every edge in the CFG. */
dom_walker::dom_walker (cdi_direction direction,
bool skip_unreachable_blocks)
: m_dom_direction (direction),
m_skip_unreachable_blocks (skip_unreachable_blocks),
m_unreachable_dom (NULL)
{
/* If we are not skipping unreachable blocks, then there is nothing
to do. */
if (!m_skip_unreachable_blocks)
return;
basic_block bb;
FOR_ALL_BB_FN (bb, cfun)
{
edge_iterator ei;
edge e;
FOR_EACH_EDGE (e, ei, bb->succs)
e->flags |= EDGE_EXECUTABLE;
}
static void
compute_nearerout (struct edge_list *edge_list, sbitmap *farthest,
sbitmap *st_avloc, sbitmap *nearer, sbitmap *nearerout)
{
int num_edges, i;
edge e;
basic_block *worklist, *tos, bb;
edge_iterator ei;
num_edges = NUM_EDGES (edge_list);
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
tos = worklist = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun) + 1);
/* Initialize NEARER for each edge and build a mapping from an edge to
its index. */
for (i = 0; i < num_edges; i++)
INDEX_EDGE (edge_list, i)->aux = (void *) (size_t) i;
/* We want a maximal solution. */
bitmap_vector_ones (nearer, num_edges);
/* Note that even though we want an optimistic setting of NEARER, we
do not want to be overly optimistic. Consider an incoming edge to
the exit block. That edge should always have a NEARER value the
same as FARTHEST for that edge. */
FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
bitmap_copy (nearer[(size_t)e->aux], farthest[(size_t)e->aux]);
/* Add all the blocks to the worklist. This prevents an early exit
from the loop given our optimistic initialization of NEARER. */
FOR_EACH_BB_FN (bb, cfun)
{
*tos++ = bb;
bb->aux = bb;
}
static void
compute_antinout_edge (sbitmap *antloc, sbitmap *transp, sbitmap *antin,
sbitmap *antout)
{
basic_block bb;
edge e;
basic_block *worklist, *qin, *qout, *qend;
unsigned int qlen;
edge_iterator ei;
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
qin = qout = worklist = XNEWVEC (basic_block, n_basic_blocks);
/* We want a maximal solution, so make an optimistic initialization of
ANTIN. */
sbitmap_vector_ones (antin, last_basic_block);
/* Put every block on the worklist; this is necessary because of the
optimistic initialization of ANTIN above. */
FOR_EACH_BB_REVERSE (bb)
{
*qin++ = bb;
bb->aux = bb;
}
qin = worklist;
qend = &worklist[n_basic_blocks - NUM_FIXED_BLOCKS];
qlen = n_basic_blocks - NUM_FIXED_BLOCKS;
/* Mark blocks which are predecessors of the exit block so that we
can easily identify them below. */
FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
e->src->aux = EXIT_BLOCK_PTR;
/* Iterate until the worklist is empty. */
while (qlen)
{
/* Take the first entry off the worklist. */
bb = *qout++;
qlen--;
if (qout >= qend)
qout = worklist;
if (bb->aux == EXIT_BLOCK_PTR)
/* Do not clear the aux field for blocks which are predecessors of
the EXIT block. That way we never add then to the worklist
again. */
sbitmap_zero (antout[bb->index]);
else
{
/* Clear the aux field of this block so that it can be added to
the worklist again if necessary. */
bb->aux = NULL;
sbitmap_intersection_of_succs (antout[bb->index], antin, bb->index);
}
if (sbitmap_a_or_b_and_c_cg (antin[bb->index], antloc[bb->index],
transp[bb->index], antout[bb->index]))
/* If the in state of this block changed, then we need
to add the predecessors of this block to the worklist
if they are not already on the worklist. */
FOR_EACH_EDGE (e, ei, bb->preds)
if (!e->src->aux && e->src != ENTRY_BLOCK_PTR)
{
*qin++ = e->src;
e->src->aux = e;
qlen++;
if (qin >= qend)
qin = worklist;
}
}
clear_aux_for_edges ();
clear_aux_for_blocks ();
free (worklist);
}
static void
compute_nearerout (struct edge_list *edge_list, sbitmap *farthest,
sbitmap *st_avloc, sbitmap *nearer, sbitmap *nearerout)
{
int num_edges, i;
edge e;
basic_block *worklist, *tos, bb;
edge_iterator ei;
num_edges = NUM_EDGES (edge_list);
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
tos = worklist = XNEWVEC (basic_block, n_basic_blocks + 1);
/* Initialize NEARER for each edge and build a mapping from an edge to
its index. */
for (i = 0; i < num_edges; i++)
INDEX_EDGE (edge_list, i)->aux = (void *) (size_t) i;
/* We want a maximal solution. */
sbitmap_vector_ones (nearer, num_edges);
/* Note that even though we want an optimistic setting of NEARER, we
do not want to be overly optimistic. Consider an incoming edge to
the exit block. That edge should always have a NEARER value the
same as FARTHEST for that edge. */
FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
sbitmap_copy (nearer[(size_t)e->aux], farthest[(size_t)e->aux]);
/* Add all the blocks to the worklist. This prevents an early exit
from the loop given our optimistic initialization of NEARER. */
FOR_EACH_BB (bb)
{
*tos++ = bb;
bb->aux = bb;
}
/* Iterate until the worklist is empty. */
while (tos != worklist)
{
/* Take the first entry off the worklist. */
bb = *--tos;
bb->aux = NULL;
/* Compute the intersection of NEARER for each outgoing edge from B. */
sbitmap_ones (nearerout[bb->index]);
FOR_EACH_EDGE (e, ei, bb->succs)
sbitmap_a_and_b (nearerout[bb->index], nearerout[bb->index],
nearer[(size_t) e->aux]);
/* Calculate NEARER for all incoming edges. */
FOR_EACH_EDGE (e, ei, bb->preds)
if (sbitmap_union_of_diff_cg (nearer[(size_t) e->aux],
farthest[(size_t) e->aux],
nearerout[e->dest->index],
st_avloc[e->dest->index])
/* If NEARER for an incoming edge was changed, then we need
to add the source of the incoming edge to the worklist. */
&& e->src != ENTRY_BLOCK_PTR && e->src->aux == 0)
{
*tos++ = e->src;
e->src->aux = e;
}
}
/* Computation of insertion and deletion points requires computing NEAREROUT
for the ENTRY block. We allocated an extra entry in the NEAREROUT array
for just this purpose. */
sbitmap_ones (nearerout[last_basic_block]);
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
sbitmap_a_and_b (nearerout[last_basic_block],
nearerout[last_basic_block],
nearer[(size_t) e->aux]);
clear_aux_for_edges ();
free (tos);
}
void
compute_available (sbitmap *avloc, sbitmap *kill, sbitmap *avout,
sbitmap *avin)
{
edge e;
basic_block *worklist, *qin, *qout, *qend, bb;
unsigned int qlen;
edge_iterator ei;
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
qin = qout = worklist =
XNEWVEC (basic_block, n_basic_blocks - NUM_FIXED_BLOCKS);
/* We want a maximal solution. */
sbitmap_vector_ones (avout, last_basic_block);
/* Put every block on the worklist; this is necessary because of the
optimistic initialization of AVOUT above. */
FOR_EACH_BB (bb)
{
*qin++ = bb;
bb->aux = bb;
}
qin = worklist;
qend = &worklist[n_basic_blocks - NUM_FIXED_BLOCKS];
qlen = n_basic_blocks - NUM_FIXED_BLOCKS;
/* Mark blocks which are successors of the entry block so that we
can easily identify them below. */
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
e->dest->aux = ENTRY_BLOCK_PTR;
/* Iterate until the worklist is empty. */
while (qlen)
{
/* Take the first entry off the worklist. */
bb = *qout++;
qlen--;
if (qout >= qend)
qout = worklist;
/* If one of the predecessor blocks is the ENTRY block, then the
intersection of avouts is the null set. We can identify such blocks
by the special value in the AUX field in the block structure. */
if (bb->aux == ENTRY_BLOCK_PTR)
/* Do not clear the aux field for blocks which are successors of the
ENTRY block. That way we never add then to the worklist again. */
sbitmap_zero (avin[bb->index]);
else
{
/* Clear the aux field of this block so that it can be added to
the worklist again if necessary. */
bb->aux = NULL;
sbitmap_intersection_of_preds (avin[bb->index], avout, bb->index);
}
if (sbitmap_union_of_diff_cg (avout[bb->index], avloc[bb->index],
avin[bb->index], kill[bb->index]))
/* If the out state of this block changed, then we need
to add the successors of this block to the worklist
if they are not already on the worklist. */
FOR_EACH_EDGE (e, ei, bb->succs)
if (!e->dest->aux && e->dest != EXIT_BLOCK_PTR)
{
*qin++ = e->dest;
e->dest->aux = e;
qlen++;
if (qin >= qend)
qin = worklist;
}
}
clear_aux_for_edges ();
clear_aux_for_blocks ();
free (worklist);
}
static void
compute_laterin (struct edge_list *edge_list, sbitmap *earliest,
sbitmap *antloc, sbitmap *later, sbitmap *laterin)
{
int num_edges, i;
edge e;
basic_block *worklist, *qin, *qout, *qend, bb;
unsigned int qlen;
edge_iterator ei;
num_edges = NUM_EDGES (edge_list);
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
qin = qout = worklist
= XNEWVEC (basic_block, n_basic_blocks);
/* Initialize a mapping from each edge to its index. */
for (i = 0; i < num_edges; i++)
INDEX_EDGE (edge_list, i)->aux = (void *) (size_t) i;
/* We want a maximal solution, so initially consider LATER true for
all edges. This allows propagation through a loop since the incoming
loop edge will have LATER set, so if all the other incoming edges
to the loop are set, then LATERIN will be set for the head of the
loop.
If the optimistic setting of LATER on that edge was incorrect (for
example the expression is ANTLOC in a block within the loop) then
this algorithm will detect it when we process the block at the head
of the optimistic edge. That will requeue the affected blocks. */
sbitmap_vector_ones (later, num_edges);
/* Note that even though we want an optimistic setting of LATER, we
do not want to be overly optimistic. Consider an outgoing edge from
the entry block. That edge should always have a LATER value the
same as EARLIEST for that edge. */
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
sbitmap_copy (later[(size_t) e->aux], earliest[(size_t) e->aux]);
/* Add all the blocks to the worklist. This prevents an early exit from
the loop given our optimistic initialization of LATER above. */
FOR_EACH_BB (bb)
{
*qin++ = bb;
bb->aux = bb;
}
/* Note that we do not use the last allocated element for our queue,
as EXIT_BLOCK is never inserted into it. */
qin = worklist;
qend = &worklist[n_basic_blocks - NUM_FIXED_BLOCKS];
qlen = n_basic_blocks - NUM_FIXED_BLOCKS;
/* Iterate until the worklist is empty. */
while (qlen)
{
/* Take the first entry off the worklist. */
bb = *qout++;
bb->aux = NULL;
qlen--;
if (qout >= qend)
qout = worklist;
/* Compute the intersection of LATERIN for each incoming edge to B. */
sbitmap_ones (laterin[bb->index]);
FOR_EACH_EDGE (e, ei, bb->preds)
sbitmap_a_and_b (laterin[bb->index], laterin[bb->index],
later[(size_t)e->aux]);
/* Calculate LATER for all outgoing edges. */
FOR_EACH_EDGE (e, ei, bb->succs)
if (sbitmap_union_of_diff_cg (later[(size_t) e->aux],
earliest[(size_t) e->aux],
laterin[e->src->index],
antloc[e->src->index])
/* If LATER for an outgoing edge was changed, then we need
to add the target of the outgoing edge to the worklist. */
&& e->dest != EXIT_BLOCK_PTR && e->dest->aux == 0)
{
*qin++ = e->dest;
e->dest->aux = e;
qlen++;
if (qin >= qend)
qin = worklist;
}
}
/* Computation of insertion and deletion points requires computing LATERIN
for the EXIT block. We allocated an extra entry in the LATERIN array
for just this purpose. */
sbitmap_ones (laterin[last_basic_block]);
FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
sbitmap_a_and_b (laterin[last_basic_block],
laterin[last_basic_block],
later[(size_t) e->aux]);
clear_aux_for_edges ();
free (worklist);
}
void
compute_available (sbitmap *avloc, sbitmap *kill, sbitmap *avout,
sbitmap *avin)
{
edge e;
basic_block *worklist, *qin, *qout, *qend, bb;
unsigned int qlen;
edge_iterator ei;
/* Allocate a worklist array/queue. Entries are only added to the
list if they were not already on the list. So the size is
bounded by the number of basic blocks. */
qin = qout = worklist =
XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS);
/* We want a maximal solution. */
bitmap_vector_ones (avout, last_basic_block_for_fn (cfun));
/* Put every block on the worklist; this is necessary because of the
optimistic initialization of AVOUT above. Use inverted postorder
to make the dataflow problem require less iterations. */
int *postorder = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
int postorder_num = inverted_post_order_compute (postorder);
for (int i = 0; i < postorder_num; ++i)
{
bb = BASIC_BLOCK_FOR_FN (cfun, postorder[i]);
if (bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
|| bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
continue;
*qin++ = bb;
bb->aux = bb;
}
free (postorder);
qin = worklist;
qend = &worklist[n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS];
qlen = n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS;
/* Mark blocks which are successors of the entry block so that we
can easily identify them below. */
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs)
e->dest->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun);
/* Iterate until the worklist is empty. */
while (qlen)
{
/* Take the first entry off the worklist. */
bb = *qout++;
qlen--;
if (qout >= qend)
qout = worklist;
/* If one of the predecessor blocks is the ENTRY block, then the
intersection of avouts is the null set. We can identify such blocks
by the special value in the AUX field in the block structure. */
if (bb->aux == ENTRY_BLOCK_PTR_FOR_FN (cfun))
/* Do not clear the aux field for blocks which are successors of the
ENTRY block. That way we never add then to the worklist again. */
bitmap_clear (avin[bb->index]);
else
{
/* Clear the aux field of this block so that it can be added to
the worklist again if necessary. */
bb->aux = NULL;
bitmap_intersection_of_preds (avin[bb->index], avout, bb);
}
if (bitmap_ior_and_compl (avout[bb->index], avloc[bb->index],
avin[bb->index], kill[bb->index]))
/* If the out state of this block changed, then we need
to add the successors of this block to the worklist
if they are not already on the worklist. */
FOR_EACH_EDGE (e, ei, bb->succs)
if (!e->dest->aux && e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
{
*qin++ = e->dest;
e->dest->aux = e;
qlen++;
if (qin >= qend)
qin = worklist;
}
}
clear_aux_for_edges ();
clear_aux_for_blocks ();
free (worklist);
}
unsigned int
execute_fixup_cfg (void)
{
basic_block bb;
gimple_stmt_iterator gsi;
int todo = gimple_in_ssa_p (cfun) ? TODO_verify_ssa : 0;
gcov_type count_scale;
edge e;
edge_iterator ei;
count_scale
= GCOV_COMPUTE_SCALE (cgraph_get_node (current_function_decl)->count,
ENTRY_BLOCK_PTR->count);
ENTRY_BLOCK_PTR->count = cgraph_get_node (current_function_decl)->count;
EXIT_BLOCK_PTR->count = apply_scale (EXIT_BLOCK_PTR->count,
count_scale);
FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
e->count = apply_scale (e->count, count_scale);
FOR_EACH_BB (bb)
{
bb->count = apply_scale (bb->count, count_scale);
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple stmt = gsi_stmt (gsi);
tree decl = is_gimple_call (stmt)
? gimple_call_fndecl (stmt)
: NULL;
if (decl)
{
int flags = gimple_call_flags (stmt);
if (flags & (ECF_CONST | ECF_PURE | ECF_LOOPING_CONST_OR_PURE))
{
if (gimple_purge_dead_abnormal_call_edges (bb))
todo |= TODO_cleanup_cfg;
if (gimple_in_ssa_p (cfun))
{
todo |= TODO_update_ssa | TODO_cleanup_cfg;
update_stmt (stmt);
}
}
if (flags & ECF_NORETURN
&& fixup_noreturn_call (stmt))
todo |= TODO_cleanup_cfg;
}
if (maybe_clean_eh_stmt (stmt)
&& gimple_purge_dead_eh_edges (bb))
todo |= TODO_cleanup_cfg;
}
FOR_EACH_EDGE (e, ei, bb->succs)
e->count = apply_scale (e->count, count_scale);
/* If we have a basic block with no successors that does not
end with a control statement or a noreturn call end it with
a call to __builtin_unreachable. This situation can occur
when inlining a noreturn call that does in fact return. */
if (EDGE_COUNT (bb->succs) == 0)
{
gimple stmt = last_stmt (bb);
if (!stmt
|| (!is_ctrl_stmt (stmt)
&& (!is_gimple_call (stmt)
|| (gimple_call_flags (stmt) & ECF_NORETURN) == 0)))
{
stmt = gimple_build_call
(builtin_decl_implicit (BUILT_IN_UNREACHABLE), 0);
gimple_stmt_iterator gsi = gsi_last_bb (bb);
gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
}
}
}
if (count_scale != REG_BR_PROB_BASE)
compute_function_frequency ();
/* We just processed all calls. */
if (cfun->gimple_df)
vec_free (MODIFIED_NORETURN_CALLS (cfun));
/* Dump a textual representation of the flowgraph. */
if (dump_file)
gimple_dump_cfg (dump_file, dump_flags);
if (current_loops
&& (todo & TODO_cleanup_cfg))
loops_state_set (LOOPS_NEED_FIXUP);
return todo;
}
static void
find_tail_calls (basic_block bb, struct tailcall **ret)
{
tree ass_var = NULL_TREE, ret_var, func, param;
gimple stmt, call = NULL;
gimple_stmt_iterator gsi, agsi;
bool tail_recursion;
struct tailcall *nw;
edge e;
tree m, a;
basic_block abb;
size_t idx;
tree var;
if (!single_succ_p (bb))
return;
for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
{
stmt = gsi_stmt (gsi);
/* Ignore labels, returns, clobbers and debug stmts. */
if (gimple_code (stmt) == GIMPLE_LABEL
|| gimple_code (stmt) == GIMPLE_RETURN
|| gimple_clobber_p (stmt)
|| is_gimple_debug (stmt))
continue;
/* Check for a call. */
if (is_gimple_call (stmt))
{
call = stmt;
ass_var = gimple_call_lhs (stmt);
break;
}
/* If the statement references memory or volatile operands, fail. */
if (gimple_references_memory_p (stmt)
|| gimple_has_volatile_ops (stmt))
return;
}
if (gsi_end_p (gsi))
{
edge_iterator ei;
/* Recurse to the predecessors. */
FOR_EACH_EDGE (e, ei, bb->preds)
find_tail_calls (e->src, ret);
return;
}
/* If the LHS of our call is not just a simple register, we can't
transform this into a tail or sibling call. This situation happens,
in (e.g.) "*p = foo()" where foo returns a struct. In this case
we won't have a temporary here, but we need to carry out the side
effect anyway, so tailcall is impossible.
??? In some situations (when the struct is returned in memory via
invisible argument) we could deal with this, e.g. by passing 'p'
itself as that argument to foo, but it's too early to do this here,
and expand_call() will not handle it anyway. If it ever can, then
we need to revisit this here, to allow that situation. */
if (ass_var && !is_gimple_reg (ass_var))
return;
/* We found the call, check whether it is suitable. */
tail_recursion = false;
func = gimple_call_fndecl (call);
if (func
&& !DECL_BUILT_IN (func)
&& recursive_call_p (current_function_decl, func))
{
tree arg;
for (param = DECL_ARGUMENTS (func), idx = 0;
param && idx < gimple_call_num_args (call);
param = DECL_CHAIN (param), idx ++)
{
arg = gimple_call_arg (call, idx);
if (param != arg)
{
/* Make sure there are no problems with copying. The parameter
have a copyable type and the two arguments must have reasonably
equivalent types. The latter requirement could be relaxed if
we emitted a suitable type conversion statement. */
if (!is_gimple_reg_type (TREE_TYPE (param))
|| !useless_type_conversion_p (TREE_TYPE (param),
TREE_TYPE (arg)))
break;
/* The parameter should be a real operand, so that phi node
created for it at the start of the function has the meaning
of copying the value. This test implies is_gimple_reg_type
from the previous condition, however this one could be
relaxed by being more careful with copying the new value
of the parameter (emitting appropriate GIMPLE_ASSIGN and
updating the virtual operands). */
if (!is_gimple_reg (param))
break;
}
}
if (idx == gimple_call_num_args (call) && !param)
tail_recursion = true;
}
/* Make sure the tail invocation of this function does not refer
to local variables. */
FOR_EACH_LOCAL_DECL (cfun, idx, var)
{
if (TREE_CODE (var) != PARM_DECL
&& auto_var_in_fn_p (var, cfun->decl)
&& (ref_maybe_used_by_stmt_p (call, var)
|| call_may_clobber_ref_p (call, var)))
return;
}
/* Now check the statements after the call. None of them has virtual
operands, so they may only depend on the call through its return
value. The return value should also be dependent on each of them,
since we are running after dce. */
m = NULL_TREE;
a = NULL_TREE;
abb = bb;
agsi = gsi;
while (1)
{
tree tmp_a = NULL_TREE;
tree tmp_m = NULL_TREE;
gsi_next (&agsi);
while (gsi_end_p (agsi))
{
ass_var = propagate_through_phis (ass_var, single_succ_edge (abb));
abb = single_succ (abb);
agsi = gsi_start_bb (abb);
}
stmt = gsi_stmt (agsi);
if (gimple_code (stmt) == GIMPLE_LABEL)
continue;
if (gimple_code (stmt) == GIMPLE_RETURN)
break;
if (gimple_clobber_p (stmt))
continue;
if (is_gimple_debug (stmt))
continue;
if (gimple_code (stmt) != GIMPLE_ASSIGN)
return;
/* This is a gimple assign. */
if (! process_assignment (stmt, gsi, &tmp_m, &tmp_a, &ass_var))
return;
if (tmp_a)
{
tree type = TREE_TYPE (tmp_a);
if (a)
a = fold_build2 (PLUS_EXPR, type, fold_convert (type, a), tmp_a);
else
a = tmp_a;
}
if (tmp_m)
{
tree type = TREE_TYPE (tmp_m);
if (m)
m = fold_build2 (MULT_EXPR, type, fold_convert (type, m), tmp_m);
else
m = tmp_m;
if (a)
a = fold_build2 (MULT_EXPR, type, fold_convert (type, a), tmp_m);
}
}
/* See if this is a tail call we can handle. */
ret_var = gimple_return_retval (stmt);
/* We may proceed if there either is no return value, or the return value
is identical to the call's return. */
if (ret_var
&& (ret_var != ass_var))
return;
/* If this is not a tail recursive call, we cannot handle addends or
multiplicands. */
if (!tail_recursion && (m || a))
return;
/* For pointers only allow additions. */
if (m && POINTER_TYPE_P (TREE_TYPE (DECL_RESULT (current_function_decl))))
return;
nw = XNEW (struct tailcall);
nw->call_gsi = gsi;
nw->tail_recursion = tail_recursion;
nw->mult = m;
nw->add = a;
nw->next = *ret;
*ret = nw;
}
unsigned int
execute_fixup_cfg (void)
{
basic_block bb;
gimple_stmt_iterator gsi;
int todo = gimple_in_ssa_p (cfun) ? TODO_verify_ssa : 0;
gcov_type count_scale;
edge e;
edge_iterator ei;
if (ENTRY_BLOCK_PTR->count)
count_scale = (cgraph_node (current_function_decl)->count * REG_BR_PROB_BASE
+ ENTRY_BLOCK_PTR->count / 2) / ENTRY_BLOCK_PTR->count;
else
count_scale = REG_BR_PROB_BASE;
ENTRY_BLOCK_PTR->count = cgraph_node (current_function_decl)->count;
EXIT_BLOCK_PTR->count = (EXIT_BLOCK_PTR->count * count_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
FOR_EACH_BB (bb)
{
bb->count = (bb->count * count_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple stmt = gsi_stmt (gsi);
tree decl = is_gimple_call (stmt)
? gimple_call_fndecl (stmt)
: NULL;
if (decl
&& gimple_call_flags (stmt) & (ECF_CONST
| ECF_PURE
| ECF_LOOPING_CONST_OR_PURE))
{
if (gimple_in_ssa_p (cfun))
{
todo |= TODO_update_ssa | TODO_cleanup_cfg;
mark_symbols_for_renaming (stmt);
update_stmt (stmt);
}
}
maybe_clean_eh_stmt (stmt);
}
if (gimple_purge_dead_eh_edges (bb))
todo |= TODO_cleanup_cfg;
FOR_EACH_EDGE (e, ei, bb->succs)
e->count = (e->count * count_scale
+ REG_BR_PROB_BASE / 2) / REG_BR_PROB_BASE;
}
if (count_scale != REG_BR_PROB_BASE)
compute_function_frequency ();
/* Dump a textual representation of the flowgraph. */
if (dump_file)
gimple_dump_cfg (dump_file, dump_flags);
return todo;
}
