void
loop_optimizer_init (unsigned flags)
{
timevar_push (TV_LOOP_INIT);
if (!current_loops)
{
gcc_assert (!(cfun->curr_properties & PROP_loops));
/* Find the loops. */
current_loops = flow_loops_find (NULL);
}
else
{
bool recorded_exits = loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS);
bool needs_fixup = loops_state_satisfies_p (LOOPS_NEED_FIXUP);
gcc_assert (cfun->curr_properties & PROP_loops);
/* Ensure that the dominators are computed, like flow_loops_find does. */
calculate_dominance_info (CDI_DOMINATORS);
#ifdef ENABLE_CHECKING
if (!needs_fixup)
verify_loop_structure ();
#endif
/* Clear all flags. */
if (recorded_exits)
release_recorded_exits ();
loops_state_clear (~0U);
if (needs_fixup)
{
/* Apply LOOPS_MAY_HAVE_MULTIPLE_LATCHES early as fix_loop_structure
re-applies flags. */
loops_state_set (flags & LOOPS_MAY_HAVE_MULTIPLE_LATCHES);
fix_loop_structure (NULL);
}
}
/* Apply flags to loops. */
apply_loop_flags (flags);
/* Dump loops. */
flow_loops_dump (dump_file, NULL, 1);
#ifdef ENABLE_CHECKING
verify_loop_structure ();
#endif
timevar_pop (TV_LOOP_INIT);
}
/* Main entry point. Perform loop unswitching on all suitable LOOPS. */
void
unswitch_loops (struct loops *loops)
{
int i, num;
struct loop *loop;
/* Go through inner loops (only original ones). */
num = loops->num;
for (i = 1; i < num; i++)
{
/* Removed loop? */
loop = loops->parray[i];
if (!loop)
continue;
if (loop->inner)
continue;
unswitch_single_loop (loops, loop, NULL_RTX, 0);
#ifdef ENABLE_CHECKING
verify_dominators (CDI_DOMINATORS);
verify_loop_structure (loops);
#endif
}
iv_analysis_done ();
}
struct loops *
loop_optimizer_init (FILE *dumpfile)
{
struct loops *loops = xcalloc (1, sizeof (struct loops));
edge e;
edge_iterator ei;
static bool first_time = true;
if (first_time)
{
first_time = false;
init_set_costs ();
}
/* Avoid annoying special cases of edges going to exit
block. */
for (ei = ei_start (EXIT_BLOCK_PTR->preds); (e = ei_safe_edge (ei)); )
if ((e->flags & EDGE_FALLTHRU) && !single_succ_p (e->src))
split_edge (e);
else
ei_next (&ei);
/* Find the loops. */
if (flow_loops_find (loops) <= 1)
{
/* No loops. */
flow_loops_free (loops);
free (loops);
return NULL;
}
/* Not going to update these. */
free (loops->cfg.rc_order);
loops->cfg.rc_order = NULL;
free (loops->cfg.dfs_order);
loops->cfg.dfs_order = NULL;
/* Create pre-headers. */
create_preheaders (loops, CP_SIMPLE_PREHEADERS);
/* Force all latches to have only single successor. */
force_single_succ_latches (loops);
/* Mark irreducible loops. */
mark_irreducible_loops (loops);
/* Dump loops. */
flow_loops_dump (loops, dumpfile, NULL, 1);
#ifdef ENABLE_CHECKING
verify_dominators (CDI_DOMINATORS);
verify_loop_structure (loops);
#endif
return loops;
}
static inline void
graphite_verify (void)
{
#ifdef ENABLE_CHECKING
verify_loop_structure ();
verify_loop_closed_ssa (true);
#endif
}
static inline void
graphite_verify (void)
{
#ifdef ENABLE_CHECKING
verify_loop_structure ();
verify_dominators (CDI_DOMINATORS);
verify_loop_closed_ssa (true);
#endif
}
/* Main entry point. Perform loop unswitching on all suitable loops. */
void
unswitch_loops (void)
{
loop_iterator li;
struct loop *loop;
/* Go through inner loops (only original ones). */
FOR_EACH_LOOP (li, loop, LI_ONLY_INNERMOST)
{
unswitch_single_loop (loop, NULL_RTX, 0);
#ifdef ENABLE_CHECKING
verify_loop_structure ();
#endif
}
void
doloop_optimize_loops (struct loops *loops)
{
unsigned i;
struct loop *loop;
for (i = 1; i < loops->num; i++)
{
loop = loops->parray[i];
if (!loop)
continue;
doloop_optimize (loop);
}
iv_analysis_done ();
#ifdef ENABLE_CHECKING
verify_dominators (CDI_DOMINATORS);
verify_loop_structure (loops);
#endif
}
int
flow_loops_find (struct loops *loops, int flags)
{
int i;
int b;
int num_loops;
edge e;
sbitmap headers;
int *dfs_order;
int *rc_order;
basic_block header;
basic_block bb;
/* This function cannot be repeatedly called with different
flags to build up the loop information. The loop tree
must always be built if this function is called. */
if (! (flags & LOOP_TREE))
abort ();
memset (loops, 0, sizeof *loops);
/* Taking care of this degenerate case makes the rest of
this code simpler. */
if (n_basic_blocks == 0)
return 0;
dfs_order = NULL;
rc_order = NULL;
/* Join loops with shared headers. */
canonicalize_loop_headers ();
/* Compute the dominators. */
calculate_dominance_info (CDI_DOMINATORS);
/* Count the number of loop headers. This should be the
same as the number of natural loops. */
headers = sbitmap_alloc (last_basic_block);
sbitmap_zero (headers);
num_loops = 0;
FOR_EACH_BB (header)
{
int more_latches = 0;
header->loop_depth = 0;
/* If we have an abnormal predecessor, do not consider the
loop (not worth the problems). */
for (e = header->pred; e; e = e->pred_next)
if (e->flags & EDGE_ABNORMAL)
break;
if (e)
continue;
for (e = header->pred; e; e = e->pred_next)
{
basic_block latch = e->src;
if (e->flags & EDGE_ABNORMAL)
abort ();
/* Look for back edges where a predecessor is dominated
by this block. A natural loop has a single entry
node (header) that dominates all the nodes in the
loop. It also has single back edge to the header
from a latch node. */
if (latch != ENTRY_BLOCK_PTR
&& dominated_by_p (CDI_DOMINATORS, latch, header))
{
/* Shared headers should be eliminated by now. */
if (more_latches)
abort ();
more_latches = 1;
SET_BIT (headers, header->index);
num_loops++;
}
}
}
/* Allocate loop structures. */
loops->parray = xcalloc (num_loops + 1, sizeof (struct loop *));
/* Dummy loop containing whole function. */
loops->parray[0] = xcalloc (1, sizeof (struct loop));
loops->parray[0]->next = NULL;
loops->parray[0]->inner = NULL;
loops->parray[0]->outer = NULL;
loops->parray[0]->depth = 0;
loops->parray[0]->pred = NULL;
loops->parray[0]->num_nodes = n_basic_blocks + 2;
loops->parray[0]->latch = EXIT_BLOCK_PTR;
loops->parray[0]->header = ENTRY_BLOCK_PTR;
ENTRY_BLOCK_PTR->loop_father = loops->parray[0];
EXIT_BLOCK_PTR->loop_father = loops->parray[0];
loops->tree_root = loops->parray[0];
/* Find and record information about all the natural loops
in the CFG. */
loops->num = 1;
FOR_EACH_BB (bb)
bb->loop_father = loops->tree_root;
if (num_loops)
{
/* Compute depth first search order of the CFG so that outer
natural loops will be found before inner natural loops. */
dfs_order = xmalloc (n_basic_blocks * sizeof (int));
rc_order = xmalloc (n_basic_blocks * sizeof (int));
flow_depth_first_order_compute (dfs_order, rc_order);
/* Save CFG derived information to avoid recomputing it. */
loops->cfg.dfs_order = dfs_order;
loops->cfg.rc_order = rc_order;
num_loops = 1;
for (b = 0; b < n_basic_blocks; b++)
{
struct loop *loop;
/* Search the nodes of the CFG in reverse completion order
so that we can find outer loops first. */
if (!TEST_BIT (headers, rc_order[b]))
continue;
header = BASIC_BLOCK (rc_order[b]);
loop = loops->parray[num_loops] = xcalloc (1, sizeof (struct loop));
loop->header = header;
loop->num = num_loops;
num_loops++;
/* Look for the latch for this header block. */
for (e = header->pred; e; e = e->pred_next)
{
basic_block latch = e->src;
if (latch != ENTRY_BLOCK_PTR
&& dominated_by_p (CDI_DOMINATORS, latch, header))
{
loop->latch = latch;
break;
}
}
flow_loop_tree_node_add (header->loop_father, loop);
loop->num_nodes = flow_loop_nodes_find (loop->header, loop);
}
/* Assign the loop nesting depth and enclosed loop level for each
loop. */
loops->levels = flow_loops_level_compute (loops);
/* Scan the loops. */
for (i = 1; i < num_loops; i++)
flow_loop_scan (loops->parray[i], flags);
loops->num = num_loops;
}
else
{
free_dominance_info (CDI_DOMINATORS);
}
sbitmap_free (headers);
loops->state = 0;
#ifdef ENABLE_CHECKING
verify_flow_info ();
verify_loop_structure (loops);
#endif
return loops->num;
}
unsigned
fix_loop_structure (bitmap changed_bbs)
{
basic_block bb;
int record_exits = 0;
struct loop *loop;
unsigned old_nloops, i;
timevar_push (TV_LOOP_INIT);
/* We need exact and fast dominance info to be available. */
gcc_assert (dom_info_state (CDI_DOMINATORS) == DOM_OK);
if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
{
release_recorded_exits ();
record_exits = LOOPS_HAVE_RECORDED_EXITS;
}
/* Remember the depth of the blocks in the loop hierarchy, so that we can
recognize blocks whose loop nesting relationship has changed. */
if (changed_bbs)
FOR_EACH_BB_FN (bb, cfun)
bb->aux = (void *) (size_t) loop_depth (bb->loop_father);
/* Remove the dead loops from structures. We start from the innermost
loops, so that when we remove the loops, we know that the loops inside
are preserved, and do not waste time relinking loops that will be
removed later. */
FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
{
/* Detect the case that the loop is no longer present even though
it wasn't marked for removal.
??? If we do that we can get away with not marking loops for
removal at all. And possibly avoid some spurious removals. */
if (loop->header
&& bb_loop_header_p (loop->header))
continue;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "fix_loop_structure: removing loop %d\n",
loop->num);
while (loop->inner)
{
struct loop *ploop = loop->inner;
flow_loop_tree_node_remove (ploop);
flow_loop_tree_node_add (loop_outer (loop), ploop);
}
/* Remove the loop. */
if (loop->header)
loop->former_header = loop->header;
else
gcc_assert (loop->former_header != NULL);
loop->header = NULL;
flow_loop_tree_node_remove (loop);
}
/* Remember the number of loops so we can return how many new loops
flow_loops_find discovered. */
old_nloops = number_of_loops (cfun);
/* Re-compute loop structure in-place. */
flow_loops_find (current_loops);
/* Mark the blocks whose loop has changed. */
if (changed_bbs)
{
FOR_EACH_BB_FN (bb, cfun)
{
if ((void *) (size_t) loop_depth (bb->loop_father) != bb->aux)
bitmap_set_bit (changed_bbs, bb->index);
bb->aux = NULL;
}
}
/* Finally free deleted loops. */
FOR_EACH_VEC_ELT (*get_loops (cfun), i, loop)
if (loop && loop->header == NULL)
{
if (dump_file
&& ((unsigned) loop->former_header->index
< basic_block_info_for_fn (cfun)->length ()))
{
basic_block former_header
= BASIC_BLOCK_FOR_FN (cfun, loop->former_header->index);
/* If the old header still exists we want to check if the
original loop is re-discovered or the old header is now
part of a newly discovered loop.
In both cases we should have avoided removing the loop. */
if (former_header == loop->former_header)
{
if (former_header->loop_father->header == former_header)
fprintf (dump_file, "fix_loop_structure: rediscovered "
"removed loop %d as loop %d with old header %d\n",
loop->num, former_header->loop_father->num,
former_header->index);
else if ((unsigned) former_header->loop_father->num
>= old_nloops)
fprintf (dump_file, "fix_loop_structure: header %d of "
"removed loop %d is part of the newly "
"discovered loop %d with header %d\n",
former_header->index, loop->num,
former_header->loop_father->num,
former_header->loop_father->header->index);
}
}
(*get_loops (cfun))[i] = NULL;
flow_loop_free (loop);
}
loops_state_clear (LOOPS_NEED_FIXUP);
/* Apply flags to loops. */
apply_loop_flags (current_loops->state | record_exits);
#ifdef ENABLE_CHECKING
verify_loop_structure ();
#endif
timevar_pop (TV_LOOP_INIT);
return number_of_loops (cfun) - old_nloops;
}
static void
copy_loop_headers (void)
{
struct loops *loops;
unsigned i;
struct loop *loop;
basic_block header;
edge exit;
basic_block *bbs;
unsigned n_bbs;
loops = loop_optimizer_init (dump_file);
if (!loops)
return;
rewrite_into_loop_closed_ssa ();
/* We do not try to keep the information about irreducible regions
up-to-date. */
loops->state &= ~LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS;
#ifdef ENABLE_CHECKING
verify_loop_structure (loops);
#endif
bbs = xmalloc (sizeof (basic_block) * n_basic_blocks);
for (i = 1; i < loops->num; i++)
{
/* Copy at most 20 insns. */
int limit = 20;
loop = loops->parray[i];
if (!loop)
continue;
header = loop->header;
/* If the loop is already a do-while style one (either because it was
written as such, or because jump threading transformed it into one),
we might be in fact peeling the first iteration of the loop. This
in general is not a good idea. */
if (do_while_loop_p (loop))
continue;
/* Iterate the header copying up to limit; this takes care of the cases
like while (a && b) {...}, where we want to have both of the conditions
copied. TODO -- handle while (a || b) - like cases, by not requiring
the header to have just a single successor and copying up to
postdominator. */
exit = NULL;
n_bbs = 0;
while (should_duplicate_loop_header_p (header, loop, &limit))
{
/* Find a successor of header that is inside a loop; i.e. the new
header after the condition is copied. */
if (flow_bb_inside_loop_p (loop, EDGE_SUCC (header, 0)->dest))
exit = EDGE_SUCC (header, 0);
else
exit = EDGE_SUCC (header, 1);
bbs[n_bbs++] = header;
header = exit->dest;
}
if (!exit)
continue;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Duplicating header of the loop %d up to edge %d->%d.\n",
loop->num, exit->src->index, exit->dest->index);
/* Ensure that the header will have just the latch as a predecessor
inside the loop. */
if (EDGE_COUNT (exit->dest->preds) > 1)
exit = EDGE_SUCC (loop_split_edge_with (exit, NULL), 0);
if (!tree_duplicate_sese_region (loop_preheader_edge (loop), exit,
bbs, n_bbs, NULL))
{
fprintf (dump_file, "Duplication failed.\n");
continue;
}
/* Ensure that the latch and the preheader is simple (we know that they
are not now, since there was the loop exit condition. */
loop_split_edge_with (loop_preheader_edge (loop), NULL);
loop_split_edge_with (loop_latch_edge (loop), NULL);
}
free (bbs);
#ifdef ENABLE_CHECKING
verify_loop_closed_ssa ();
#endif
loop_optimizer_finalize (loops, NULL);
}
static unsigned int
copy_loop_headers (void)
{
struct loops *loops;
unsigned i;
struct loop *loop;
basic_block header;
edge exit, entry;
basic_block *bbs, *copied_bbs;
unsigned n_bbs;
unsigned bbs_size;
loops = loop_optimizer_init (LOOPS_HAVE_PREHEADERS
| LOOPS_HAVE_SIMPLE_LATCHES);
if (!loops)
return 0;
#ifdef ENABLE_CHECKING
verify_loop_structure (loops);
#endif
bbs = XNEWVEC (basic_block, n_basic_blocks);
copied_bbs = XNEWVEC (basic_block, n_basic_blocks);
bbs_size = n_basic_blocks;
for (i = 1; i < loops->num; i++)
{
/* Copy at most 20 insns. */
int limit = 20;
loop = loops->parray[i];
if (!loop)
continue;
header = loop->header;
/* If the loop is already a do-while style one (either because it was
written as such, or because jump threading transformed it into one),
we might be in fact peeling the first iteration of the loop. This
in general is not a good idea. */
if (do_while_loop_p (loop))
continue;
/* Iterate the header copying up to limit; this takes care of the cases
like while (a && b) {...}, where we want to have both of the conditions
copied. TODO -- handle while (a || b) - like cases, by not requiring
the header to have just a single successor and copying up to
postdominator. */
exit = NULL;
n_bbs = 0;
while (should_duplicate_loop_header_p (header, loop, &limit))
{
/* Find a successor of header that is inside a loop; i.e. the new
header after the condition is copied. */
if (flow_bb_inside_loop_p (loop, EDGE_SUCC (header, 0)->dest))
exit = EDGE_SUCC (header, 0);
else
exit = EDGE_SUCC (header, 1);
bbs[n_bbs++] = header;
gcc_assert (bbs_size > n_bbs);
header = exit->dest;
}
if (!exit)
continue;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Duplicating header of the loop %d up to edge %d->%d.\n",
loop->num, exit->src->index, exit->dest->index);
/* Ensure that the header will have just the latch as a predecessor
inside the loop. */
if (!single_pred_p (exit->dest))
exit = single_pred_edge (loop_split_edge_with (exit, NULL));
entry = loop_preheader_edge (loop);
if (!tree_duplicate_sese_region (entry, exit, bbs, n_bbs, copied_bbs))
{
fprintf (dump_file, "Duplication failed.\n");
continue;
}
/* If the loop has the form "for (i = j; i < j + 10; i++)" then
this copying can introduce a case where we rely on undefined
signed overflow to eliminate the preheader condition, because
we assume that "j < j + 10" is true. We don't want to warn
about that case for -Wstrict-overflow, because in general we
don't warn about overflow involving loops. Prevent the
warning by setting TREE_NO_WARNING. */
if (warn_strict_overflow > 0)
{
unsigned int i;
for (i = 0; i < n_bbs; ++i)
{
tree last;
last = last_stmt (copied_bbs[i]);
if (TREE_CODE (last) == COND_EXPR)
TREE_NO_WARNING (last) = 1;
}
}
/* Ensure that the latch and the preheader is simple (we know that they
are not now, since there was the loop exit condition. */
loop_split_edge_with (loop_preheader_edge (loop), NULL);
loop_split_edge_with (loop_latch_edge (loop), NULL);
}
free (bbs);
free (copied_bbs);
loop_optimizer_finalize (loops);
return 0;
}
