static bool
stmt_has_simple_data_refs_p (loop_p outermost_loop ATTRIBUTE_UNUSED,
gimple stmt)
{
data_reference_p dr;
unsigned i;
int j;
bool res = true;
vec<data_reference_p> drs = vNULL;
loop_p outer;
for (outer = loop_containing_stmt (stmt); outer; outer = loop_outer (outer))
{
graphite_find_data_references_in_stmt (outer,
loop_containing_stmt (stmt),
stmt, &drs);
FOR_EACH_VEC_ELT (drs, j, dr)
for (i = 0; i < DR_NUM_DIMENSIONS (dr); i++)
if (!graphite_can_represent_scev (DR_ACCESS_FN (dr, i)))
{
res = false;
goto done;
}
free_data_refs (drs);
drs.create (0);
}
done:
free_data_refs (drs);
return res;
}
static struct loop *
outermost_invariant_loop (tree def, struct loop *loop)
{
tree def_stmt;
basic_block def_bb;
struct loop *max_loop;
if (TREE_CODE (def) != SSA_NAME)
return superloop_at_depth (loop, 1);
def_stmt = SSA_NAME_DEF_STMT (def);
def_bb = bb_for_stmt (def_stmt);
if (!def_bb)
return superloop_at_depth (loop, 1);
max_loop = find_common_loop (loop, def_bb->loop_father);
if (LIM_DATA (def_stmt) && LIM_DATA (def_stmt)->max_loop)
max_loop = find_common_loop (max_loop,
loop_outer (LIM_DATA (def_stmt)->max_loop));
if (max_loop == loop)
return NULL;
max_loop = superloop_at_depth (loop, loop_depth (max_loop) + 1);
return max_loop;
}
static void
set_level (tree stmt, struct loop *orig_loop, struct loop *level)
{
struct loop *stmt_loop = bb_for_stmt (stmt)->loop_father;
struct depend *dep;
stmt_loop = find_common_loop (orig_loop, stmt_loop);
if (LIM_DATA (stmt) && LIM_DATA (stmt)->tgt_loop)
stmt_loop = find_common_loop (stmt_loop,
loop_outer (LIM_DATA (stmt)->tgt_loop));
if (flow_loop_nested_p (stmt_loop, level))
return;
gcc_assert (LIM_DATA (stmt));
gcc_assert (level == LIM_DATA (stmt)->max_loop
|| flow_loop_nested_p (LIM_DATA (stmt)->max_loop, level));
LIM_DATA (stmt)->tgt_loop = level;
for (dep = LIM_DATA (stmt)->depends; dep; dep = dep->next)
set_level (dep->stmt, orig_loop, level);
}
static bool
stmt_has_simple_data_refs_p (loop_p outermost_loop ATTRIBUTE_UNUSED,
gimple *stmt)
{
data_reference_p dr;
int j;
bool res = true;
vec<data_reference_p> drs = vNULL;
loop_p outer;
for (outer = loop_containing_stmt (stmt); outer; outer = loop_outer (outer))
{
graphite_find_data_references_in_stmt (outer,
loop_containing_stmt (stmt),
stmt, &drs);
FOR_EACH_VEC_ELT (drs, j, dr)
{
int nb_subscripts = DR_NUM_DIMENSIONS (dr);
tree ref = DR_REF (dr);
for (int i = nb_subscripts - 1; i >= 0; i--)
{
if (!graphite_can_represent_scev (DR_ACCESS_FN (dr, i))
|| (TREE_CODE (ref) != ARRAY_REF
&& TREE_CODE (ref) != MEM_REF
&& TREE_CODE (ref) != COMPONENT_REF))
{
free_data_refs (drs);
return false;
}
ref = TREE_OPERAND (ref, 0);
}
}
free_data_refs (drs);
drs.create (0);
}
unsigned
fix_loop_structure (bitmap changed_bbs)
{
basic_block bb;
int record_exits = 0;
struct loop *loop;
unsigned old_nloops, i;
timevar_push (TV_LOOP_INIT);
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "fix_loop_structure: fixing up loops for function\n");
/* We need exact and fast dominance info to be available. */
gcc_assert (dom_info_state (CDI_DOMINATORS) == DOM_OK);
if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
{
release_recorded_exits (cfun);
record_exits = LOOPS_HAVE_RECORDED_EXITS;
}
/* Remember the depth of the blocks in the loop hierarchy, so that we can
recognize blocks whose loop nesting relationship has changed. */
if (changed_bbs)
FOR_EACH_BB_FN (bb, cfun)
bb->aux = (void *) (size_t) loop_depth (bb->loop_father);
/* Remove the dead loops from structures. We start from the innermost
loops, so that when we remove the loops, we know that the loops inside
are preserved, and do not waste time relinking loops that will be
removed later. */
FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
{
/* Detect the case that the loop is no longer present even though
it wasn't marked for removal.
??? If we do that we can get away with not marking loops for
removal at all. And possibly avoid some spurious removals. */
if (loop->header
&& bb_loop_header_p (loop->header))
continue;
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "fix_loop_structure: removing loop %d\n",
loop->num);
while (loop->inner)
{
struct loop *ploop = loop->inner;
flow_loop_tree_node_remove (ploop);
flow_loop_tree_node_add (loop_outer (loop), ploop);
}
/* Remove the loop. */
if (loop->header)
loop->former_header = loop->header;
else
gcc_assert (loop->former_header != NULL);
loop->header = NULL;
flow_loop_tree_node_remove (loop);
}
/* Remember the number of loops so we can return how many new loops
flow_loops_find discovered. */
old_nloops = number_of_loops (cfun);
/* Re-compute loop structure in-place. */
flow_loops_find (current_loops);
/* Mark the blocks whose loop has changed. */
if (changed_bbs)
{
FOR_EACH_BB_FN (bb, cfun)
{
if ((void *) (size_t) loop_depth (bb->loop_father) != bb->aux)
bitmap_set_bit (changed_bbs, bb->index);
bb->aux = NULL;
}
}
/* Finally free deleted loops. */
bool any_deleted = false;
FOR_EACH_VEC_ELT (*get_loops (cfun), i, loop)
if (loop && loop->header == NULL)
{
if (dump_file
&& ((unsigned) loop->former_header->index
< basic_block_info_for_fn (cfun)->length ()))
{
basic_block former_header
= BASIC_BLOCK_FOR_FN (cfun, loop->former_header->index);
/* If the old header still exists we want to check if the
original loop is re-discovered or the old header is now
part of a newly discovered loop.
In both cases we should have avoided removing the loop. */
if (former_header == loop->former_header)
{
if (former_header->loop_father->header == former_header)
fprintf (dump_file, "fix_loop_structure: rediscovered "
"removed loop %d as loop %d with old header %d\n",
loop->num, former_header->loop_father->num,
former_header->index);
else if ((unsigned) former_header->loop_father->num
>= old_nloops)
fprintf (dump_file, "fix_loop_structure: header %d of "
"removed loop %d is part of the newly "
"discovered loop %d with header %d\n",
former_header->index, loop->num,
former_header->loop_father->num,
former_header->loop_father->header->index);
}
}
(*get_loops (cfun))[i] = NULL;
flow_loop_free (loop);
any_deleted = true;
}
/* If we deleted loops then the cached scalar evolutions refering to
those loops become invalid. */
if (any_deleted && scev_initialized_p ())
scev_reset_htab ();
loops_state_clear (LOOPS_NEED_FIXUP);
/* Apply flags to loops. */
apply_loop_flags (current_loops->state | record_exits);
checking_verify_loop_structure ();
timevar_pop (TV_LOOP_INIT);
return number_of_loops (cfun) - old_nloops;
}
static void
determine_loop_nest_reuse (struct loop *loop, struct mem_ref_group *refs,
bool no_other_refs)
{
struct loop *nest, *aloop;
VEC (data_reference_p, heap) *datarefs = NULL;
VEC (ddr_p, heap) *dependences = NULL;
struct mem_ref_group *gr;
struct mem_ref *ref, *refb;
VEC (loop_p, heap) *vloops = NULL;
unsigned *loop_data_size;
unsigned i, j, n;
unsigned volume, dist, adist;
HOST_WIDE_INT vol;
data_reference_p dr;
ddr_p dep;
if (loop->inner)
return;
/* Find the outermost loop of the loop nest of loop (we require that
there are no sibling loops inside the nest). */
nest = loop;
while (1)
{
aloop = loop_outer (nest);
if (aloop == current_loops->tree_root
|| aloop->inner->next)
break;
nest = aloop;
}
/* For each loop, determine the amount of data accessed in each iteration.
We use this to estimate whether the reference is evicted from the
cache before its reuse. */
find_loop_nest (nest, &vloops);
n = VEC_length (loop_p, vloops);
loop_data_size = XNEWVEC (unsigned, n);
volume = volume_of_references (refs);
i = n;
while (i-- != 0)
{
loop_data_size[i] = volume;
/* Bound the volume by the L2 cache size, since above this bound,
all dependence distances are equivalent. */
if (volume > L2_CACHE_SIZE_BYTES)
continue;
aloop = VEC_index (loop_p, vloops, i);
vol = estimated_loop_iterations_int (aloop, false);
if (vol < 0)
vol = expected_loop_iterations (aloop);
volume *= vol;
}
/* Prepare the references in the form suitable for data dependence
analysis. We ignore unanalyzable data references (the results
are used just as a heuristics to estimate temporality of the
references, hence we do not need to worry about correctness). */
for (gr = refs; gr; gr = gr->next)
for (ref = gr->refs; ref; ref = ref->next)
{
dr = create_data_ref (nest, ref->mem, ref->stmt, !ref->write_p);
if (dr)
{
ref->reuse_distance = volume;
dr->aux = ref;
VEC_safe_push (data_reference_p, heap, datarefs, dr);
}
else
no_other_refs = false;
}
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
{
dist = self_reuse_distance (dr, loop_data_size, n, loop);
ref = (struct mem_ref *) dr->aux;
if (ref->reuse_distance > dist)
ref->reuse_distance = dist;
if (no_other_refs)
ref->independent_p = true;
}
compute_all_dependences (datarefs, &dependences, vloops, true);
for (i = 0; VEC_iterate (ddr_p, dependences, i, dep); i++)
{
if (DDR_ARE_DEPENDENT (dep) == chrec_known)
continue;
ref = (struct mem_ref *) DDR_A (dep)->aux;
refb = (struct mem_ref *) DDR_B (dep)->aux;
if (DDR_ARE_DEPENDENT (dep) == chrec_dont_know
|| DDR_NUM_DIST_VECTS (dep) == 0)
{
/* If the dependence cannot be analyzed, assume that there might be
a reuse. */
dist = 0;
ref->independent_p = false;
refb->independent_p = false;
}
else
{
/* The distance vectors are normalized to be always lexicographically
positive, hence we cannot tell just from them whether DDR_A comes
before DDR_B or vice versa. However, it is not important,
anyway -- if DDR_A is close to DDR_B, then it is either reused in
DDR_B (and it is not nontemporal), or it reuses the value of DDR_B
in cache (and marking it as nontemporal would not affect
anything). */
dist = volume;
for (j = 0; j < DDR_NUM_DIST_VECTS (dep); j++)
{
adist = volume_of_dist_vector (DDR_DIST_VECT (dep, j),
loop_data_size, n);
/* If this is a dependence in the innermost loop (i.e., the
distances in all superloops are zero) and it is not
the trivial self-dependence with distance zero, record that
the references are not completely independent. */
if (lambda_vector_zerop (DDR_DIST_VECT (dep, j), n - 1)
&& (ref != refb
|| DDR_DIST_VECT (dep, j)[n-1] != 0))
{
ref->independent_p = false;
refb->independent_p = false;
}
/* Ignore accesses closer than
L1_CACHE_SIZE_BYTES / NONTEMPORAL_FRACTION,
so that we use nontemporal prefetches e.g. if single memory
location is accessed several times in a single iteration of
the loop. */
if (adist < L1_CACHE_SIZE_BYTES / NONTEMPORAL_FRACTION)
continue;
if (adist < dist)
dist = adist;
}
}
if (ref->reuse_distance > dist)
ref->reuse_distance = dist;
if (refb->reuse_distance > dist)
refb->reuse_distance = dist;
}
free_dependence_relations (dependences);
free_data_refs (datarefs);
free (loop_data_size);
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Reuse distances:\n");
for (gr = refs; gr; gr = gr->next)
for (ref = gr->refs; ref; ref = ref->next)
fprintf (dump_file, " ref %p distance %u\n",
(void *) ref, ref->reuse_distance);
}
}
