static inline tree
chrec_fold_multiply_poly_poly (tree type,
tree poly0,
tree poly1)
{
tree t0, t1, t2;
int var;
struct loop *loop0 = get_chrec_loop (poly0);
struct loop *loop1 = get_chrec_loop (poly1);
gcc_assert (poly0);
gcc_assert (poly1);
gcc_assert (TREE_CODE (poly0) == POLYNOMIAL_CHREC);
gcc_assert (TREE_CODE (poly1) == POLYNOMIAL_CHREC);
gcc_checking_assert (useless_type_conversion_p (type, chrec_type (poly0))
&& useless_type_conversion_p (type, chrec_type (poly1)));
/* {a, +, b}_1 * {c, +, d}_2 -> {c*{a, +, b}_1, +, d}_2,
{a, +, b}_2 * {c, +, d}_1 -> {a*{c, +, d}_1, +, b}_2,
{a, +, b}_x * {c, +, d}_x -> {a*c, +, a*d + b*c + b*d, +, 2*b*d}_x. */
if (flow_loop_nested_p (loop0, loop1))
/* poly0 is a constant wrt. poly1. */
return build_polynomial_chrec
(CHREC_VARIABLE (poly1),
chrec_fold_multiply (type, CHREC_LEFT (poly1), poly0),
CHREC_RIGHT (poly1));
if (flow_loop_nested_p (loop1, loop0))
/* poly1 is a constant wrt. poly0. */
return build_polynomial_chrec
(CHREC_VARIABLE (poly0),
chrec_fold_multiply (type, CHREC_LEFT (poly0), poly1),
CHREC_RIGHT (poly0));
gcc_assert (loop0 == loop1);
/* poly0 and poly1 are two polynomials in the same variable,
{a, +, b}_x * {c, +, d}_x -> {a*c, +, a*d + b*c + b*d, +, 2*b*d}_x. */
/* "a*c". */
t0 = chrec_fold_multiply (type, CHREC_LEFT (poly0), CHREC_LEFT (poly1));
/* "a*d + b*c". */
t1 = chrec_fold_multiply (type, CHREC_LEFT (poly0), CHREC_RIGHT (poly1));
t1 = chrec_fold_plus (type, t1, chrec_fold_multiply (type,
CHREC_RIGHT (poly0),
CHREC_LEFT (poly1)));
/* "b*d". */
t2 = chrec_fold_multiply (type, CHREC_RIGHT (poly0), CHREC_RIGHT (poly1));
/* "a*d + b*c + b*d". */
t1 = chrec_fold_plus (type, t1, t2);
/* "2*b*d". */
t2 = chrec_fold_multiply (type, SCALAR_FLOAT_TYPE_P (type)
? build_real (type, dconst2)
: build_int_cst (type, 2), t2);
var = CHREC_VARIABLE (poly0);
return build_polynomial_chrec (var, t0,
build_polynomial_chrec (var, t1, t2));
}
tree
hide_evolution_in_other_loops_than_loop (tree chrec,
unsigned loop_num)
{
struct loop *loop = get_loop (cfun, loop_num), *chloop;
if (automatically_generated_chrec_p (chrec))
return chrec;
switch (TREE_CODE (chrec))
{
case POLYNOMIAL_CHREC:
chloop = get_chrec_loop (chrec);
if (chloop == loop)
return build_polynomial_chrec
(loop_num,
hide_evolution_in_other_loops_than_loop (CHREC_LEFT (chrec),
loop_num),
CHREC_RIGHT (chrec));
else if (flow_loop_nested_p (chloop, loop))
/* There is no evolution in this loop. */
return initial_condition (chrec);
else
{
gcc_assert (flow_loop_nested_p (loop, chloop));
return hide_evolution_in_other_loops_than_loop (CHREC_LEFT (chrec),
loop_num);
}
default:
return chrec;
}
}
static tree
chrec_component_in_loop_num (tree chrec,
unsigned loop_num,
bool right)
{
tree component;
struct loop *loop = get_loop (cfun, loop_num), *chloop;
if (automatically_generated_chrec_p (chrec))
return chrec;
switch (TREE_CODE (chrec))
{
case POLYNOMIAL_CHREC:
chloop = get_chrec_loop (chrec);
if (chloop == loop)
{
if (right)
component = CHREC_RIGHT (chrec);
else
component = CHREC_LEFT (chrec);
if (TREE_CODE (CHREC_LEFT (chrec)) != POLYNOMIAL_CHREC
|| CHREC_VARIABLE (CHREC_LEFT (chrec)) != CHREC_VARIABLE (chrec))
return component;
else
return build_polynomial_chrec
(loop_num,
chrec_component_in_loop_num (CHREC_LEFT (chrec),
loop_num,
right),
component);
}
else if (flow_loop_nested_p (chloop, loop))
/* There is no evolution part in this loop. */
return NULL_TREE;
else
{
gcc_assert (flow_loop_nested_p (loop, chloop));
return chrec_component_in_loop_num (CHREC_LEFT (chrec),
loop_num,
right);
}
default:
if (right)
return NULL_TREE;
else
return chrec;
}
}
tree
reset_evolution_in_loop (unsigned loop_num,
tree chrec,
tree new_evol)
{
struct loop *loop = get_loop (cfun, loop_num);
if (POINTER_TYPE_P (chrec_type (chrec)))
gcc_assert (ptrofftype_p (chrec_type (new_evol)));
else
gcc_assert (chrec_type (chrec) == chrec_type (new_evol));
if (TREE_CODE (chrec) == POLYNOMIAL_CHREC
&& flow_loop_nested_p (loop, get_chrec_loop (chrec)))
{
tree left = reset_evolution_in_loop (loop_num, CHREC_LEFT (chrec),
new_evol);
tree right = reset_evolution_in_loop (loop_num, CHREC_RIGHT (chrec),
new_evol);
return build3 (POLYNOMIAL_CHREC, TREE_TYPE (left),
CHREC_VAR (chrec), left, right);
}
while (TREE_CODE (chrec) == POLYNOMIAL_CHREC
&& CHREC_VARIABLE (chrec) == loop_num)
chrec = CHREC_LEFT (chrec);
return build_polynomial_chrec (loop_num, chrec, new_evol);
}
static tree
chrec_evaluate (unsigned var, tree chrec, tree n, unsigned int k)
{
tree arg0, arg1, binomial_n_k;
tree type = TREE_TYPE (chrec);
struct loop *var_loop = get_loop (cfun, var);
while (TREE_CODE (chrec) == POLYNOMIAL_CHREC
&& flow_loop_nested_p (var_loop, get_chrec_loop (chrec)))
chrec = CHREC_LEFT (chrec);
if (TREE_CODE (chrec) == POLYNOMIAL_CHREC
&& CHREC_VARIABLE (chrec) == var)
{
arg1 = chrec_evaluate (var, CHREC_RIGHT (chrec), n, k + 1);
if (arg1 == chrec_dont_know)
return chrec_dont_know;
binomial_n_k = tree_fold_binomial (type, n, k);
if (!binomial_n_k)
return chrec_dont_know;
arg0 = fold_build2 (MULT_EXPR, type,
CHREC_LEFT (chrec), binomial_n_k);
return chrec_fold_plus (type, arg0, arg1);
}
binomial_n_k = tree_fold_binomial (type, n, k);
if (!binomial_n_k)
return chrec_dont_know;
return fold_build2 (MULT_EXPR, type, chrec, binomial_n_k);
}
static struct loop *
outermost_invariant_loop_expr (tree expr, struct loop *loop)
{
enum tree_code_class codeclass = TREE_CODE_CLASS (TREE_CODE (expr));
unsigned i, nops;
struct loop *max_loop = superloop_at_depth (loop, 1), *aloop;
if (TREE_CODE (expr) == SSA_NAME
|| TREE_CODE (expr) == INTEGER_CST
|| is_gimple_min_invariant (expr))
return outermost_invariant_loop (expr, loop);
if (codeclass != tcc_unary
&& codeclass != tcc_binary
&& codeclass != tcc_expression
&& codeclass != tcc_vl_exp
&& codeclass != tcc_comparison)
return NULL;
nops = TREE_OPERAND_LENGTH (expr);
for (i = 0; i < nops; i++)
{
aloop = outermost_invariant_loop_expr (TREE_OPERAND (expr, i), loop);
if (!aloop)
return NULL;
if (flow_loop_nested_p (max_loop, aloop))
max_loop = aloop;
}
return max_loop;
}
/* Return nonzero if basic block BB belongs to LOOP. */
bool
flow_bb_inside_loop_p (const struct loop *loop, const basic_block bb)
{
struct loop *source_loop;
if (bb == ENTRY_BLOCK_PTR || bb == EXIT_BLOCK_PTR)
return 0;
source_loop = bb->loop_father;
return loop == source_loop || flow_loop_nested_p (loop, source_loop);
}
static void
set_level (tree stmt, struct loop *orig_loop, struct loop *level)
{
struct loop *stmt_loop = bb_for_stmt (stmt)->loop_father;
struct depend *dep;
stmt_loop = find_common_loop (orig_loop, stmt_loop);
if (LIM_DATA (stmt) && LIM_DATA (stmt)->tgt_loop)
stmt_loop = find_common_loop (stmt_loop,
loop_outer (LIM_DATA (stmt)->tgt_loop));
if (flow_loop_nested_p (stmt_loop, level))
return;
gcc_assert (LIM_DATA (stmt));
gcc_assert (level == LIM_DATA (stmt)->max_loop
|| flow_loop_nested_p (LIM_DATA (stmt)->max_loop, level));
LIM_DATA (stmt)->tgt_loop = level;
for (dep = LIM_DATA (stmt)->depends; dep; dep = dep->next)
set_level (dep->stmt, orig_loop, level);
}
static bool
evolution_function_is_invariant_rec_p (tree chrec, int loopnum)
{
if (evolution_function_is_constant_p (chrec))
return true;
if (TREE_CODE (chrec) == SSA_NAME
&& (loopnum == 0
|| expr_invariant_in_loop_p (get_loop (cfun, loopnum), chrec)))
return true;
if (TREE_CODE (chrec) == POLYNOMIAL_CHREC)
{
if (CHREC_VARIABLE (chrec) == (unsigned) loopnum
|| flow_loop_nested_p (get_loop (cfun, loopnum),
get_chrec_loop (chrec))
|| !evolution_function_is_invariant_rec_p (CHREC_RIGHT (chrec),
loopnum)
|| !evolution_function_is_invariant_rec_p (CHREC_LEFT (chrec),
loopnum))
return false;
return true;
}
switch (TREE_OPERAND_LENGTH (chrec))
{
case 2:
if (!evolution_function_is_invariant_rec_p (TREE_OPERAND (chrec, 1),
loopnum))
return false;
case 1:
if (!evolution_function_is_invariant_rec_p (TREE_OPERAND (chrec, 0),
loopnum))
return false;
return true;
default:
return false;
}
return false;
}
static bool
add_dependency (tree def, struct lim_aux_data *data, struct loop *loop,
bool add_cost)
{
tree def_stmt = SSA_NAME_DEF_STMT (def);
basic_block def_bb = bb_for_stmt (def_stmt);
struct loop *max_loop;
struct depend *dep;
if (!def_bb)
return true;
max_loop = outermost_invariant_loop (def, loop);
if (!max_loop)
return false;
if (flow_loop_nested_p (data->max_loop, max_loop))
data->max_loop = max_loop;
if (!LIM_DATA (def_stmt))
return true;
if (add_cost
/* Only add the cost if the statement defining DEF is inside LOOP,
i.e. if it is likely that by moving the invariants dependent
on it, we will be able to avoid creating a new register for
it (since it will be only used in these dependent invariants). */
&& def_bb->loop_father == loop)
data->cost += LIM_DATA (def_stmt)->cost;
dep = XNEW (struct depend);
dep->stmt = def_stmt;
dep->next = data->depends;
data->depends = dep;
return true;
}
static inline tree
chrec_fold_plus_poly_poly (enum tree_code code,
tree type,
tree poly0,
tree poly1)
{
tree left, right;
struct loop *loop0 = get_chrec_loop (poly0);
struct loop *loop1 = get_chrec_loop (poly1);
tree rtype = code == POINTER_PLUS_EXPR ? chrec_type (poly1) : type;
gcc_assert (poly0);
gcc_assert (poly1);
gcc_assert (TREE_CODE (poly0) == POLYNOMIAL_CHREC);
gcc_assert (TREE_CODE (poly1) == POLYNOMIAL_CHREC);
if (POINTER_TYPE_P (chrec_type (poly0)))
gcc_checking_assert (ptrofftype_p (chrec_type (poly1))
&& useless_type_conversion_p (type, chrec_type (poly0)));
else
gcc_checking_assert (useless_type_conversion_p (type, chrec_type (poly0))
&& useless_type_conversion_p (type, chrec_type (poly1)));
/*
{a, +, b}_1 + {c, +, d}_2 -> {{a, +, b}_1 + c, +, d}_2,
{a, +, b}_2 + {c, +, d}_1 -> {{c, +, d}_1 + a, +, b}_2,
{a, +, b}_x + {c, +, d}_x -> {a+c, +, b+d}_x. */
if (flow_loop_nested_p (loop0, loop1))
{
if (code == PLUS_EXPR || code == POINTER_PLUS_EXPR)
return build_polynomial_chrec
(CHREC_VARIABLE (poly1),
chrec_fold_plus (type, poly0, CHREC_LEFT (poly1)),
CHREC_RIGHT (poly1));
else
return build_polynomial_chrec
(CHREC_VARIABLE (poly1),
chrec_fold_minus (type, poly0, CHREC_LEFT (poly1)),
chrec_fold_multiply (type, CHREC_RIGHT (poly1),
SCALAR_FLOAT_TYPE_P (type)
? build_real (type, dconstm1)
: build_int_cst_type (type, -1)));
}
if (flow_loop_nested_p (loop1, loop0))
{
if (code == PLUS_EXPR || code == POINTER_PLUS_EXPR)
return build_polynomial_chrec
(CHREC_VARIABLE (poly0),
chrec_fold_plus (type, CHREC_LEFT (poly0), poly1),
CHREC_RIGHT (poly0));
else
return build_polynomial_chrec
(CHREC_VARIABLE (poly0),
chrec_fold_minus (type, CHREC_LEFT (poly0), poly1),
CHREC_RIGHT (poly0));
}
/* This function should never be called for chrecs of loops that
do not belong to the same loop nest. */
gcc_assert (loop0 == loop1);
if (code == PLUS_EXPR || code == POINTER_PLUS_EXPR)
{
left = chrec_fold_plus
(type, CHREC_LEFT (poly0), CHREC_LEFT (poly1));
right = chrec_fold_plus
(rtype, CHREC_RIGHT (poly0), CHREC_RIGHT (poly1));
}
else
{
left = chrec_fold_minus
(type, CHREC_LEFT (poly0), CHREC_LEFT (poly1));
right = chrec_fold_minus
(type, CHREC_RIGHT (poly0), CHREC_RIGHT (poly1));
}
if (chrec_zerop (right))
return left;
else
return build_polynomial_chrec
(CHREC_VARIABLE (poly0), left, right);
}
static void
gather_interchange_stats (varray_type dependence_relations,
varray_type datarefs,
struct loop *loop,
struct loop *first_loop,
unsigned int *dependence_steps,
unsigned int *nb_deps_not_carried_by_loop,
unsigned int *access_strides)
{
unsigned int i;
*dependence_steps = 0;
*nb_deps_not_carried_by_loop = 0;
*access_strides = 0;
for (i = 0; i < VARRAY_ACTIVE_SIZE (dependence_relations); i++)
{
int dist;
struct data_dependence_relation *ddr =
(struct data_dependence_relation *)
VARRAY_GENERIC_PTR (dependence_relations, i);
/* If we don't know anything about this dependence, or the distance
vector is NULL, or there is no dependence, then there is no reuse of
data. */
if (DDR_DIST_VECT (ddr) == NULL
|| DDR_ARE_DEPENDENT (ddr) == chrec_dont_know
|| DDR_ARE_DEPENDENT (ddr) == chrec_known)
continue;
dist = DDR_DIST_VECT (ddr)[loop->depth - first_loop->depth];
if (dist == 0)
(*nb_deps_not_carried_by_loop) += 1;
else if (dist < 0)
(*dependence_steps) += -dist;
else
(*dependence_steps) += dist;
}
/* Compute the access strides. */
for (i = 0; i < VARRAY_ACTIVE_SIZE (datarefs); i++)
{
unsigned int it;
struct data_reference *dr = VARRAY_GENERIC_PTR (datarefs, i);
tree stmt = DR_STMT (dr);
struct loop *stmt_loop = loop_containing_stmt (stmt);
struct loop *inner_loop = first_loop->inner;
if (inner_loop != stmt_loop
&& !flow_loop_nested_p (inner_loop, stmt_loop))
continue;
for (it = 0; it < DR_NUM_DIMENSIONS (dr); it++)
{
tree chrec = DR_ACCESS_FN (dr, it);
tree tstride = evolution_part_in_loop_num
(chrec, loop->num);
if (tstride == NULL_TREE
|| TREE_CODE (tstride) != INTEGER_CST)
continue;
(*access_strides) += int_cst_value (tstride);
}
}
}