static unsigned int
increase_alignment (void)
{
struct varpool_node *vnode;
/* Increase the alignment of all global arrays for vectorization. */
FOR_EACH_DEFINED_VARIABLE (vnode)
{
tree vectype, decl = vnode->symbol.decl;
tree t;
unsigned int alignment;
t = TREE_TYPE(decl);
if (TREE_CODE (t) != ARRAY_TYPE)
continue;
vectype = get_vectype_for_scalar_type (strip_array_types (t));
if (!vectype)
continue;
alignment = TYPE_ALIGN (vectype);
if (DECL_ALIGN (decl) >= alignment)
continue;
if (vect_can_force_dr_alignment_p (decl, alignment))
{
DECL_ALIGN (decl) = TYPE_ALIGN (vectype);
DECL_USER_ALIGN (decl) = 1;
dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
dump_printf (MSG_NOTE, "\n");
}
}
return 0;
}
static unsigned int
increase_alignment (void)
{
struct varpool_node *vnode;
/* Increase the alignment of all global arrays for vectorization. */
for (vnode = varpool_nodes_queue;
vnode;
vnode = vnode->next_needed)
{
tree vectype, decl = vnode->decl;
tree t;
unsigned int alignment;
t = TREE_TYPE(decl);
if (TREE_CODE (t) != ARRAY_TYPE)
continue;
vectype = get_vectype_for_scalar_type (strip_array_types (t));
if (!vectype)
continue;
alignment = TYPE_ALIGN (vectype);
if (DECL_ALIGN (decl) >= alignment)
continue;
if (vect_can_force_dr_alignment_p (decl, alignment))
{
DECL_ALIGN (decl) = TYPE_ALIGN (vectype);
DECL_USER_ALIGN (decl) = 1;
if (dump_file)
{
fprintf (dump_file, "Increasing alignment of decl: ");
print_generic_expr (dump_file, decl, TDF_SLIM);
fprintf (dump_file, "\n");
}
}
}
return 0;
}
static void
vect_pattern_recog_1 (
gimple (* vect_recog_func) (gimple, tree *, tree *),
gimple_stmt_iterator si)
{
gimple stmt = gsi_stmt (si), pattern_stmt;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info pattern_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree pattern_vectype;
tree type_in, type_out;
enum tree_code code;
int i;
gimple next;
pattern_stmt = (* vect_recog_func) (stmt, &type_in, &type_out);
if (!pattern_stmt)
return;
if (VECTOR_MODE_P (TYPE_MODE (type_in)))
{
/* No need to check target support (already checked by the pattern
recognition function). */
if (type_out)
gcc_assert (VECTOR_MODE_P (TYPE_MODE (type_out)));
pattern_vectype = type_out ? type_out : type_in;
}
else
{
enum machine_mode vec_mode;
enum insn_code icode;
optab optab;
/* Check target support */
type_in = get_vectype_for_scalar_type (type_in);
if (!type_in)
return;
if (type_out)
type_out = get_vectype_for_scalar_type (type_out);
else
type_out = type_in;
if (!type_out)
return;
pattern_vectype = type_out;
if (is_gimple_assign (pattern_stmt))
code = gimple_assign_rhs_code (pattern_stmt);
else
{
gcc_assert (is_gimple_call (pattern_stmt));
code = CALL_EXPR;
}
optab = optab_for_tree_code (code, type_in, optab_default);
vec_mode = TYPE_MODE (type_in);
if (!optab
|| (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
|| (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
return;
}
/* Found a vectorizable pattern. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "pattern recognized: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* Mark the stmts that are involved in the pattern. */
gsi_insert_before (&si, pattern_stmt, GSI_SAME_STMT);
set_vinfo_for_stmt (pattern_stmt,
new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL));
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = stmt;
STMT_VINFO_DEF_TYPE (pattern_stmt_info) = STMT_VINFO_DEF_TYPE (stmt_info);
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
STMT_VINFO_IN_PATTERN_P (stmt_info) = true;
STMT_VINFO_RELATED_STMT (stmt_info) = pattern_stmt;
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
if (next == stmt)
VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
}
static gimple
vect_recog_pow_pattern (gimple last_stmt, tree *type_in, tree *type_out)
{
tree fn, base, exp = NULL;
gimple stmt;
tree var;
if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
return NULL;
fn = gimple_call_fndecl (last_stmt);
if (fn == NULL_TREE || DECL_BUILT_IN_CLASS (fn) != BUILT_IN_NORMAL)
return NULL;
switch (DECL_FUNCTION_CODE (fn))
{
case BUILT_IN_POWIF:
case BUILT_IN_POWI:
case BUILT_IN_POWF:
case BUILT_IN_POW:
base = gimple_call_arg (last_stmt, 0);
exp = gimple_call_arg (last_stmt, 1);
if (TREE_CODE (exp) != REAL_CST
&& TREE_CODE (exp) != INTEGER_CST)
return NULL;
break;
default:
return NULL;
}
/* We now have a pow or powi builtin function call with a constant
exponent. */
*type_out = NULL_TREE;
/* Catch squaring. */
if ((host_integerp (exp, 0)
&& tree_low_cst (exp, 0) == 2)
|| (TREE_CODE (exp) == REAL_CST
&& REAL_VALUES_EQUAL (TREE_REAL_CST (exp), dconst2)))
{
*type_in = TREE_TYPE (base);
var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
stmt = gimple_build_assign_with_ops (MULT_EXPR, var, base, base);
SSA_NAME_DEF_STMT (var) = stmt;
return stmt;
}
/* Catch square root. */
if (TREE_CODE (exp) == REAL_CST
&& REAL_VALUES_EQUAL (TREE_REAL_CST (exp), dconsthalf))
{
tree newfn = mathfn_built_in (TREE_TYPE (base), BUILT_IN_SQRT);
*type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
if (*type_in)
{
gimple stmt = gimple_build_call (newfn, 1, base);
if (vectorizable_function (stmt, *type_in, *type_in)
!= NULL_TREE)
{
var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
gimple_call_set_lhs (stmt, var);
return stmt;
}
}
}
return NULL;
}
static gimple
vect_recog_widen_mult_pattern (gimple last_stmt,
tree *type_in,
tree *type_out)
{
gimple def_stmt0, def_stmt1;
tree oprnd0, oprnd1;
tree type, half_type0, half_type1;
gimple pattern_stmt;
tree vectype, vectype_out;
tree dummy;
tree var;
enum tree_code dummy_code;
int dummy_int;
VEC (tree, heap) *dummy_vec;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
/* Check argument 0 */
if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0))
return NULL;
oprnd0 = gimple_assign_rhs1 (def_stmt0);
/* Check argument 1 */
if (!widened_name_p (oprnd1, last_stmt, &half_type1, &def_stmt1))
return NULL;
oprnd1 = gimple_assign_rhs1 (def_stmt1);
if (!types_compatible_p (half_type0, half_type1))
return NULL;
/* Pattern detected. */
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "vect_recog_widen_mult_pattern: detected: ");
/* Check target support */
vectype = get_vectype_for_scalar_type (half_type0);
vectype_out = get_vectype_for_scalar_type (type);
if (!vectype
|| !vectype_out
|| !supportable_widening_operation (WIDEN_MULT_EXPR, last_stmt,
vectype_out, vectype,
&dummy, &dummy, &dummy_code,
&dummy_code, &dummy_int, &dummy_vec))
return NULL;
*type_in = vectype;
*type_out = vectype_out;
/* Pattern supported. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops (WIDEN_MULT_EXPR, var, oprnd0,
oprnd1);
SSA_NAME_DEF_STMT (var) = pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
return pattern_stmt;
}
static void
vect_pattern_recog_1 (
gimple (* vect_recog_func) (gimple, tree *, tree *),
gimple_stmt_iterator si)
{
gimple stmt = gsi_stmt (si), pattern_stmt;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info pattern_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree pattern_vectype;
tree type_in, type_out;
enum tree_code code;
pattern_stmt = (* vect_recog_func) (stmt, &type_in, &type_out);
if (!pattern_stmt)
return;
if (VECTOR_MODE_P (TYPE_MODE (type_in)))
{
/* No need to check target support (already checked by the pattern
recognition function). */
pattern_vectype = type_in;
}
else
{
enum tree_code vec_mode;
enum insn_code icode;
optab optab;
/* Check target support */
pattern_vectype = get_vectype_for_scalar_type (type_in);
if (!pattern_vectype)
return;
if (is_gimple_assign (pattern_stmt))
code = gimple_assign_rhs_code (pattern_stmt);
else
{
gcc_assert (is_gimple_call (pattern_stmt));
code = CALL_EXPR;
}
optab = optab_for_tree_code (code, pattern_vectype, optab_default);
vec_mode = TYPE_MODE (pattern_vectype);
if (!optab
|| (icode = optab_handler (optab, vec_mode)->insn_code) ==
CODE_FOR_nothing
|| (type_out
&& (!get_vectype_for_scalar_type (type_out)
|| (insn_data[icode].operand[0].mode !=
TYPE_MODE (get_vectype_for_scalar_type (type_out))))))
return;
}
/* Found a vectorizable pattern. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "pattern recognized: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* Mark the stmts that are involved in the pattern. */
gsi_insert_before (&si, pattern_stmt, GSI_SAME_STMT);
set_vinfo_for_stmt (pattern_stmt,
new_stmt_vec_info (pattern_stmt, loop_vinfo));
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = stmt;
STMT_VINFO_DEF_TYPE (pattern_stmt_info) = STMT_VINFO_DEF_TYPE (stmt_info);
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
STMT_VINFO_IN_PATTERN_P (stmt_info) = true;
STMT_VINFO_RELATED_STMT (stmt_info) = pattern_stmt;
return;
}
