static bool
widened_name_p (tree name, tree use_stmt, tree *half_type, tree *def_stmt)
{
tree dummy;
loop_vec_info loop_vinfo;
stmt_vec_info stmt_vinfo;
tree expr;
tree type = TREE_TYPE (name);
tree oprnd0;
enum vect_def_type dt;
tree def;
stmt_vinfo = vinfo_for_stmt (use_stmt);
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
if (!vect_is_simple_use (name, loop_vinfo, def_stmt, &def, &dt))
return false;
if (dt != vect_loop_def
&& dt != vect_invariant_def && dt != vect_constant_def)
return false;
if (! *def_stmt)
return false;
if (TREE_CODE (*def_stmt) != MODIFY_EXPR)
return false;
expr = TREE_OPERAND (*def_stmt, 1);
if (TREE_CODE (expr) != NOP_EXPR)
return false;
oprnd0 = TREE_OPERAND (expr, 0);
*half_type = TREE_TYPE (oprnd0);
if (!INTEGRAL_TYPE_P (type) || !INTEGRAL_TYPE_P (*half_type)
|| (TYPE_UNSIGNED (type) != TYPE_UNSIGNED (*half_type))
|| (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 2)))
return false;
if (!vect_is_simple_use (oprnd0, loop_vinfo, &dummy, &dummy, &dt))
return false;
if (dt != vect_invariant_def && dt != vect_constant_def
&& dt != vect_loop_def)
return false;
return true;
}
static bool
widened_name_p (tree name, gimple use_stmt, tree *half_type, gimple *def_stmt,
bool check_sign)
{
tree dummy;
gimple dummy_gimple;
loop_vec_info loop_vinfo;
stmt_vec_info stmt_vinfo;
tree type = TREE_TYPE (name);
tree oprnd0;
enum vect_def_type dt;
tree def;
stmt_vinfo = vinfo_for_stmt (use_stmt);
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
if (!vect_is_simple_use (name, loop_vinfo, NULL, def_stmt, &def, &dt))
return false;
if (dt != vect_internal_def
&& dt != vect_external_def && dt != vect_constant_def)
return false;
if (! *def_stmt)
return false;
if (!is_gimple_assign (*def_stmt))
return false;
if (gimple_assign_rhs_code (*def_stmt) != NOP_EXPR)
return false;
oprnd0 = gimple_assign_rhs1 (*def_stmt);
*half_type = TREE_TYPE (oprnd0);
if (!INTEGRAL_TYPE_P (type) || !INTEGRAL_TYPE_P (*half_type)
|| ((TYPE_UNSIGNED (type) != TYPE_UNSIGNED (*half_type)) && check_sign)
|| (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 2)))
return false;
if (!vect_is_simple_use (oprnd0, loop_vinfo, NULL, &dummy_gimple, &dummy,
&dt))
return false;
return true;
}
Note: The dot-prod idiom is a widening reduction pattern that is
vectorized without preserving all the intermediate results. It
produces only N/2 (widened) results (by summing up pairs of
intermediate results) rather than all N results. Therefore, we
cannot allow this pattern when we want to get all the results and in
the correct order (as is the case when this computation is in an
inner-loop nested in an outer-loop that us being vectorized). */
static gimple
vect_recog_dot_prod_pattern (VEC (gimple, heap) **stmts, tree *type_in,
tree *type_out)
{
gimple stmt, last_stmt = VEC_index (gimple, *stmts, 0);
tree oprnd0, oprnd1;
tree oprnd00, oprnd01;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
gimple pattern_stmt;
tree prod_type;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
tree var;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Look for the following pattern
DX = (TYPE1) X;
DY = (TYPE1) Y;
static void
vect_pattern_recog_1 (
gimple (* vect_recog_func) (gimple, tree *, tree *),
gimple_stmt_iterator si)
{
gimple stmt = gsi_stmt (si), pattern_stmt;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info pattern_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree pattern_vectype;
tree type_in, type_out;
enum tree_code code;
int i;
gimple next;
pattern_stmt = (* vect_recog_func) (stmt, &type_in, &type_out);
if (!pattern_stmt)
return;
if (VECTOR_MODE_P (TYPE_MODE (type_in)))
{
/* No need to check target support (already checked by the pattern
recognition function). */
if (type_out)
gcc_assert (VECTOR_MODE_P (TYPE_MODE (type_out)));
pattern_vectype = type_out ? type_out : type_in;
}
else
{
enum machine_mode vec_mode;
enum insn_code icode;
optab optab;
/* Check target support */
type_in = get_vectype_for_scalar_type (type_in);
if (!type_in)
return;
if (type_out)
type_out = get_vectype_for_scalar_type (type_out);
else
type_out = type_in;
if (!type_out)
return;
pattern_vectype = type_out;
if (is_gimple_assign (pattern_stmt))
code = gimple_assign_rhs_code (pattern_stmt);
else
{
gcc_assert (is_gimple_call (pattern_stmt));
code = CALL_EXPR;
}
optab = optab_for_tree_code (code, type_in, optab_default);
vec_mode = TYPE_MODE (type_in);
if (!optab
|| (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
|| (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
return;
}
/* Found a vectorizable pattern. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "pattern recognized: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* Mark the stmts that are involved in the pattern. */
gsi_insert_before (&si, pattern_stmt, GSI_SAME_STMT);
set_vinfo_for_stmt (pattern_stmt,
new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL));
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = stmt;
STMT_VINFO_DEF_TYPE (pattern_stmt_info) = STMT_VINFO_DEF_TYPE (stmt_info);
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
STMT_VINFO_IN_PATTERN_P (stmt_info) = true;
STMT_VINFO_RELATED_STMT (stmt_info) = pattern_stmt;
/* Patterns cannot be vectorized using SLP, because they change the order of
computation. */
FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
if (next == stmt)
VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
}
static gimple
vect_recog_widen_sum_pattern (gimple last_stmt, tree *type_in, tree *type_out)
{
gimple stmt;
tree oprnd0, oprnd1;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
gimple pattern_stmt;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
tree var;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Look for the following pattern
DX = (TYPE) X;
sum_1 = DX + sum_0;
In which DX is at least double the size of X, and sum_1 has been
recognized as a reduction variable.
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a cast from type 'type' to type
'TYPE'. */
if (!widened_name_p (oprnd0, last_stmt, &half_type, &stmt))
return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops (WIDEN_SUM_EXPR, var,
oprnd0, oprnd1);
SSA_NAME_DEF_STMT (var) = pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_widen_sum_pattern: detected: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* We don't allow changing the order of the computation in the inner-loop
when doing outer-loop vectorization. */
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
return pattern_stmt;
}
static gimple
vect_recog_dot_prod_pattern (gimple last_stmt, tree *type_in, tree *type_out)
{
gimple stmt;
tree oprnd0, oprnd1;
tree oprnd00, oprnd01;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
gimple pattern_stmt;
tree prod_type;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
tree var;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Look for the following pattern
DX = (TYPE1) X;
DY = (TYPE1) Y;
DPROD = DX * DY;
DDPROD = (TYPE2) DPROD;
sum_1 = DDPROD + sum_0;
In which
- DX is double the size of X
- DY is double the size of Y
- DX, DY, DPROD all have the same type
- sum is the same size of DPROD or bigger
- sum has been recognized as a reduction variable.
This is equivalent to:
DPROD = X w* Y; #widen mult
sum_1 = DPROD w+ sum_0; #widen summation
or
DPROD = X w* Y; #widen mult
sum_1 = DPROD + sum_0; #summation
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as widening-summation? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
type = gimple_expr_type (stmt);
if (gimple_assign_rhs_code (stmt) != WIDEN_SUM_EXPR)
return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
oprnd1 = gimple_assign_rhs2 (stmt);
half_type = TREE_TYPE (oprnd0);
}
else
{
gimple def_stmt;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
stmt = last_stmt;
if (widened_name_p (oprnd0, stmt, &half_type, &def_stmt))
{
stmt = def_stmt;
oprnd0 = gimple_assign_rhs1 (stmt);
}
else
half_type = type;
}
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a (widen_)mult_expr */
prod_type = half_type;
stmt = SSA_NAME_DEF_STMT (oprnd0);
/* It could not be the dot_prod pattern if the stmt is outside the loop. */
if (!gimple_bb (stmt) || !flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
return NULL;
/* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
inside the loop (in case we are analyzing an outer-loop). */
if (!is_gimple_assign (stmt))
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_internal_def)
return NULL;
if (gimple_assign_rhs_code (stmt) != MULT_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as a widening multiplication? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
if (gimple_assign_rhs_code (stmt) != WIDEN_MULT_EXPR)
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_internal_def);
oprnd00 = gimple_assign_rhs1 (stmt);
oprnd01 = gimple_assign_rhs2 (stmt);
}
else
{
tree half_type0, half_type1;
gimple def_stmt;
tree oprnd0, oprnd1;
oprnd0 = gimple_assign_rhs1 (stmt);
oprnd1 = gimple_assign_rhs2 (stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), prod_type)
|| !types_compatible_p (TREE_TYPE (oprnd1), prod_type))
return NULL;
if (!widened_name_p (oprnd0, stmt, &half_type0, &def_stmt))
return NULL;
oprnd00 = gimple_assign_rhs1 (def_stmt);
if (!widened_name_p (oprnd1, stmt, &half_type1, &def_stmt))
return NULL;
oprnd01 = gimple_assign_rhs1 (def_stmt);
if (!types_compatible_p (half_type0, half_type1))
return NULL;
if (TYPE_PRECISION (prod_type) != TYPE_PRECISION (half_type0) * 2)
return NULL;
}
half_type = TREE_TYPE (oprnd00);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops3 (DOT_PROD_EXPR, var,
oprnd00, oprnd01, oprnd1);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_dot_prod_pattern: detected: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* We don't allow changing the order of the computation in the inner-loop
when doing outer-loop vectorization. */
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
return pattern_stmt;
}
static tree
vect_recog_dot_prod_pattern (tree last_stmt, tree *type_in, tree *type_out)
{
tree stmt, expr;
tree oprnd0, oprnd1;
tree oprnd00, oprnd01;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
tree pattern_expr;
tree prod_type;
if (TREE_CODE (last_stmt) != MODIFY_EXPR)
return NULL;
expr = TREE_OPERAND (last_stmt, 1);
type = TREE_TYPE (expr);
/* Look for the following pattern
DX = (TYPE1) X;
DY = (TYPE1) Y;
DPROD = DX * DY;
DDPROD = (TYPE2) DPROD;
sum_1 = DDPROD + sum_0;
In which
- DX is double the size of X
- DY is double the size of Y
- DX, DY, DPROD all have the same type
- sum is the same size of DPROD or bigger
- sum has been recognized as a reduction variable.
This is equivalent to:
DPROD = X w* Y; #widen mult
sum_1 = DPROD w+ sum_0; #widen summation
or
DPROD = X w* Y; #widen mult
sum_1 = DPROD + sum_0; #summation
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (TREE_CODE (expr) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as widening-summation? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
expr = TREE_OPERAND (stmt, 1);
type = TREE_TYPE (expr);
if (TREE_CODE (expr) != WIDEN_SUM_EXPR)
return NULL;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
half_type = TREE_TYPE (oprnd0);
}
else
{
tree def_stmt;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
if (TYPE_MAIN_VARIANT (TREE_TYPE (oprnd0)) != TYPE_MAIN_VARIANT (type)
|| TYPE_MAIN_VARIANT (TREE_TYPE (oprnd1)) != TYPE_MAIN_VARIANT (type))
return NULL;
stmt = last_stmt;
if (widened_name_p (oprnd0, stmt, &half_type, &def_stmt))
{
stmt = def_stmt;
expr = TREE_OPERAND (stmt, 1);
oprnd0 = TREE_OPERAND (expr, 0);
}
else
half_type = type;
}
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a (widen_)mult_expr */
prod_type = half_type;
stmt = SSA_NAME_DEF_STMT (oprnd0);
gcc_assert (stmt);
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_loop_def)
return NULL;
expr = TREE_OPERAND (stmt, 1);
if (TREE_CODE (expr) != MULT_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as a widening multiplication? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
expr = TREE_OPERAND (stmt, 1);
if (TREE_CODE (expr) != WIDEN_MULT_EXPR)
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_loop_def);
oprnd00 = TREE_OPERAND (expr, 0);
oprnd01 = TREE_OPERAND (expr, 1);
}
else
{
tree half_type0, half_type1;
tree def_stmt;
tree oprnd0, oprnd1;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
if (TYPE_MAIN_VARIANT (TREE_TYPE (oprnd0))
!= TYPE_MAIN_VARIANT (prod_type)
|| TYPE_MAIN_VARIANT (TREE_TYPE (oprnd1))
!= TYPE_MAIN_VARIANT (prod_type))
return NULL;
if (!widened_name_p (oprnd0, stmt, &half_type0, &def_stmt))
return NULL;
oprnd00 = TREE_OPERAND (TREE_OPERAND (def_stmt, 1), 0);
if (!widened_name_p (oprnd1, stmt, &half_type1, &def_stmt))
return NULL;
oprnd01 = TREE_OPERAND (TREE_OPERAND (def_stmt, 1), 0);
if (TYPE_MAIN_VARIANT (half_type0) != TYPE_MAIN_VARIANT (half_type1))
return NULL;
if (TYPE_PRECISION (prod_type) != TYPE_PRECISION (half_type0) * 2)
return NULL;
}
half_type = TREE_TYPE (oprnd00);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
pattern_expr = build3 (DOT_PROD_EXPR, type, oprnd00, oprnd01, oprnd1);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_dot_prod_pattern: detected: ");
print_generic_expr (vect_dump, pattern_expr, TDF_SLIM);
}
return pattern_expr;
}
static void
vect_pattern_recog_1 (
gimple (* vect_recog_func) (gimple, tree *, tree *),
gimple_stmt_iterator si)
{
gimple stmt = gsi_stmt (si), pattern_stmt;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info pattern_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree pattern_vectype;
tree type_in, type_out;
enum tree_code code;
pattern_stmt = (* vect_recog_func) (stmt, &type_in, &type_out);
if (!pattern_stmt)
return;
if (VECTOR_MODE_P (TYPE_MODE (type_in)))
{
/* No need to check target support (already checked by the pattern
recognition function). */
pattern_vectype = type_in;
}
else
{
enum tree_code vec_mode;
enum insn_code icode;
optab optab;
/* Check target support */
pattern_vectype = get_vectype_for_scalar_type (type_in);
if (!pattern_vectype)
return;
if (is_gimple_assign (pattern_stmt))
code = gimple_assign_rhs_code (pattern_stmt);
else
{
gcc_assert (is_gimple_call (pattern_stmt));
code = CALL_EXPR;
}
optab = optab_for_tree_code (code, pattern_vectype, optab_default);
vec_mode = TYPE_MODE (pattern_vectype);
if (!optab
|| (icode = optab_handler (optab, vec_mode)->insn_code) ==
CODE_FOR_nothing
|| (type_out
&& (!get_vectype_for_scalar_type (type_out)
|| (insn_data[icode].operand[0].mode !=
TYPE_MODE (get_vectype_for_scalar_type (type_out))))))
return;
}
/* Found a vectorizable pattern. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "pattern recognized: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* Mark the stmts that are involved in the pattern. */
gsi_insert_before (&si, pattern_stmt, GSI_SAME_STMT);
set_vinfo_for_stmt (pattern_stmt,
new_stmt_vec_info (pattern_stmt, loop_vinfo));
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = stmt;
STMT_VINFO_DEF_TYPE (pattern_stmt_info) = STMT_VINFO_DEF_TYPE (stmt_info);
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
STMT_VINFO_IN_PATTERN_P (stmt_info) = true;
STMT_VINFO_RELATED_STMT (stmt_info) = pattern_stmt;
return;
}