static gimple
vect_recog_widen_sum_pattern (gimple last_stmt, tree *type_in, tree *type_out)
{
gimple stmt;
tree oprnd0, oprnd1;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
gimple pattern_stmt;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
tree var;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Look for the following pattern
DX = (TYPE) X;
sum_1 = DX + sum_0;
In which DX is at least double the size of X, and sum_1 has been
recognized as a reduction variable.
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a cast from type 'type' to type
'TYPE'. */
if (!widened_name_p (oprnd0, last_stmt, &half_type, &stmt))
return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops (WIDEN_SUM_EXPR, var,
oprnd0, oprnd1);
SSA_NAME_DEF_STMT (var) = pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_widen_sum_pattern: detected: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* We don't allow changing the order of the computation in the inner-loop
when doing outer-loop vectorization. */
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
return pattern_stmt;
}
static gimple
vect_recog_dot_prod_pattern (gimple last_stmt, tree *type_in, tree *type_out)
{
gimple stmt;
tree oprnd0, oprnd1;
tree oprnd00, oprnd01;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
gimple pattern_stmt;
tree prod_type;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
tree var;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Look for the following pattern
DX = (TYPE1) X;
DY = (TYPE1) Y;
DPROD = DX * DY;
DDPROD = (TYPE2) DPROD;
sum_1 = DDPROD + sum_0;
In which
- DX is double the size of X
- DY is double the size of Y
- DX, DY, DPROD all have the same type
- sum is the same size of DPROD or bigger
- sum has been recognized as a reduction variable.
This is equivalent to:
DPROD = X w* Y; #widen mult
sum_1 = DPROD w+ sum_0; #widen summation
or
DPROD = X w* Y; #widen mult
sum_1 = DPROD + sum_0; #summation
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as widening-summation? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
type = gimple_expr_type (stmt);
if (gimple_assign_rhs_code (stmt) != WIDEN_SUM_EXPR)
return NULL;
oprnd0 = gimple_assign_rhs1 (stmt);
oprnd1 = gimple_assign_rhs2 (stmt);
half_type = TREE_TYPE (oprnd0);
}
else
{
gimple def_stmt;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
stmt = last_stmt;
if (widened_name_p (oprnd0, stmt, &half_type, &def_stmt))
{
stmt = def_stmt;
oprnd0 = gimple_assign_rhs1 (stmt);
}
else
half_type = type;
}
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a (widen_)mult_expr */
prod_type = half_type;
stmt = SSA_NAME_DEF_STMT (oprnd0);
/* It could not be the dot_prod pattern if the stmt is outside the loop. */
if (!gimple_bb (stmt) || !flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
return NULL;
/* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
inside the loop (in case we are analyzing an outer-loop). */
if (!is_gimple_assign (stmt))
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_internal_def)
return NULL;
if (gimple_assign_rhs_code (stmt) != MULT_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as a widening multiplication? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
if (gimple_assign_rhs_code (stmt) != WIDEN_MULT_EXPR)
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_internal_def);
oprnd00 = gimple_assign_rhs1 (stmt);
oprnd01 = gimple_assign_rhs2 (stmt);
}
else
{
tree half_type0, half_type1;
gimple def_stmt;
tree oprnd0, oprnd1;
oprnd0 = gimple_assign_rhs1 (stmt);
oprnd1 = gimple_assign_rhs2 (stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), prod_type)
|| !types_compatible_p (TREE_TYPE (oprnd1), prod_type))
return NULL;
if (!widened_name_p (oprnd0, stmt, &half_type0, &def_stmt))
return NULL;
oprnd00 = gimple_assign_rhs1 (def_stmt);
if (!widened_name_p (oprnd1, stmt, &half_type1, &def_stmt))
return NULL;
oprnd01 = gimple_assign_rhs1 (def_stmt);
if (!types_compatible_p (half_type0, half_type1))
return NULL;
if (TYPE_PRECISION (prod_type) != TYPE_PRECISION (half_type0) * 2)
return NULL;
}
half_type = TREE_TYPE (oprnd00);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops3 (DOT_PROD_EXPR, var,
oprnd00, oprnd01, oprnd1);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_dot_prod_pattern: detected: ");
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
/* We don't allow changing the order of the computation in the inner-loop
when doing outer-loop vectorization. */
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
return pattern_stmt;
}
static gimple
vect_recog_widen_mult_pattern (gimple last_stmt,
tree *type_in,
tree *type_out)
{
gimple def_stmt0, def_stmt1;
tree oprnd0, oprnd1;
tree type, half_type0, half_type1;
gimple pattern_stmt;
tree vectype, vectype_out;
tree dummy;
tree var;
enum tree_code dummy_code;
int dummy_int;
VEC (tree, heap) *dummy_vec;
if (!is_gimple_assign (last_stmt))
return NULL;
type = gimple_expr_type (last_stmt);
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
return NULL;
oprnd0 = gimple_assign_rhs1 (last_stmt);
oprnd1 = gimple_assign_rhs2 (last_stmt);
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
return NULL;
/* Check argument 0 */
if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0))
return NULL;
oprnd0 = gimple_assign_rhs1 (def_stmt0);
/* Check argument 1 */
if (!widened_name_p (oprnd1, last_stmt, &half_type1, &def_stmt1))
return NULL;
oprnd1 = gimple_assign_rhs1 (def_stmt1);
if (!types_compatible_p (half_type0, half_type1))
return NULL;
/* Pattern detected. */
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "vect_recog_widen_mult_pattern: detected: ");
/* Check target support */
vectype = get_vectype_for_scalar_type (half_type0);
vectype_out = get_vectype_for_scalar_type (type);
if (!vectype
|| !vectype_out
|| !supportable_widening_operation (WIDEN_MULT_EXPR, last_stmt,
vectype_out, vectype,
&dummy, &dummy, &dummy_code,
&dummy_code, &dummy_int, &dummy_vec))
return NULL;
*type_in = vectype;
*type_out = vectype_out;
/* Pattern supported. Create a stmt to be used to replace the pattern: */
var = vect_recog_temp_ssa_var (type, NULL);
pattern_stmt = gimple_build_assign_with_ops (WIDEN_MULT_EXPR, var, oprnd0,
oprnd1);
SSA_NAME_DEF_STMT (var) = pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
return pattern_stmt;
}
static tree
vect_recog_dot_prod_pattern (tree last_stmt, tree *type_in, tree *type_out)
{
tree stmt, expr;
tree oprnd0, oprnd1;
tree oprnd00, oprnd01;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
tree type, half_type;
tree pattern_expr;
tree prod_type;
if (TREE_CODE (last_stmt) != MODIFY_EXPR)
return NULL;
expr = TREE_OPERAND (last_stmt, 1);
type = TREE_TYPE (expr);
/* Look for the following pattern
DX = (TYPE1) X;
DY = (TYPE1) Y;
DPROD = DX * DY;
DDPROD = (TYPE2) DPROD;
sum_1 = DDPROD + sum_0;
In which
- DX is double the size of X
- DY is double the size of Y
- DX, DY, DPROD all have the same type
- sum is the same size of DPROD or bigger
- sum has been recognized as a reduction variable.
This is equivalent to:
DPROD = X w* Y; #widen mult
sum_1 = DPROD w+ sum_0; #widen summation
or
DPROD = X w* Y; #widen mult
sum_1 = DPROD + sum_0; #summation
*/
/* Starting from LAST_STMT, follow the defs of its uses in search
of the above pattern. */
if (TREE_CODE (expr) != PLUS_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as widening-summation? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
expr = TREE_OPERAND (stmt, 1);
type = TREE_TYPE (expr);
if (TREE_CODE (expr) != WIDEN_SUM_EXPR)
return NULL;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
half_type = TREE_TYPE (oprnd0);
}
else
{
tree def_stmt;
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
return NULL;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
if (TYPE_MAIN_VARIANT (TREE_TYPE (oprnd0)) != TYPE_MAIN_VARIANT (type)
|| TYPE_MAIN_VARIANT (TREE_TYPE (oprnd1)) != TYPE_MAIN_VARIANT (type))
return NULL;
stmt = last_stmt;
if (widened_name_p (oprnd0, stmt, &half_type, &def_stmt))
{
stmt = def_stmt;
expr = TREE_OPERAND (stmt, 1);
oprnd0 = TREE_OPERAND (expr, 0);
}
else
half_type = type;
}
/* So far so good. Since last_stmt was detected as a (summation) reduction,
we know that oprnd1 is the reduction variable (defined by a loop-header
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
Left to check that oprnd0 is defined by a (widen_)mult_expr */
prod_type = half_type;
stmt = SSA_NAME_DEF_STMT (oprnd0);
gcc_assert (stmt);
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_loop_def)
return NULL;
expr = TREE_OPERAND (stmt, 1);
if (TREE_CODE (expr) != MULT_EXPR)
return NULL;
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
{
/* Has been detected as a widening multiplication? */
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
expr = TREE_OPERAND (stmt, 1);
if (TREE_CODE (expr) != WIDEN_MULT_EXPR)
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_loop_def);
oprnd00 = TREE_OPERAND (expr, 0);
oprnd01 = TREE_OPERAND (expr, 1);
}
else
{
tree half_type0, half_type1;
tree def_stmt;
tree oprnd0, oprnd1;
oprnd0 = TREE_OPERAND (expr, 0);
oprnd1 = TREE_OPERAND (expr, 1);
if (TYPE_MAIN_VARIANT (TREE_TYPE (oprnd0))
!= TYPE_MAIN_VARIANT (prod_type)
|| TYPE_MAIN_VARIANT (TREE_TYPE (oprnd1))
!= TYPE_MAIN_VARIANT (prod_type))
return NULL;
if (!widened_name_p (oprnd0, stmt, &half_type0, &def_stmt))
return NULL;
oprnd00 = TREE_OPERAND (TREE_OPERAND (def_stmt, 1), 0);
if (!widened_name_p (oprnd1, stmt, &half_type1, &def_stmt))
return NULL;
oprnd01 = TREE_OPERAND (TREE_OPERAND (def_stmt, 1), 0);
if (TYPE_MAIN_VARIANT (half_type0) != TYPE_MAIN_VARIANT (half_type1))
return NULL;
if (TYPE_PRECISION (prod_type) != TYPE_PRECISION (half_type0) * 2)
return NULL;
}
half_type = TREE_TYPE (oprnd00);
*type_in = half_type;
*type_out = type;
/* Pattern detected. Create a stmt to be used to replace the pattern: */
pattern_expr = build3 (DOT_PROD_EXPR, type, oprnd00, oprnd01, oprnd1);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "vect_recog_dot_prod_pattern: detected: ");
print_generic_expr (vect_dump, pattern_expr, TDF_SLIM);
}
return pattern_expr;
}