/* SLP - Basic Block Vectorization
- Copyright (C) 2007, 2008, 2009, 2010, 2011
+ Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012
Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
static void
vect_free_slp_tree (slp_tree node)
{
+ int i;
+ slp_void_p child;
+
if (!node)
return;
- if (SLP_TREE_LEFT (node))
- vect_free_slp_tree (SLP_TREE_LEFT (node));
-
- if (SLP_TREE_RIGHT (node))
- vect_free_slp_tree (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_free_slp_tree ((slp_tree) child);
+ VEC_free (slp_void_p, heap, SLP_TREE_CHILDREN (node));
VEC_free (gimple, heap, SLP_TREE_SCALAR_STMTS (node));
if (SLP_TREE_VEC_STMTS (node))
}
-/* Get the defs for the rhs of STMT (collect them in DEF_STMTS0/1), check that
- they are of a legal type and that they match the defs of the first stmt of
- the SLP group (stored in FIRST_STMT_...). */
+/* Create an SLP node for SCALAR_STMTS. */
+
+static slp_tree
+vect_create_new_slp_node (VEC (gimple, heap) *scalar_stmts)
+{
+ slp_tree node;
+ gimple stmt = VEC_index (gimple, scalar_stmts, 0);
+ unsigned int nops;
+
+ if (is_gimple_call (stmt))
+ nops = gimple_call_num_args (stmt);
+ else if (is_gimple_assign (stmt))
+ {
+ nops = gimple_num_ops (stmt) - 1;
+ if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+ nops++;
+ }
+ else
+ return NULL;
+
+ node = XNEW (struct _slp_tree);
+ SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
+ SLP_TREE_VEC_STMTS (node) = NULL;
+ SLP_TREE_CHILDREN (node) = VEC_alloc (slp_void_p, heap, nops);
+ SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
+ SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
+
+ return node;
+}
+
+
+/* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
+ operand. */
+static VEC (slp_oprnd_info, heap) *
+vect_create_oprnd_info (int nops, int group_size)
+{
+ int i;
+ slp_oprnd_info oprnd_info;
+ VEC (slp_oprnd_info, heap) *oprnds_info;
+
+ oprnds_info = VEC_alloc (slp_oprnd_info, heap, nops);
+ for (i = 0; i < nops; i++)
+ {
+ oprnd_info = XNEW (struct _slp_oprnd_info);
+ oprnd_info->def_stmts = VEC_alloc (gimple, heap, group_size);
+ oprnd_info->first_dt = vect_uninitialized_def;
+ oprnd_info->first_def_type = NULL_TREE;
+ oprnd_info->first_const_oprnd = NULL_TREE;
+ oprnd_info->first_pattern = false;
+ VEC_quick_push (slp_oprnd_info, oprnds_info, oprnd_info);
+ }
+
+ return oprnds_info;
+}
+
+
+/* Free operands info. */
+
+static void
+vect_free_oprnd_info (VEC (slp_oprnd_info, heap) **oprnds_info)
+{
+ int i;
+ slp_oprnd_info oprnd_info;
+
+ FOR_EACH_VEC_ELT (slp_oprnd_info, *oprnds_info, i, oprnd_info)
+ {
+ VEC_free (gimple, heap, oprnd_info->def_stmts);
+ XDELETE (oprnd_info);
+ }
+
+ VEC_free (slp_oprnd_info, heap, *oprnds_info);
+}
+
+
+/* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
+ they are of a valid type and that they match the defs of the first stmt of
+ the SLP group (stored in OPRNDS_INFO). */
static bool
vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
slp_tree slp_node, gimple stmt,
- VEC (gimple, heap) **def_stmts0,
- VEC (gimple, heap) **def_stmts1,
- enum vect_def_type *first_stmt_dt0,
- enum vect_def_type *first_stmt_dt1,
- tree *first_stmt_def0_type,
- tree *first_stmt_def1_type,
- tree *first_stmt_const_oprnd,
- int ncopies_for_cost,
- bool *pattern0, bool *pattern1)
+ int ncopies_for_cost, bool first,
+ VEC (slp_oprnd_info, heap) **oprnds_info)
{
tree oprnd;
unsigned int i, number_of_oprnds;
- tree def[2];
+ tree def, def_op0 = NULL_TREE;
gimple def_stmt;
- enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
- stmt_vec_info stmt_info =
- vinfo_for_stmt (VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0));
- enum gimple_rhs_class rhs_class;
+ enum vect_def_type dt = vect_uninitialized_def;
+ enum vect_def_type dt_op0 = vect_uninitialized_def;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree lhs = gimple_get_lhs (stmt);
struct loop *loop = NULL;
enum tree_code rhs_code;
bool different_types = false;
+ bool pattern = false;
+ slp_oprnd_info oprnd_info, oprnd0_info, oprnd1_info;
+ int op_idx = 1;
+ tree compare_rhs = NULL_TREE;
if (loop_vinfo)
loop = LOOP_VINFO_LOOP (loop_vinfo);
- rhs_class = get_gimple_rhs_class (gimple_assign_rhs_code (stmt));
- number_of_oprnds = gimple_num_ops (stmt) - 1; /* RHS only */
+ if (is_gimple_call (stmt))
+ {
+ number_of_oprnds = gimple_call_num_args (stmt);
+ op_idx = 3;
+ }
+ else if (is_gimple_assign (stmt))
+ {
+ number_of_oprnds = gimple_num_ops (stmt) - 1;
+ if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+ number_of_oprnds++;
+ }
+ else
+ return false;
for (i = 0; i < number_of_oprnds; i++)
{
- oprnd = gimple_op (stmt, i + 1);
+ if (compare_rhs)
+ {
+ oprnd = compare_rhs;
+ compare_rhs = NULL_TREE;
+ }
+ else
+ oprnd = gimple_op (stmt, op_idx++);
+
+ oprnd_info = VEC_index (slp_oprnd_info, *oprnds_info, i);
- if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def[i],
- &dt[i])
- || (!def_stmt && dt[i] != vect_constant_def))
+ if (COMPARISON_CLASS_P (oprnd))
+ {
+ compare_rhs = TREE_OPERAND (oprnd, 1);
+ oprnd = TREE_OPERAND (oprnd, 0);
+ }
+
+ if (!vect_is_simple_use (oprnd, NULL, loop_vinfo, bb_vinfo, &def_stmt,
+ &def, &dt)
+ || (!def_stmt && dt != vect_constant_def))
{
if (vect_print_dump_info (REPORT_SLP))
{
&& !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
&& !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
{
- if (!*first_stmt_dt0)
- *pattern0 = true;
- else
- {
- if (i == 1 && !*first_stmt_dt1)
- *pattern1 = true;
- else if ((i == 0 && !*pattern0) || (i == 1 && !*pattern1))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "Build SLP failed: some of the stmts"
- " are in a pattern, and others are not ");
- print_generic_expr (vect_dump, oprnd, TDF_SLIM);
- }
+ pattern = true;
+ if (!first && !oprnd_info->first_pattern)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Build SLP failed: some of the stmts"
+ " are in a pattern, and others are not ");
+ print_generic_expr (vect_dump, oprnd, TDF_SLIM);
+ }
- return false;
- }
+ return false;
}
def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
- dt[i] = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
+ dt = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
- if (*dt == vect_unknown_def_type)
+ if (dt == vect_unknown_def_type)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Unsupported pattern.");
switch (gimple_code (def_stmt))
{
case GIMPLE_PHI:
- def[i] = gimple_phi_result (def_stmt);
+ def = gimple_phi_result (def_stmt);
break;
case GIMPLE_ASSIGN:
- def[i] = gimple_assign_lhs (def_stmt);
+ def = gimple_assign_lhs (def_stmt);
break;
default:
}
}
- if (!*first_stmt_dt0)
+ if (first)
{
- /* op0 of the first stmt of the group - store its info. */
- *first_stmt_dt0 = dt[i];
- if (def[i])
- *first_stmt_def0_type = TREE_TYPE (def[i]);
+ oprnd_info->first_dt = dt;
+ oprnd_info->first_pattern = pattern;
+ if (def)
+ {
+ oprnd_info->first_def_type = TREE_TYPE (def);
+ oprnd_info->first_const_oprnd = NULL_TREE;
+ }
else
- *first_stmt_const_oprnd = oprnd;
+ {
+ oprnd_info->first_def_type = NULL_TREE;
+ oprnd_info->first_const_oprnd = oprnd;
+ }
- /* Analyze costs (for the first stmt of the group only). */
- if (rhs_class != GIMPLE_SINGLE_RHS)
- /* Not memory operation (we don't call this functions for loads). */
- vect_model_simple_cost (stmt_info, ncopies_for_cost, dt, slp_node);
- else
- /* Store. */
- vect_model_store_cost (stmt_info, ncopies_for_cost, false,
- dt[0], slp_node);
+ if (i == 0)
+ {
+ def_op0 = def;
+ dt_op0 = dt;
+ /* Analyze costs (for the first stmt of the group only). */
+ if (REFERENCE_CLASS_P (lhs))
+ /* Store. */
+ vect_model_store_cost (stmt_info, ncopies_for_cost, false,
+ dt, slp_node);
+ else
+ /* Not memory operation (we don't call this function for
+ loads). */
+ vect_model_simple_cost (stmt_info, ncopies_for_cost, &dt,
+ slp_node);
+ }
}
-
else
{
- if (!*first_stmt_dt1 && i == 1)
+ /* Not first stmt of the group, check that the def-stmt/s match
+ the def-stmt/s of the first stmt. Allow different definition
+ types for reduction chains: the first stmt must be a
+ vect_reduction_def (a phi node), and the rest
+ vect_internal_def. */
+ if (((oprnd_info->first_dt != dt
+ && !(oprnd_info->first_dt == vect_reduction_def
+ && dt == vect_internal_def))
+ || (oprnd_info->first_def_type != NULL_TREE
+ && def
+ && !types_compatible_p (oprnd_info->first_def_type,
+ TREE_TYPE (def))))
+ || (!def
+ && !types_compatible_p (TREE_TYPE (oprnd_info->first_const_oprnd),
+ TREE_TYPE (oprnd)))
+ || different_types)
{
- /* op1 of the first stmt of the group - store its info. */
- *first_stmt_dt1 = dt[i];
- if (def[i])
- *first_stmt_def1_type = TREE_TYPE (def[i]);
- else
+ if (number_of_oprnds != 2)
{
- /* We assume that the stmt contains only one constant
- operand. We fail otherwise, to be on the safe side. */
- if (*first_stmt_const_oprnd)
- {
- if (vect_print_dump_info (REPORT_SLP))
- fprintf (vect_dump, "Build SLP failed: two constant "
- "oprnds in stmt");
- return false;
- }
- *first_stmt_const_oprnd = oprnd;
- }
- }
- else
- {
- /* Not first stmt of the group, check that the def-stmt/s match
- the def-stmt/s of the first stmt. Allow different definition
- types for reduction chains: the first stmt must be a
- vect_reduction_def (a phi node), and the rest
- vect_internal_def. */
- if ((i == 0
- && ((*first_stmt_dt0 != dt[i]
- && !(*first_stmt_dt0 == vect_reduction_def
- && dt[i] == vect_internal_def))
- || (*first_stmt_def0_type && def[0]
- && !types_compatible_p (*first_stmt_def0_type,
- TREE_TYPE (def[0])))))
- || (i == 1
- && ((*first_stmt_dt1 != dt[i]
- && !(*first_stmt_dt1 == vect_reduction_def
- && dt[i] == vect_internal_def))
- || (*first_stmt_def1_type && def[1]
- && !types_compatible_p (*first_stmt_def1_type,
- TREE_TYPE (def[1])))))
- || (!def[i]
- && !types_compatible_p (TREE_TYPE (*first_stmt_const_oprnd),
- TREE_TYPE (oprnd)))
- || different_types)
+ if (vect_print_dump_info (REPORT_SLP))
+ fprintf (vect_dump, "Build SLP failed: different types ");
+
+ return false;
+ }
+
+ /* Try to swap operands in case of binary operation. */
+ if (i == 0)
+ different_types = true;
+ else
{
- if (i != number_of_oprnds - 1)
- different_types = true;
- else
- {
- if (is_gimple_assign (stmt)
- && (rhs_code = gimple_assign_rhs_code (stmt))
- && TREE_CODE_CLASS (rhs_code) == tcc_binary
- && commutative_tree_code (rhs_code)
- && *first_stmt_dt0 == dt[1]
- && *first_stmt_dt1 == dt[0]
- && def[0] && def[1]
- && !(*first_stmt_def0_type
- && !types_compatible_p (*first_stmt_def0_type,
- TREE_TYPE (def[1])))
- && !(*first_stmt_def1_type
- && !types_compatible_p (*first_stmt_def1_type,
- TREE_TYPE (def[0]))))
- {
- if (vect_print_dump_info (REPORT_SLP))
- {
- fprintf (vect_dump, "Swapping operands of ");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
-
- swap_tree_operands (stmt, gimple_assign_rhs1_ptr (stmt),
- gimple_assign_rhs2_ptr (stmt));
+ oprnd0_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ if (is_gimple_assign (stmt)
+ && (rhs_code = gimple_assign_rhs_code (stmt))
+ && TREE_CODE_CLASS (rhs_code) == tcc_binary
+ && commutative_tree_code (rhs_code)
+ && oprnd0_info->first_dt == dt
+ && oprnd_info->first_dt == dt_op0
+ && def_op0 && def
+ && !(oprnd0_info->first_def_type
+ && !types_compatible_p (oprnd0_info->first_def_type,
+ TREE_TYPE (def)))
+ && !(oprnd_info->first_def_type
+ && !types_compatible_p (oprnd_info->first_def_type,
+ TREE_TYPE (def_op0))))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump, "Swapping operands of ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
- else
- {
- if (vect_print_dump_info (REPORT_SLP))
- fprintf (vect_dump, "Build SLP failed: different types ");
- return false;
- }
+ swap_tree_operands (stmt, gimple_assign_rhs1_ptr (stmt),
+ gimple_assign_rhs2_ptr (stmt));
+ }
+ else
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ fprintf (vect_dump, "Build SLP failed: different types ");
+
+ return false;
}
}
}
}
/* Check the types of the definitions. */
- switch (dt[i])
+ switch (dt)
{
case vect_constant_def:
case vect_external_def:
+ case vect_reduction_def:
break;
case vect_internal_def:
- case vect_reduction_def:
- if ((i == 0 && !different_types) || (i == 1 && different_types))
- VEC_safe_push (gimple, heap, *def_stmts0, def_stmt);
+ if (different_types)
+ {
+ oprnd0_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ oprnd1_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ if (i == 0)
+ VEC_quick_push (gimple, oprnd1_info->def_stmts, def_stmt);
+ else
+ VEC_quick_push (gimple, oprnd0_info->def_stmts, def_stmt);
+ }
else
- VEC_safe_push (gimple, heap, *def_stmts1, def_stmt);
+ VEC_quick_push (gimple, oprnd_info->def_stmts, def_stmt);
+
break;
default:
if (vect_print_dump_info (REPORT_SLP))
{
fprintf (vect_dump, "Build SLP failed: illegal type of def ");
- print_generic_expr (vect_dump, def[i], TDF_SLIM);
+ print_generic_expr (vect_dump, def, TDF_SLIM);
}
return false;
VEC (slp_tree, heap) **loads,
unsigned int vectorization_factor, bool *loads_permuted)
{
- VEC (gimple, heap) *def_stmts0 = VEC_alloc (gimple, heap, group_size);
- VEC (gimple, heap) *def_stmts1 = VEC_alloc (gimple, heap, group_size);
unsigned int i;
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
gimple stmt = VEC_index (gimple, stmts, 0);
- enum vect_def_type first_stmt_dt0 = vect_uninitialized_def;
- enum vect_def_type first_stmt_dt1 = vect_uninitialized_def;
enum tree_code first_stmt_code = ERROR_MARK, rhs_code = ERROR_MARK;
- tree first_stmt_def1_type = NULL_TREE, first_stmt_def0_type = NULL_TREE;
+ enum tree_code first_cond_code = ERROR_MARK;
tree lhs;
bool stop_recursion = false, need_same_oprnds = false;
tree vectype, scalar_type, first_op1 = NULL_TREE;
int icode;
enum machine_mode optab_op2_mode;
enum machine_mode vec_mode;
- tree first_stmt_const_oprnd = NULL_TREE;
struct data_reference *first_dr;
- bool pattern0 = false, pattern1 = false;
HOST_WIDE_INT dummy;
bool permutation = false;
unsigned int load_place;
gimple first_load, prev_first_load = NULL;
+ VEC (slp_oprnd_info, heap) *oprnds_info;
+ unsigned int nops;
+ slp_oprnd_info oprnd_info;
+ tree cond;
+
+ if (is_gimple_call (stmt))
+ nops = gimple_call_num_args (stmt);
+ else if (is_gimple_assign (stmt))
+ {
+ nops = gimple_num_ops (stmt) - 1;
+ if (gimple_assign_rhs_code (stmt) == COND_EXPR)
+ nops++;
+ }
+ else
+ return false;
+
+ oprnds_info = vect_create_oprnd_info (nops, group_size);
/* For every stmt in NODE find its def stmt/s. */
FOR_EACH_VEC_ELT (gimple, stmts, i, stmt)
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
if (vect_print_dump_info (REPORT_SLP))
{
fprintf (vect_dump,
- "Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL");
+ "Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
+ if (is_gimple_assign (stmt)
+ && gimple_assign_rhs_code (stmt) == COND_EXPR
+ && (cond = gimple_assign_rhs1 (stmt))
+ && !COMPARISON_CLASS_P (cond))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump,
+ "Build SLP failed: condition is not comparison ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
+
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
vectype = get_vectype_for_scalar_type (scalar_type);
if (!vectype)
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
}
+
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
ncopies = vectorization_factor / TYPE_VECTOR_SUBPARTS (vectype);
if (is_gimple_call (stmt))
- rhs_code = CALL_EXPR;
+ {
+ rhs_code = CALL_EXPR;
+ if (gimple_call_internal_p (stmt)
+ || gimple_call_tail_p (stmt)
+ || gimple_call_noreturn_p (stmt)
+ || !gimple_call_nothrow_p (stmt)
+ || gimple_call_chain (stmt))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump,
+ "Build SLP failed: unsupported call type ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
+ }
else
rhs_code = gimple_assign_rhs_code (stmt);
{
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: no optab.");
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
icode = (int) optab_handler (optab, vec_mode);
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: "
"op not supported by target.");
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
optab_op2_mode = insn_data[icode].operand[2].mode;
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
+
+ if (rhs_code == CALL_EXPR)
+ {
+ gimple first_stmt = VEC_index (gimple, stmts, 0);
+ if (gimple_call_num_args (stmt) != nops
+ || !operand_equal_p (gimple_call_fn (first_stmt),
+ gimple_call_fn (stmt), 0)
+ || gimple_call_fntype (first_stmt)
+ != gimple_call_fntype (stmt))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump,
+ "Build SLP failed: different calls in ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
+ }
}
/* Strided store or load. */
{
/* Store. */
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node,
- stmt, &def_stmts0, &def_stmts1,
- &first_stmt_dt0,
- &first_stmt_dt1,
- &first_stmt_def0_type,
- &first_stmt_def1_type,
- &first_stmt_const_oprnd,
- ncopies_for_cost,
- &pattern0, &pattern1))
- return false;
+ stmt, ncopies_for_cost,
+ (i == 0), &oprnds_info))
+ {
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
}
else
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
{
if (TREE_CODE_CLASS (rhs_code) == tcc_reference)
{
- /* Not strided load. */
+ /* Not strided load. */
if (vect_print_dump_info (REPORT_SLP))
{
fprintf (vect_dump, "Build SLP failed: not strided load ");
}
/* FORNOW: Not strided loads are not supported. */
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
/* Not memory operation. */
if (TREE_CODE_CLASS (rhs_code) != tcc_binary
- && TREE_CODE_CLASS (rhs_code) != tcc_unary)
+ && TREE_CODE_CLASS (rhs_code) != tcc_unary
+ && rhs_code != COND_EXPR
+ && rhs_code != CALL_EXPR)
{
if (vect_print_dump_info (REPORT_SLP))
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info);
return false;
}
+ if (rhs_code == COND_EXPR)
+ {
+ tree cond_expr = gimple_assign_rhs1 (stmt);
+
+ if (i == 0)
+ first_cond_code = TREE_CODE (cond_expr);
+ else if (first_cond_code != TREE_CODE (cond_expr))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump, "Build SLP failed: different"
+ " operation");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
+ }
+
/* Find the def-stmts. */
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
- &def_stmts0, &def_stmts1,
- &first_stmt_dt0, &first_stmt_dt1,
- &first_stmt_def0_type,
- &first_stmt_def1_type,
- &first_stmt_const_oprnd,
- ncopies_for_cost,
- &pattern0, &pattern1))
- return false;
+ ncopies_for_cost, (i == 0),
+ &oprnds_info))
+ {
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
}
}
*loads_permuted = true;
}
+ vect_free_oprnd_info (&oprnds_info);
return true;
}
/* Create SLP_TREE nodes for the definition node/s. */
- if (first_stmt_dt0 == vect_internal_def)
- {
- slp_tree left_node = XNEW (struct _slp_tree);
- SLP_TREE_SCALAR_STMTS (left_node) = def_stmts0;
- SLP_TREE_VEC_STMTS (left_node) = NULL;
- SLP_TREE_LEFT (left_node) = NULL;
- SLP_TREE_RIGHT (left_node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (left_node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (left_node) = 0;
- if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &left_node, group_size,
- inside_cost, outside_cost, ncopies_for_cost,
- max_nunits, load_permutation, loads,
- vectorization_factor, loads_permuted))
- return false;
+ FOR_EACH_VEC_ELT (slp_oprnd_info, oprnds_info, i, oprnd_info)
+ {
+ slp_tree child;
- SLP_TREE_LEFT (*node) = left_node;
- }
+ if (oprnd_info->first_dt != vect_internal_def)
+ continue;
- if (first_stmt_dt1 == vect_internal_def)
- {
- slp_tree right_node = XNEW (struct _slp_tree);
- SLP_TREE_SCALAR_STMTS (right_node) = def_stmts1;
- SLP_TREE_VEC_STMTS (right_node) = NULL;
- SLP_TREE_LEFT (right_node) = NULL;
- SLP_TREE_RIGHT (right_node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (right_node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (right_node) = 0;
- if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &right_node, group_size,
+ child = vect_create_new_slp_node (oprnd_info->def_stmts);
+ if (!child
+ || !vect_build_slp_tree (loop_vinfo, bb_vinfo, &child, group_size,
inside_cost, outside_cost, ncopies_for_cost,
max_nunits, load_permutation, loads,
vectorization_factor, loads_permuted))
- return false;
+ {
+ if (child)
+ oprnd_info->def_stmts = NULL;
+ vect_free_slp_tree (child);
+ vect_free_oprnd_info (&oprnds_info);
+ return false;
+ }
- SLP_TREE_RIGHT (*node) = right_node;
+ oprnd_info->def_stmts = NULL;
+ VEC_quick_push (slp_void_p, SLP_TREE_CHILDREN (*node), child);
}
+ vect_free_oprnd_info (&oprnds_info);
return true;
}
{
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return;
}
fprintf (vect_dump, "\n");
- vect_print_slp_tree (SLP_TREE_LEFT (node));
- vect_print_slp_tree (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_print_slp_tree ((slp_tree) child);
}
{
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return;
if (j < 0 || i == j)
STMT_SLP_TYPE (vinfo_for_stmt (stmt)) = mark;
- vect_mark_slp_stmts (SLP_TREE_LEFT (node), mark, j);
- vect_mark_slp_stmts (SLP_TREE_RIGHT (node), mark, j);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_mark_slp_stmts ((slp_tree) child, mark, j);
}
int i;
gimple stmt;
stmt_vec_info stmt_info;
+ slp_void_p child;
if (!node)
return;
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
}
- vect_mark_slp_stmts_relevant (SLP_TREE_LEFT (node));
- vect_mark_slp_stmts_relevant (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_mark_slp_stmts_relevant ((slp_tree) child);
}
gimple stmt;
VEC (gimple, heap) *tmp_stmts;
unsigned int index, i;
+ slp_void_p child;
if (!node)
return;
- vect_slp_rearrange_stmts (SLP_TREE_LEFT (node), group_size, permutation);
- vect_slp_rearrange_stmts (SLP_TREE_RIGHT (node), group_size, permutation);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_slp_rearrange_stmts ((slp_tree) child, group_size, permutation);
gcc_assert (group_size == VEC_length (gimple, SLP_TREE_SCALAR_STMTS (node)));
tmp_stmts = VEC_alloc (gimple, heap, group_size);
gimple stmt)
{
slp_instance new_instance;
- slp_tree node = XNEW (struct _slp_tree);
+ slp_tree node;
unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (stmt));
unsigned int unrolling_factor = 1, nunits;
tree vectype, scalar_type = NULL_TREE;
VEC (slp_tree, heap) *loads;
struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
bool loads_permuted = false;
+ VEC (gimple, heap) *scalar_stmts;
if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)))
{
}
/* Create a node (a root of the SLP tree) for the packed strided stores. */
- SLP_TREE_SCALAR_STMTS (node) = VEC_alloc (gimple, heap, group_size);
+ scalar_stmts = VEC_alloc (gimple, heap, group_size);
next = stmt;
if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)))
{
/* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
while (next)
{
- VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
+ if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (next))
+ && STMT_VINFO_RELATED_STMT (vinfo_for_stmt (next)))
+ VEC_safe_push (gimple, heap, scalar_stmts,
+ STMT_VINFO_RELATED_STMT (vinfo_for_stmt (next)));
+ else
+ VEC_safe_push (gimple, heap, scalar_stmts, next);
next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next));
}
}
else
{
/* Collect reduction statements. */
- for (i = 0; VEC_iterate (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i,
- next);
- i++)
- VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
+ VEC (gimple, heap) *reductions = LOOP_VINFO_REDUCTIONS (loop_vinfo);
+ for (i = 0; VEC_iterate (gimple, reductions, i, next); i++)
+ VEC_safe_push (gimple, heap, scalar_stmts, next);
}
- SLP_TREE_VEC_STMTS (node) = NULL;
- SLP_TREE_NUMBER_OF_VEC_STMTS (node) = 0;
- SLP_TREE_LEFT (node) = NULL;
- SLP_TREE_RIGHT (node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
+ node = vect_create_new_slp_node (scalar_stmts);
/* Calculate the number of vector stmts to create based on the unrolling
factor (number of vectors is 1 if NUNITS >= GROUP_SIZE, and is
vect_detect_hybrid_slp_stmts (slp_tree node)
{
int i;
- gimple stmt;
+ VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (node);
+ gimple stmt = VEC_index (gimple, stmts, 0);
imm_use_iterator imm_iter;
gimple use_stmt;
- stmt_vec_info stmt_vinfo;
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ slp_void_p child;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ struct loop *loop = NULL;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ basic_block bb = NULL;
if (!node)
return;
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ else
+ bb = BB_VINFO_BB (bb_vinfo);
+
FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt)
if (PURE_SLP_STMT (vinfo_for_stmt (stmt))
&& TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0))
- if ((stmt_vinfo = vinfo_for_stmt (use_stmt))
+ if (gimple_bb (use_stmt)
+ && ((loop && flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ || bb == gimple_bb (use_stmt))
+ && (stmt_vinfo = vinfo_for_stmt (use_stmt))
&& !STMT_SLP_TYPE (stmt_vinfo)
&& (STMT_VINFO_RELEVANT (stmt_vinfo)
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
- && !(gimple_code (use_stmt) == GIMPLE_PHI
- && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (use_stmt))
- == vect_reduction_def))
+ && !(gimple_code (use_stmt) == GIMPLE_PHI
+ && STMT_VINFO_DEF_TYPE (stmt_vinfo)
+ == vect_reduction_def))
vect_mark_slp_stmts (node, hybrid, i);
- vect_detect_hybrid_slp_stmts (SLP_TREE_LEFT (node));
- vect_detect_hybrid_slp_stmts (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_detect_hybrid_slp_stmts ((slp_tree) child);
}
bool dummy;
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return true;
- if (!vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_LEFT (node))
- || !vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_RIGHT (node)))
- return false;
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ if (!vect_slp_analyze_node_operations (bb_vinfo, (slp_tree) child))
+ return false;
FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt)
{
VEC (tree, heap) *voprnds = VEC_alloc (tree, heap, number_of_vectors);
bool constant_p, is_store;
tree neutral_op = NULL;
- enum tree_code code = gimple_assign_rhs_code (stmt);
+ enum tree_code code = gimple_expr_code (stmt);
gimple def_stmt;
struct loop *loop;
For example, we have two scalar operands, s1 and s2 (e.g., group of
strided accesses of size two), while NUNITS is four (i.e., four scalars
- of this type can be packed in a vector). The output vector will contain
- two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
+ of this type can be packed in a vector). The output vector will contain
+ two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
will be 2).
If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
containing the operands.
For example, NUNITS is four as before, and the group size is 8
- (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
+ (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
{s5, s6, s7, s8}. */
number_of_copies = least_common_multiple (nunits, group_size) / group_size;
if (is_store)
op = gimple_assign_rhs1 (stmt);
else
- op = gimple_op (stmt, op_num + 1);
+ {
+ switch (code)
+ {
+ case COND_EXPR:
+ if (op_num == 0 || op_num == 1)
+ {
+ tree cond = gimple_assign_rhs1 (stmt);
+ op = TREE_OPERAND (cond, op_num);
+ }
+ else
+ {
+ if (op_num == 2)
+ op = gimple_assign_rhs2 (stmt);
+ else
+ op = gimple_assign_rhs3 (stmt);
+ }
+ break;
+
+ case CALL_EXPR:
+ op = gimple_call_arg (stmt, op_num);
+ break;
+
+ default:
+ op = gimple_op (stmt, op_num + 1);
+ }
+ }
if (reduc_index != -1)
{
If the scalar definitions are loop invariants or constants, collect them and
call vect_get_constant_vectors() to create vector stmts.
Otherwise, the def-stmts must be already vectorized and the vectorized stmts
- must be stored in the LEFT/RIGHT node of SLP_NODE, and we call
- vect_get_slp_vect_defs() to retrieve them.
- If VEC_OPRNDS1 is NULL, don't get vector defs for the second operand (from
- the right node. This is used when the second operand must remain scalar. */
+ must be stored in the corresponding child of SLP_NODE, and we call
+ vect_get_slp_vect_defs () to retrieve them. */
void
-vect_get_slp_defs (tree op0, tree op1, slp_tree slp_node,
- VEC (tree,heap) **vec_oprnds0,
- VEC (tree,heap) **vec_oprnds1, int reduc_index)
+vect_get_slp_defs (VEC (tree, heap) *ops, slp_tree slp_node,
+ VEC (slp_void_p, heap) **vec_oprnds, int reduc_index)
{
- gimple first_stmt;
- enum tree_code code;
- int number_of_vects;
+ gimple first_stmt, first_def;
+ int number_of_vects = 0, i;
+ unsigned int child_index = 0;
HOST_WIDE_INT lhs_size_unit, rhs_size_unit;
+ slp_tree child = NULL;
+ VEC (tree, heap) *vec_defs;
+ tree oprnd, def_lhs;
+ bool vectorized_defs;
first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
- /* The number of vector defs is determined by the number of vector statements
- in the node from which we get those statements. */
- if (SLP_TREE_LEFT (slp_node))
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node));
- else
- {
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
- /* Number of vector stmts was calculated according to LHS in
- vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
- necessary. See vect_get_smallest_scalar_type () for details. */
- vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
- &rhs_size_unit);
- if (rhs_size_unit != lhs_size_unit)
+ FOR_EACH_VEC_ELT (tree, ops, i, oprnd)
+ {
+ /* For each operand we check if it has vectorized definitions in a child
+ node or we need to create them (for invariants and constants). We
+ check if the LHS of the first stmt of the next child matches OPRND.
+ If it does, we found the correct child. Otherwise, we call
+ vect_get_constant_vectors (), and not advance CHILD_INDEX in order
+ to check this child node for the next operand. */
+ vectorized_defs = false;
+ if (VEC_length (slp_void_p, SLP_TREE_CHILDREN (slp_node)) > child_index)
{
- number_of_vects *= rhs_size_unit;
- number_of_vects /= lhs_size_unit;
- }
- }
-
- /* Allocate memory for vectorized defs. */
- *vec_oprnds0 = VEC_alloc (tree, heap, number_of_vects);
-
- /* SLP_NODE corresponds either to a group of stores or to a group of
- unary/binary operations. We don't call this function for loads.
- For reduction defs we call vect_get_constant_vectors(), since we are
- looking for initial loop invariant values. */
- if (SLP_TREE_LEFT (slp_node) && reduc_index == -1)
- /* The defs are already vectorized. */
- vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node), vec_oprnds0);
- else
- /* Build vectors from scalar defs. */
- vect_get_constant_vectors (op0, slp_node, vec_oprnds0, 0, number_of_vects,
- reduc_index);
+ child = (slp_tree) VEC_index (slp_void_p,
+ SLP_TREE_CHILDREN (slp_node),
+ child_index);
+ first_def = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (child), 0);
+
+ /* In the end of a pattern sequence we have a use of the original stmt,
+ so we need to compare OPRND with the original def. */
+ if (is_pattern_stmt_p (vinfo_for_stmt (first_def))
+ && !STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (first_stmt))
+ && !is_pattern_stmt_p (vinfo_for_stmt (first_stmt)))
+ first_def = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (first_def));
+
+ if (is_gimple_call (first_def))
+ def_lhs = gimple_call_lhs (first_def);
+ else
+ def_lhs = gimple_assign_lhs (first_def);
- if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))
- /* Since we don't call this function with loads, this is a group of
- stores. */
- return;
+ if (operand_equal_p (oprnd, def_lhs, 0))
+ {
+ /* The number of vector defs is determined by the number of
+ vector statements in the node from which we get those
+ statements. */
+ number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (child);
+ vectorized_defs = true;
+ child_index++;
+ }
+ }
- /* For reductions, we only need initial values. */
- if (reduc_index != -1)
- return;
+ if (!vectorized_defs)
+ {
+ if (i == 0)
+ {
+ number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ /* Number of vector stmts was calculated according to LHS in
+ vect_schedule_slp_instance (), fix it by replacing LHS with
+ RHS, if necessary. See vect_get_smallest_scalar_type () for
+ details. */
+ vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
+ &rhs_size_unit);
+ if (rhs_size_unit != lhs_size_unit)
+ {
+ number_of_vects *= rhs_size_unit;
+ number_of_vects /= lhs_size_unit;
+ }
+ }
+ }
- code = gimple_assign_rhs_code (first_stmt);
- if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS || !vec_oprnds1 || !op1)
- return;
+ /* Allocate memory for vectorized defs. */
+ vec_defs = VEC_alloc (tree, heap, number_of_vects);
- /* The number of vector defs is determined by the number of vector statements
- in the node from which we get those statements. */
- if (SLP_TREE_RIGHT (slp_node))
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_RIGHT (slp_node));
- else
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ /* For reduction defs we call vect_get_constant_vectors (), since we are
+ looking for initial loop invariant values. */
+ if (vectorized_defs && reduc_index == -1)
+ /* The defs are already vectorized. */
+ vect_get_slp_vect_defs (child, &vec_defs);
+ else
+ /* Build vectors from scalar defs. */
+ vect_get_constant_vectors (oprnd, slp_node, &vec_defs, i,
+ number_of_vects, reduc_index);
- *vec_oprnds1 = VEC_alloc (tree, heap, number_of_vects);
+ VEC_quick_push (slp_void_p, *vec_oprnds, (slp_void_p) vec_defs);
- if (SLP_TREE_RIGHT (slp_node))
- /* The defs are already vectorized. */
- vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node), vec_oprnds1);
- else
- /* Build vectors from scalar defs. */
- vect_get_constant_vectors (op1, slp_node, vec_oprnds1, 1, number_of_vects,
- -1);
+ /* For reductions, we only need initial values. */
+ if (reduc_index != -1)
+ return;
+ }
}
tree vectype;
int i;
slp_tree loads_node;
+ slp_void_p child;
if (!node)
return false;
- vect_schedule_slp_instance (SLP_TREE_LEFT (node), instance,
- vectorization_factor);
- vect_schedule_slp_instance (SLP_TREE_RIGHT (node), instance,
- vectorization_factor);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_schedule_slp_instance ((slp_tree) child, instance,
+ vectorization_factor);
stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
stmt_info = vinfo_for_stmt (stmt);
&& REFERENCE_CLASS_P (gimple_get_lhs (stmt)))
{
gimple last_store = vect_find_last_store_in_slp_instance (instance);
+ if (is_pattern_stmt_p (vinfo_for_stmt (last_store)))
+ last_store = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (last_store));
si = gsi_for_stmt (last_store);
}
return is_store;
}
+/* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
+ For loop vectorization this is done in vectorizable_call, but for SLP
+ it needs to be deferred until end of vect_schedule_slp, because multiple
+ SLP instances may refer to the same scalar stmt. */
+
+static void
+vect_remove_slp_scalar_calls (slp_tree node)
+{
+ gimple stmt, new_stmt;
+ gimple_stmt_iterator gsi;
+ int i;
+ slp_void_p child;
+ tree lhs;
+ stmt_vec_info stmt_info;
+
+ if (!node)
+ return;
+
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_remove_slp_scalar_calls ((slp_tree) child);
+
+ FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt)
+ {
+ if (!is_gimple_call (stmt) || gimple_bb (stmt) == NULL)
+ continue;
+ stmt_info = vinfo_for_stmt (stmt);
+ if (stmt_info == NULL
+ || is_pattern_stmt_p (stmt_info)
+ || !PURE_SLP_STMT (stmt_info))
+ continue;
+ lhs = gimple_call_lhs (stmt);
+ new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
+ set_vinfo_for_stmt (new_stmt, stmt_info);
+ set_vinfo_for_stmt (stmt, NULL);
+ STMT_VINFO_STMT (stmt_info) = new_stmt;
+ gsi = gsi_for_stmt (stmt);
+ gsi_replace (&gsi, new_stmt, false);
+ SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
+ }
+}
/* Generate vector code for all SLP instances in the loop/basic block. */
unsigned int j;
gimple_stmt_iterator gsi;
+ vect_remove_slp_scalar_calls (root);
+
for (j = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (root), j, store)
&& j < SLP_INSTANCE_GROUP_SIZE (instance); j++)
{
if (!STMT_VINFO_DATA_REF (vinfo_for_stmt (store)))
break;
+ if (is_pattern_stmt_p (vinfo_for_stmt (store)))
+ store = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (store));
/* Free the attached stmt_vec_info and remove the stmt. */
gsi = gsi_for_stmt (store);
gsi_remove (&gsi, true);