/* SLP - Basic Block Vectorization
- Copyright (C) 2007, 2008, 2009, 2010
+ Copyright (C) 2007, 2008, 2009, 2010, 2011
Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "recog.h"
#include "optabs.h"
#include "tree-vectorizer.h"
+#include "langhooks.h"
/* Extract the location of the basic block in the source code.
Return the basic block location if succeed and NULL if not. */
static void
vect_free_slp_tree (slp_tree node)
{
+ int i;
+ slp_void_p child;
+
if (!node)
return;
- if (SLP_TREE_LEFT (node))
- vect_free_slp_tree (SLP_TREE_LEFT (node));
-
- if (SLP_TREE_RIGHT (node))
- vect_free_slp_tree (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_free_slp_tree ((slp_tree)child);
VEC_free (gimple, heap, SLP_TREE_SCALAR_STMTS (node));
}
-/* Get the defs for the rhs of STMT (collect them in DEF_STMTS0/1), check that
- they are of a legal type and that they match the defs of the first stmt of
- the SLP group (stored in FIRST_STMT_...). */
+/* Create an SLP node for SCALAR_STMTS. */
+
+static slp_tree
+vect_create_new_slp_node (VEC (gimple, heap) *scalar_stmts)
+{
+ slp_tree node = XNEW (struct _slp_tree);
+ gimple stmt = VEC_index (gimple, scalar_stmts, 0);
+ unsigned int nops;
+
+ if (is_gimple_call (stmt))
+ nops = gimple_call_num_args (stmt);
+ else if (is_gimple_assign (stmt))
+ nops = gimple_num_ops (stmt) - 1;
+ else
+ return NULL;
+
+ SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
+ SLP_TREE_VEC_STMTS (node) = NULL;
+ SLP_TREE_CHILDREN (node) = VEC_alloc (slp_void_p, heap, nops);
+ SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
+ SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
+
+ return node;
+}
+
+
+/* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
+ operand. */
+static VEC (slp_oprnd_info, heap) *
+vect_create_oprnd_info (int nops, int group_size)
+{
+ int i;
+ slp_oprnd_info oprnd_info;
+ VEC (slp_oprnd_info, heap) *oprnds_info;
+
+ oprnds_info = VEC_alloc (slp_oprnd_info, heap, nops);
+ for (i = 0; i < nops; i++)
+ {
+ oprnd_info = XNEW (struct _slp_oprnd_info);
+ oprnd_info->def_stmts = VEC_alloc (gimple, heap, group_size);
+ oprnd_info->first_dt = vect_uninitialized_def;
+ oprnd_info->first_def_type = NULL_TREE;
+ oprnd_info->first_const_oprnd = NULL_TREE;
+ oprnd_info->first_pattern = false;
+ VEC_quick_push (slp_oprnd_info, oprnds_info, oprnd_info);
+ }
+
+ return oprnds_info;
+}
+
+
+/* Free operands info. Free def-stmts in FREE_DEF_STMTS is true.
+ (FREE_DEF_STMTS is true when the SLP analysis fails, and false when it
+ succeds. In the later case we don't need the operands info that we used to
+ check isomorphism of the stmts, but we still need the def-stmts - they are
+ used as scalar stmts in SLP nodes. */
+static void
+vect_free_oprnd_info (VEC (slp_oprnd_info, heap) **oprnds_info,
+ bool free_def_stmts)
+{
+ int i;
+ slp_oprnd_info oprnd_info;
+
+ if (free_def_stmts)
+ FOR_EACH_VEC_ELT (slp_oprnd_info, *oprnds_info, i, oprnd_info)
+ VEC_free (gimple, heap, oprnd_info->def_stmts);
+
+ VEC_free (slp_oprnd_info, heap, *oprnds_info);
+}
+
+
+/* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
+ they are of a valid type and that they match the defs of the first stmt of
+ the SLP group (stored in OPRNDS_INFO). */
static bool
vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
slp_tree slp_node, gimple stmt,
- VEC (gimple, heap) **def_stmts0,
- VEC (gimple, heap) **def_stmts1,
- enum vect_def_type *first_stmt_dt0,
- enum vect_def_type *first_stmt_dt1,
- tree *first_stmt_def0_type,
- tree *first_stmt_def1_type,
- tree *first_stmt_const_oprnd,
- int ncopies_for_cost,
- bool *pattern0, bool *pattern1)
+ int ncopies_for_cost, bool first,
+ VEC (slp_oprnd_info, heap) **oprnds_info)
{
tree oprnd;
unsigned int i, number_of_oprnds;
- tree def;
+ tree def, def_op0 = NULL_TREE;
gimple def_stmt;
- enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
- stmt_vec_info stmt_info =
- vinfo_for_stmt (VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0));
- enum gimple_rhs_class rhs_class;
+ enum vect_def_type dt = vect_uninitialized_def;
+ enum vect_def_type dt_op0 = vect_uninitialized_def;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree lhs = gimple_get_lhs (stmt);
struct loop *loop = NULL;
+ enum tree_code rhs_code;
+ bool different_types = false;
+ bool pattern = false;
+ slp_oprnd_info oprnd_info, oprnd0_info, oprnd1_info;
if (loop_vinfo)
loop = LOOP_VINFO_LOOP (loop_vinfo);
- rhs_class = get_gimple_rhs_class (gimple_assign_rhs_code (stmt));
- number_of_oprnds = gimple_num_ops (stmt) - 1; /* RHS only */
+ if (is_gimple_call (stmt))
+ number_of_oprnds = gimple_call_num_args (stmt);
+ else
+ number_of_oprnds = gimple_num_ops (stmt) - 1;
for (i = 0; i < number_of_oprnds; i++)
{
oprnd = gimple_op (stmt, i + 1);
+ oprnd_info = VEC_index (slp_oprnd_info, *oprnds_info, i);
if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def,
- &dt[i])
- || (!def_stmt && dt[i] != vect_constant_def))
+ &dt)
+ || (!def_stmt && dt != vect_constant_def))
{
if (vect_print_dump_info (REPORT_SLP))
{
&& !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
&& !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt)))
{
- if (!*first_stmt_dt0)
- *pattern0 = true;
- else
- {
- if (i == 1 && !*first_stmt_dt1)
- *pattern1 = true;
- else if ((i == 0 && !*pattern0) || (i == 1 && !*pattern1))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "Build SLP failed: some of the stmts"
- " are in a pattern, and others are not ");
- print_generic_expr (vect_dump, oprnd, TDF_SLIM);
- }
+ pattern = true;
+ if (!first && !oprnd_info->first_pattern)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Build SLP failed: some of the stmts"
+ " are in a pattern, and others are not ");
+ print_generic_expr (vect_dump, oprnd, TDF_SLIM);
+ }
- return false;
- }
+ return false;
}
def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
- dt[i] = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
+ dt = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
- if (*dt == vect_unknown_def_type)
+ if (dt == vect_unknown_def_type
+ || STMT_VINFO_PATTERN_DEF_STMT (vinfo_for_stmt (def_stmt)))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Unsupported pattern.");
}
}
- if (!*first_stmt_dt0)
+ if (first)
{
- /* op0 of the first stmt of the group - store its info. */
- *first_stmt_dt0 = dt[i];
+ oprnd_info->first_dt = dt;
+ oprnd_info->first_pattern = pattern;
if (def)
- *first_stmt_def0_type = TREE_TYPE (def);
- else
- *first_stmt_const_oprnd = oprnd;
-
- /* Analyze costs (for the first stmt of the group only). */
- if (rhs_class != GIMPLE_SINGLE_RHS)
- /* Not memory operation (we don't call this functions for loads). */
- vect_model_simple_cost (stmt_info, ncopies_for_cost, dt, slp_node);
+ {
+ oprnd_info->first_def_type = TREE_TYPE (def);
+ oprnd_info->first_const_oprnd = NULL_TREE;
+ }
else
- /* Store. */
- vect_model_store_cost (stmt_info, ncopies_for_cost, false,
- dt[0], slp_node);
- }
+ {
+ oprnd_info->first_def_type = NULL_TREE;
+ oprnd_info->first_const_oprnd = oprnd;
+ }
- else
- {
- if (!*first_stmt_dt1 && i == 1)
+ if (i == 0)
{
- /* op1 of the first stmt of the group - store its info. */
- *first_stmt_dt1 = dt[i];
- if (def)
- *first_stmt_def1_type = TREE_TYPE (def);
+ def_op0 = def;
+ dt_op0 = dt;
+ /* Analyze costs (for the first stmt of the group only). */
+ if (REFERENCE_CLASS_P (lhs))
+ /* Store. */
+ vect_model_store_cost (stmt_info, ncopies_for_cost, false,
+ dt, slp_node);
else
- {
- /* We assume that the stmt contains only one constant
- operand. We fail otherwise, to be on the safe side. */
- if (*first_stmt_const_oprnd)
- {
- if (vect_print_dump_info (REPORT_SLP))
- fprintf (vect_dump, "Build SLP failed: two constant "
- "oprnds in stmt");
- return false;
- }
- *first_stmt_const_oprnd = oprnd;
- }
+ /* Not memory operation (we don't call this function for
+ loads). */
+ vect_model_simple_cost (stmt_info, ncopies_for_cost, &dt,
+ slp_node);
}
- else
+ }
+ else
+ {
+ /* Not first stmt of the group, check that the def-stmt/s match
+ the def-stmt/s of the first stmt. Allow different definition
+ types for reduction chains: the first stmt must be a
+ vect_reduction_def (a phi node), and the rest
+ vect_internal_def. */
+ if (((oprnd_info->first_dt != dt
+ && !(oprnd_info->first_dt == vect_reduction_def
+ && dt == vect_internal_def))
+ || (oprnd_info->first_def_type != NULL_TREE
+ && def
+ && !types_compatible_p (oprnd_info->first_def_type,
+ TREE_TYPE (def))))
+ || (!def
+ && !types_compatible_p (TREE_TYPE (oprnd_info->first_const_oprnd),
+ TREE_TYPE (oprnd)))
+ || different_types)
{
- /* Not first stmt of the group, check that the def-stmt/s match
- the def-stmt/s of the first stmt. Allow different definition
- types for reduction chains: the first stmt must be a
- vect_reduction_def (a phi node), and the rest
- vect_internal_def. */
- if ((i == 0
- && ((*first_stmt_dt0 != dt[i]
- && !(*first_stmt_dt0 == vect_reduction_def
- && dt[i] == vect_internal_def))
- || (*first_stmt_def0_type && def
- && !types_compatible_p (*first_stmt_def0_type,
- TREE_TYPE (def)))))
- || (i == 1
- && ((*first_stmt_dt1 != dt[i]
- && !(*first_stmt_dt1 == vect_reduction_def
- && dt[i] == vect_internal_def))
- || (*first_stmt_def1_type && def
- && !types_compatible_p (*first_stmt_def1_type,
- TREE_TYPE (def)))))
- || (!def
- && !types_compatible_p (TREE_TYPE (*first_stmt_const_oprnd),
- TREE_TYPE (oprnd))))
+ if (number_of_oprnds != 2)
{
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: different types ");
return false;
+ }
+
+ /* Try to swap operands in case of binary operation. */
+ if (i == 0)
+ different_types = true;
+ else
+ {
+ oprnd0_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ if (is_gimple_assign (stmt)
+ && (rhs_code = gimple_assign_rhs_code (stmt))
+ && TREE_CODE_CLASS (rhs_code) == tcc_binary
+ && commutative_tree_code (rhs_code)
+ && oprnd0_info->first_dt == dt
+ && oprnd_info->first_dt == dt_op0
+ && def_op0 && def
+ && !(oprnd0_info->first_def_type
+ && !types_compatible_p (oprnd0_info->first_def_type,
+ TREE_TYPE (def)))
+ && !(oprnd_info->first_def_type
+ && !types_compatible_p (oprnd_info->first_def_type,
+ TREE_TYPE (def_op0))))
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump, "Swapping operands of ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ swap_tree_operands (stmt, gimple_assign_rhs1_ptr (stmt),
+ gimple_assign_rhs2_ptr (stmt));
+ }
+ else
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ fprintf (vect_dump, "Build SLP failed: different types ");
+
+ return false;
+ }
}
}
}
/* Check the types of the definitions. */
- switch (dt[i])
+ switch (dt)
{
case vect_constant_def:
case vect_external_def:
+ case vect_reduction_def:
break;
case vect_internal_def:
- case vect_reduction_def:
- if (i == 0)
- VEC_safe_push (gimple, heap, *def_stmts0, def_stmt);
+ if (different_types)
+ {
+ oprnd0_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ oprnd1_info = VEC_index (slp_oprnd_info, *oprnds_info, 0);
+ if (i == 0)
+ VEC_quick_push (gimple, oprnd1_info->def_stmts, def_stmt);
+ else
+ VEC_quick_push (gimple, oprnd0_info->def_stmts, def_stmt);
+ }
else
- VEC_safe_push (gimple, heap, *def_stmts1, def_stmt);
+ VEC_quick_push (gimple, oprnd_info->def_stmts, def_stmt);
+
break;
default:
int ncopies_for_cost, unsigned int *max_nunits,
VEC (int, heap) **load_permutation,
VEC (slp_tree, heap) **loads,
- unsigned int vectorization_factor)
+ unsigned int vectorization_factor, bool *loads_permuted)
{
- VEC (gimple, heap) *def_stmts0 = VEC_alloc (gimple, heap, group_size);
- VEC (gimple, heap) *def_stmts1 = VEC_alloc (gimple, heap, group_size);
unsigned int i;
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
gimple stmt = VEC_index (gimple, stmts, 0);
- enum vect_def_type first_stmt_dt0 = vect_uninitialized_def;
- enum vect_def_type first_stmt_dt1 = vect_uninitialized_def;
enum tree_code first_stmt_code = ERROR_MARK, rhs_code = ERROR_MARK;
- tree first_stmt_def1_type = NULL_TREE, first_stmt_def0_type = NULL_TREE;
tree lhs;
bool stop_recursion = false, need_same_oprnds = false;
tree vectype, scalar_type, first_op1 = NULL_TREE;
int icode;
enum machine_mode optab_op2_mode;
enum machine_mode vec_mode;
- tree first_stmt_const_oprnd = NULL_TREE;
struct data_reference *first_dr;
- bool pattern0 = false, pattern1 = false;
HOST_WIDE_INT dummy;
bool permutation = false;
unsigned int load_place;
gimple first_load, prev_first_load = NULL;
+ VEC (slp_oprnd_info, heap) *oprnds_info;
+ unsigned int nops;
+ slp_oprnd_info oprnd_info;
+
+ if (is_gimple_call (stmt))
+ nops = gimple_call_num_args (stmt);
+ else
+ nops = gimple_num_ops (stmt) - 1;
+
+ oprnds_info = vect_create_oprnd_info (nops, group_size);
/* For every stmt in NODE find its def stmt/s. */
FOR_EACH_VEC_ELT (gimple, stmts, i, stmt)
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
if (vect_print_dump_info (REPORT_SLP))
{
fprintf (vect_dump,
- "Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL");
+ "Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
}
+
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
- ncopies = vectorization_factor / TYPE_VECTOR_SUBPARTS (vectype);
- if (ncopies != 1)
+ /* In case of multiple types we need to detect the smallest type. */
+ if (*max_nunits < TYPE_VECTOR_SUBPARTS (vectype))
{
- if (vect_print_dump_info (REPORT_SLP))
- fprintf (vect_dump, "SLP with multiple types ");
-
- /* FORNOW: multiple types are unsupported in BB SLP. */
- if (bb_vinfo)
- return false;
+ *max_nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (bb_vinfo)
+ vectorization_factor = *max_nunits;
}
- /* In case of multiple types we need to detect the smallest type. */
- if (*max_nunits < TYPE_VECTOR_SUBPARTS (vectype))
- *max_nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ ncopies = vectorization_factor / TYPE_VECTOR_SUBPARTS (vectype);
if (is_gimple_call (stmt))
rhs_code = CALL_EXPR;
{
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: no optab.");
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
icode = (int) optab_handler (optab, vec_mode);
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: "
"op not supported by target.");
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
optab_op2_mode = insn_data[icode].operand[2].mode;
}
}
}
+ else if (rhs_code == WIDEN_LSHIFT_EXPR)
+ {
+ need_same_oprnds = true;
+ first_op1 = gimple_assign_rhs2 (stmt);
+ }
}
else
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
}
{
/* Store. */
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node,
- stmt, &def_stmts0, &def_stmts1,
- &first_stmt_dt0,
- &first_stmt_dt1,
- &first_stmt_def0_type,
- &first_stmt_def1_type,
- &first_stmt_const_oprnd,
- ncopies_for_cost,
- &pattern0, &pattern1))
- return false;
+ stmt, ncopies_for_cost,
+ (i == 0), &oprnds_info))
+ {
+ vect_free_oprnd_info (&oprnds_info, true);
+ return false;
+ }
}
else
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
/* Check that the size of interleaved loads group is not
greater than the SLP group size. */
- if (GROUP_SIZE (vinfo_for_stmt (stmt)) > ncopies * group_size)
+ if (loop_vinfo
+ && GROUP_SIZE (vinfo_for_stmt (stmt)) > ncopies * group_size)
{
if (vect_print_dump_info (REPORT_SLP))
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
{
if (TREE_CODE_CLASS (rhs_code) == tcc_reference)
{
- /* Not strided load. */
+ /* Not strided load. */
if (vect_print_dump_info (REPORT_SLP))
{
fprintf (vect_dump, "Build SLP failed: not strided load ");
}
/* FORNOW: Not strided loads are not supported. */
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
+ vect_free_oprnd_info (&oprnds_info, true);
return false;
}
/* Find the def-stmts. */
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
- &def_stmts0, &def_stmts1,
- &first_stmt_dt0, &first_stmt_dt1,
- &first_stmt_def0_type,
- &first_stmt_def1_type,
- &first_stmt_const_oprnd,
- ncopies_for_cost,
- &pattern0, &pattern1))
- return false;
+ ncopies_for_cost, (i == 0),
+ &oprnds_info))
+ {
+ vect_free_oprnd_info (&oprnds_info, true);
+ return false;
+ }
}
}
/* Strided loads were reached - stop the recursion. */
if (stop_recursion)
{
+ VEC_safe_push (slp_tree, heap, *loads, *node);
if (permutation)
{
- VEC_safe_push (slp_tree, heap, *loads, *node);
+
+ *loads_permuted = true;
*inside_cost
+= targetm.vectorize.builtin_vectorization_cost (vec_perm, NULL, 0)
* group_size;
}
else
- {
- /* We don't check here complex numbers chains, so we keep them in
- LOADS for further check in vect_supported_load_permutation_p. */
+ {
+ /* We don't check here complex numbers chains, so we set
+ LOADS_PERMUTED for further check in
+ vect_supported_load_permutation_p. */
if (rhs_code == REALPART_EXPR || rhs_code == IMAGPART_EXPR)
- VEC_safe_push (slp_tree, heap, *loads, *node);
+ *loads_permuted = true;
}
return true;
}
/* Create SLP_TREE nodes for the definition node/s. */
- if (first_stmt_dt0 == vect_internal_def)
- {
- slp_tree left_node = XNEW (struct _slp_tree);
- SLP_TREE_SCALAR_STMTS (left_node) = def_stmts0;
- SLP_TREE_VEC_STMTS (left_node) = NULL;
- SLP_TREE_LEFT (left_node) = NULL;
- SLP_TREE_RIGHT (left_node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (left_node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (left_node) = 0;
- if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &left_node, group_size,
- inside_cost, outside_cost, ncopies_for_cost,
- max_nunits, load_permutation, loads,
- vectorization_factor))
- return false;
+ FOR_EACH_VEC_ELT (slp_oprnd_info, oprnds_info, i, oprnd_info)
+ {
+ slp_tree child;
- SLP_TREE_LEFT (*node) = left_node;
- }
+ if (oprnd_info->first_dt != vect_internal_def)
+ continue;
- if (first_stmt_dt1 == vect_internal_def)
- {
- slp_tree right_node = XNEW (struct _slp_tree);
- SLP_TREE_SCALAR_STMTS (right_node) = def_stmts1;
- SLP_TREE_VEC_STMTS (right_node) = NULL;
- SLP_TREE_LEFT (right_node) = NULL;
- SLP_TREE_RIGHT (right_node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (right_node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (right_node) = 0;
- if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &right_node, group_size,
+ child = vect_create_new_slp_node (oprnd_info->def_stmts);
+ if (!child
+ || !vect_build_slp_tree (loop_vinfo, bb_vinfo, &child, group_size,
inside_cost, outside_cost, ncopies_for_cost,
max_nunits, load_permutation, loads,
- vectorization_factor))
- return false;
+ vectorization_factor, loads_permuted))
+ {
+ free (child);
+ vect_free_oprnd_info (&oprnds_info, true);
+ return false;
+ }
- SLP_TREE_RIGHT (*node) = right_node;
+ VEC_quick_push (slp_void_p, SLP_TREE_CHILDREN (*node), child);
}
+ vect_free_oprnd_info (&oprnds_info, false);
return true;
}
{
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return;
}
fprintf (vect_dump, "\n");
- vect_print_slp_tree (SLP_TREE_LEFT (node));
- vect_print_slp_tree (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_print_slp_tree ((slp_tree) child);
}
{
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return;
if (j < 0 || i == j)
STMT_SLP_TYPE (vinfo_for_stmt (stmt)) = mark;
- vect_mark_slp_stmts (SLP_TREE_LEFT (node), mark, j);
- vect_mark_slp_stmts (SLP_TREE_RIGHT (node), mark, j);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_mark_slp_stmts ((slp_tree) child, mark, j);
}
int i;
gimple stmt;
stmt_vec_info stmt_info;
+ slp_void_p child;
if (!node)
return;
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
}
- vect_mark_slp_stmts_relevant (SLP_TREE_LEFT (node));
- vect_mark_slp_stmts_relevant (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_mark_slp_stmts_relevant ((slp_tree) child);
}
gimple stmt;
VEC (gimple, heap) *tmp_stmts;
unsigned int index, i;
+ slp_void_p child;
if (!node)
return;
- vect_slp_rearrange_stmts (SLP_TREE_LEFT (node), group_size, permutation);
- vect_slp_rearrange_stmts (SLP_TREE_RIGHT (node), group_size, permutation);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_slp_rearrange_stmts ((slp_tree) child, group_size, permutation);
gcc_assert (group_size == VEC_length (gimple, SLP_TREE_SCALAR_STMTS (node)));
tmp_stmts = VEC_alloc (gimple, heap, group_size);
bool supported, bad_permutation = false;
sbitmap load_index;
slp_tree node, other_complex_node;
- gimple stmt, first = NULL, other_node_first;
+ gimple stmt, first = NULL, other_node_first, load, next_load, first_load;
unsigned complex_numbers = 0;
+ struct data_reference *dr;
+ bb_vec_info bb_vinfo;
/* FORNOW: permutations are only supported in SLP. */
if (!slp_instn)
}
}
+ /* In basic block vectorization we allow any subchain of an interleaving
+ chain.
+ FORNOW: not supported in loop SLP because of realignment compications. */
+ bb_vinfo = STMT_VINFO_BB_VINFO (vinfo_for_stmt (stmt));
+ bad_permutation = false;
+ /* Check that for every node in the instance teh loads form a subchain. */
+ if (bb_vinfo)
+ {
+ FOR_EACH_VEC_ELT (slp_tree, SLP_INSTANCE_LOADS (slp_instn), i, node)
+ {
+ next_load = NULL;
+ first_load = NULL;
+ FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), j, load)
+ {
+ if (!first_load)
+ first_load = GROUP_FIRST_ELEMENT (vinfo_for_stmt (load));
+ else if (first_load
+ != GROUP_FIRST_ELEMENT (vinfo_for_stmt (load)))
+ {
+ bad_permutation = true;
+ break;
+ }
+
+ if (j != 0 && next_load != load)
+ {
+ bad_permutation = true;
+ break;
+ }
+
+ next_load = GROUP_NEXT_ELEMENT (vinfo_for_stmt (load));
+ }
+
+ if (bad_permutation)
+ break;
+ }
+
+ /* Check that the alignment of the first load in every subchain, i.e.,
+ the first statement in every load node, is supported. */
+ if (!bad_permutation)
+ {
+ FOR_EACH_VEC_ELT (slp_tree, SLP_INSTANCE_LOADS (slp_instn), i, node)
+ {
+ first_load = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
+ if (first_load
+ != GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_load)))
+ {
+ dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_load));
+ if (vect_supportable_dr_alignment (dr, false)
+ == dr_unaligned_unsupported)
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ {
+ fprintf (vect_dump, "unsupported unaligned load ");
+ print_gimple_stmt (vect_dump, first_load, 0,
+ TDF_SLIM);
+ }
+ bad_permutation = true;
+ break;
+ }
+ }
+ }
+
+ if (!bad_permutation)
+ {
+ VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (slp_instn));
+ return true;
+ }
+ }
+ }
+
/* FORNOW: the only supported permutation is 0..01..1.. of length equal to
GROUP_SIZE and where each sequence of same drs is of GROUP_SIZE length as
well (unless it's reduction). */
gimple stmt)
{
slp_instance new_instance;
- slp_tree node = XNEW (struct _slp_tree);
+ slp_tree node;
unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (stmt));
unsigned int unrolling_factor = 1, nunits;
tree vectype, scalar_type = NULL_TREE;
VEC (int, heap) *load_permutation;
VEC (slp_tree, heap) *loads;
struct data_reference *dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
+ bool loads_permuted = false;
+ VEC (gimple, heap) *scalar_stmts;
if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)))
{
if (loop_vinfo)
vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
else
- /* No multitypes in BB SLP. */
vectorization_factor = nunits;
/* Calculate the unrolling factor. */
}
/* Create a node (a root of the SLP tree) for the packed strided stores. */
- SLP_TREE_SCALAR_STMTS (node) = VEC_alloc (gimple, heap, group_size);
+ scalar_stmts = VEC_alloc (gimple, heap, group_size);
next = stmt;
if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)))
{
/* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
while (next)
{
- VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
+ VEC_safe_push (gimple, heap, scalar_stmts, next);
next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next));
}
}
else
{
/* Collect reduction statements. */
- for (i = 0; VEC_iterate (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i,
- next);
- i++)
- VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
+ VEC (gimple, heap) *reductions = LOOP_VINFO_REDUCTIONS (loop_vinfo);
+ for (i = 0; VEC_iterate (gimple, reductions, i, next); i++)
+ VEC_safe_push (gimple, heap, scalar_stmts, next);
}
- SLP_TREE_VEC_STMTS (node) = NULL;
- SLP_TREE_NUMBER_OF_VEC_STMTS (node) = 0;
- SLP_TREE_LEFT (node) = NULL;
- SLP_TREE_RIGHT (node) = NULL;
- SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
- SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
+ node = vect_create_new_slp_node (scalar_stmts);
/* Calculate the number of vector stmts to create based on the unrolling
factor (number of vectors is 1 if NUNITS >= GROUP_SIZE, and is
if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &node, group_size,
&inside_cost, &outside_cost, ncopies_for_cost,
&max_nunits, &load_permutation, &loads,
- vectorization_factor))
+ vectorization_factor, &loads_permuted))
{
- /* Create a new SLP instance. */
- new_instance = XNEW (struct _slp_instance);
- SLP_INSTANCE_TREE (new_instance) = node;
- SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
- /* Calculate the unrolling factor based on the smallest type in the
- loop. */
+ /* Calculate the unrolling factor based on the smallest type. */
if (max_nunits > nunits)
unrolling_factor = least_common_multiple (max_nunits, group_size)
/ group_size;
+ if (unrolling_factor != 1 && !loop_vinfo)
+ {
+ if (vect_print_dump_info (REPORT_SLP))
+ fprintf (vect_dump, "Build SLP failed: unrolling required in basic"
+ " block SLP");
+ return false;
+ }
+
+ /* Create a new SLP instance. */
+ new_instance = XNEW (struct _slp_instance);
+ SLP_INSTANCE_TREE (new_instance) = node;
+ SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance) = outside_cost;
SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance) = inside_cost;
SLP_INSTANCE_LOADS (new_instance) = loads;
SLP_INSTANCE_FIRST_LOAD_STMT (new_instance) = NULL;
SLP_INSTANCE_LOAD_PERMUTATION (new_instance) = load_permutation;
- if (VEC_length (slp_tree, loads))
+
+ if (loads_permuted)
{
if (!vect_supported_load_permutation_p (new_instance, group_size,
load_permutation))
imm_use_iterator imm_iter;
gimple use_stmt;
stmt_vec_info stmt_vinfo;
+ slp_void_p child;
if (!node)
return;
== vect_reduction_def))
vect_mark_slp_stmts (node, hybrid, i);
- vect_detect_hybrid_slp_stmts (SLP_TREE_LEFT (node));
- vect_detect_hybrid_slp_stmts (SLP_TREE_RIGHT (node));
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_detect_hybrid_slp_stmts ((slp_tree) child);
}
bool dummy;
int i;
gimple stmt;
+ slp_void_p child;
if (!node)
return true;
- if (!vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_LEFT (node))
- || !vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_RIGHT (node)))
- return false;
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ if (!vect_slp_analyze_node_operations (bb_vinfo, (slp_tree) child))
+ return false;
FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt)
{
return true;
}
-/* Check if loads and stores are mixed in the basic block (in that
- case if we are not sure that the accesses differ, we can't vectorize the
- basic block). Also return FALSE in case that there is statement marked as
- not vectorizable. */
-
-static bool
-vect_bb_vectorizable_with_dependencies (bb_vec_info bb_vinfo)
-{
- basic_block bb = BB_VINFO_BB (bb_vinfo);
- gimple_stmt_iterator si;
- bool detected_store = false;
- gimple stmt;
- struct data_reference *dr;
-
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
- {
- stmt = gsi_stmt (si);
-
- /* We can't allow not analyzed statements, since they may contain data
- accesses. */
- if (!STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
- return false;
-
- if (!STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)))
- continue;
-
- dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
- if (DR_IS_READ (dr) && detected_store)
- return false;
-
- if (!DR_IS_READ (dr))
- detected_store = true;
- }
-
- return true;
-}
-
/* Check if vectorization of the basic block is profitable. */
static bool
int i;
int min_vf = 2;
int max_vf = MAX_VECTORIZATION_FACTOR;
- bool data_dependence_in_bb = false;
bb_vinfo = new_bb_vec_info (bb);
if (!bb_vinfo)
return NULL;
}
- if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf,
- &data_dependence_in_bb)
- || min_vf > max_vf
- || (data_dependence_in_bb
- && !vect_bb_vectorizable_with_dependencies (bb_vinfo)))
+ if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf)
+ || min_vf > max_vf)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unhandled data dependence "
If the scalar definitions are loop invariants or constants, collect them and
call vect_get_constant_vectors() to create vector stmts.
Otherwise, the def-stmts must be already vectorized and the vectorized stmts
- must be stored in the LEFT/RIGHT node of SLP_NODE, and we call
- vect_get_slp_vect_defs() to retrieve them.
- If VEC_OPRNDS1 is NULL, don't get vector defs for the second operand (from
- the right node. This is used when the second operand must remain scalar. */
+ must be stored in the corresponding child of SLP_NODE, and we call
+ vect_get_slp_vect_defs () to retrieve them. */
void
-vect_get_slp_defs (tree op0, tree op1, slp_tree slp_node,
- VEC (tree,heap) **vec_oprnds0,
- VEC (tree,heap) **vec_oprnds1, int reduc_index)
+vect_get_slp_defs (VEC (tree, heap) *ops, slp_tree slp_node,
+ VEC (slp_void_p, heap) **vec_oprnds, int reduc_index)
{
- gimple first_stmt;
- enum tree_code code;
- int number_of_vects;
+ gimple first_stmt, first_def;
+ int number_of_vects = 0, i;
+ unsigned int child_index = 0;
HOST_WIDE_INT lhs_size_unit, rhs_size_unit;
+ slp_tree child = NULL;
+ VEC (tree, heap) *vec_defs;
+ tree oprnd, def_lhs;
+ bool vectorized_defs;
first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
- /* The number of vector defs is determined by the number of vector statements
- in the node from which we get those statements. */
- if (SLP_TREE_LEFT (slp_node))
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node));
- else
- {
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
- /* Number of vector stmts was calculated according to LHS in
- vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
- necessary. See vect_get_smallest_scalar_type () for details. */
- vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
- &rhs_size_unit);
- if (rhs_size_unit != lhs_size_unit)
+ FOR_EACH_VEC_ELT (tree, ops, i, oprnd)
+ {
+ /* For each operand we check if it has vectorized definitions in a child
+ node or we need to create them (for invariants and constants). We
+ check if the LHS of the first stmt of the next child matches OPRND.
+ If it does, we found the correct child. Otherwise, we call
+ vect_get_constant_vectors (), and not advance CHILD_INDEX in order
+ to check this child node for the next operand. */
+ vectorized_defs = false;
+ if (VEC_length (slp_void_p, SLP_TREE_CHILDREN (slp_node)) > child_index)
{
- number_of_vects *= rhs_size_unit;
- number_of_vects /= lhs_size_unit;
- }
- }
-
- /* Allocate memory for vectorized defs. */
- *vec_oprnds0 = VEC_alloc (tree, heap, number_of_vects);
-
- /* SLP_NODE corresponds either to a group of stores or to a group of
- unary/binary operations. We don't call this function for loads.
- For reduction defs we call vect_get_constant_vectors(), since we are
- looking for initial loop invariant values. */
- if (SLP_TREE_LEFT (slp_node) && reduc_index == -1)
- /* The defs are already vectorized. */
- vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node), vec_oprnds0);
- else
- /* Build vectors from scalar defs. */
- vect_get_constant_vectors (op0, slp_node, vec_oprnds0, 0, number_of_vects,
- reduc_index);
+ child = (slp_tree) VEC_index (slp_void_p,
+ SLP_TREE_CHILDREN (slp_node),
+ child_index);
+ first_def = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (child), 0);
+
+ /* In the end of a pattern sequence we have a use of the original stmt,
+ so we need to compare OPRND with the original def. */
+ if (is_pattern_stmt_p (vinfo_for_stmt (first_def))
+ && !STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (first_stmt))
+ && !is_pattern_stmt_p (vinfo_for_stmt (first_stmt)))
+ first_def = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (first_def));
+
+ if (is_gimple_call (first_def))
+ def_lhs = gimple_call_lhs (first_def);
+ else
+ def_lhs = gimple_assign_lhs (first_def);
- if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))
- /* Since we don't call this function with loads, this is a group of
- stores. */
- return;
+ if (operand_equal_p (oprnd, def_lhs, 0))
+ {
+ /* The number of vector defs is determined by the number of
+ vector statements in the node from which we get those
+ statements. */
+ number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (child);
+ vectorized_defs = true;
+ child_index++;
+ }
+ }
- /* For reductions, we only need initial values. */
- if (reduc_index != -1)
- return;
+ if (!vectorized_defs)
+ {
+ if (i == 0)
+ {
+ number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ /* Number of vector stmts was calculated according to LHS in
+ vect_schedule_slp_instance (), fix it by replacing LHS with
+ RHS, if necessary. See vect_get_smallest_scalar_type () for
+ details. */
+ vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
+ &rhs_size_unit);
+ if (rhs_size_unit != lhs_size_unit)
+ {
+ number_of_vects *= rhs_size_unit;
+ number_of_vects /= lhs_size_unit;
+ }
+ }
+ }
- code = gimple_assign_rhs_code (first_stmt);
- if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS || !vec_oprnds1 || !op1)
- return;
+ /* Allocate memory for vectorized defs. */
+ vec_defs = VEC_alloc (tree, heap, number_of_vects);
- /* The number of vector defs is determined by the number of vector statements
- in the node from which we get those statements. */
- if (SLP_TREE_RIGHT (slp_node))
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_RIGHT (slp_node));
- else
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ /* For reduction defs we call vect_get_constant_vectors (), since we are
+ looking for initial loop invariant values. */
+ if (vectorized_defs && reduc_index == -1)
+ /* The defs are already vectorized. */
+ vect_get_slp_vect_defs (child, &vec_defs);
+ else
+ /* Build vectors from scalar defs. */
+ vect_get_constant_vectors (oprnd, slp_node, &vec_defs, i,
+ number_of_vects, reduc_index);
- *vec_oprnds1 = VEC_alloc (tree, heap, number_of_vects);
+ VEC_quick_push (slp_void_p, *vec_oprnds, (slp_void_p) vec_defs);
- if (SLP_TREE_RIGHT (slp_node))
- /* The defs are already vectorized. */
- vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node), vec_oprnds1);
- else
- /* Build vectors from scalar defs. */
- vect_get_constant_vectors (op1, slp_node, vec_oprnds1, 1, number_of_vects,
- -1);
+ /* For reductions, we only need initial values. */
+ if (reduc_index != -1)
+ return;
+ }
}
vect_create_mask_and_perm (gimple stmt, gimple next_scalar_stmt,
tree mask, int first_vec_indx, int second_vec_indx,
gimple_stmt_iterator *gsi, slp_tree node,
- tree builtin_decl, tree vectype,
- VEC(tree,heap) *dr_chain,
+ tree vectype, VEC(tree,heap) *dr_chain,
int ncopies, int vect_stmts_counter)
{
tree perm_dest;
second_vec = VEC_index (tree, dr_chain, second_vec_indx);
/* Generate the permute statement. */
- perm_stmt = gimple_build_call (builtin_decl,
- 3, first_vec, second_vec, mask);
+ perm_stmt = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, perm_dest,
+ first_vec, second_vec, mask);
data_ref = make_ssa_name (perm_dest, perm_stmt);
- gimple_call_set_lhs (perm_stmt, data_ref);
+ gimple_set_lhs (perm_stmt, data_ref);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
/* Store the vector statement in NODE. */
static bool
vect_get_mask_element (gimple stmt, int first_mask_element, int m,
int mask_nunits, bool only_one_vec, int index,
- int *mask, int *current_mask_element,
+ unsigned char *mask, int *current_mask_element,
bool *need_next_vector, int *number_of_mask_fixes,
bool *mask_fixed, bool *needs_first_vector)
{
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree mask_element_type = NULL_TREE, mask_type;
- int i, j, k, m, scale, mask_nunits, nunits, vec_index = 0, scalar_index;
+ int i, j, k, nunits, vec_index = 0, scalar_index;
slp_tree node;
- tree vectype = STMT_VINFO_VECTYPE (stmt_info), builtin_decl;
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
gimple next_scalar_stmt;
int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
int first_mask_element;
- int index, unroll_factor, *mask, current_mask_element, ncopies;
+ int index, unroll_factor, current_mask_element, ncopies;
+ unsigned char *mask;
bool only_one_vec = false, need_next_vector = false;
int first_vec_index, second_vec_index, orig_vec_stmts_num, vect_stmts_counter;
int number_of_mask_fixes = 1;
bool mask_fixed = false;
bool needs_first_vector = false;
+ enum machine_mode mode;
- if (!targetm.vectorize.builtin_vec_perm)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "no builtin for vect permute for ");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
-
- return false;
- }
+ mode = TYPE_MODE (vectype);
- builtin_decl = targetm.vectorize.builtin_vec_perm (vectype,
- &mask_element_type);
- if (!builtin_decl || !mask_element_type)
+ if (!can_vec_perm_p (mode, false, NULL))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
- fprintf (vect_dump, "no builtin for vect permute for ");
+ fprintf (vect_dump, "no vect permute for ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
-
- return false;
+ return false;
}
+ /* The generic VEC_PERM_EXPR code always uses an integral type of the
+ same size as the vector element being permuted. */
+ mask_element_type
+ = lang_hooks.types.type_for_size
+ (TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (vectype))), 1);
mask_type = get_vectype_for_scalar_type (mask_element_type);
- mask_nunits = TYPE_VECTOR_SUBPARTS (mask_type);
- mask = (int *) xmalloc (sizeof (int) * mask_nunits);
nunits = TYPE_VECTOR_SUBPARTS (vectype);
- scale = mask_nunits / nunits;
+ mask = XALLOCAVEC (unsigned char, nunits);
unroll_factor = SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance);
/* The number of vector stmts to generate based only on SLP_NODE_INSTANCE
for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
...
- The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9} (in target
- scpecific type, e.g., in bytes for Altivec.
+ The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
The last mask is illegal since we assume two operands for permute
operation, and the mask element values can't be outside that range.
Hence, the last mask must be converted into {2,5,5,5}.
{
for (k = 0; k < group_size; k++)
{
- first_mask_element = (i + j * group_size) * scale;
- for (m = 0; m < scale; m++)
- {
- if (!vect_get_mask_element (stmt, first_mask_element, m,
- mask_nunits, only_one_vec, index, mask,
- ¤t_mask_element, &need_next_vector,
- &number_of_mask_fixes, &mask_fixed,
- &needs_first_vector))
- return false;
-
- mask[index++] = current_mask_element;
- }
+ first_mask_element = i + j * group_size;
+ if (!vect_get_mask_element (stmt, first_mask_element, 0,
+ nunits, only_one_vec, index,
+ mask, ¤t_mask_element,
+ &need_next_vector,
+ &number_of_mask_fixes, &mask_fixed,
+ &needs_first_vector))
+ return false;
+ mask[index++] = current_mask_element;
- if (index == mask_nunits)
+ if (index == nunits)
{
tree mask_vec = NULL;
+ if (!can_vec_perm_p (mode, false, mask))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "unsupported vect permute { ");
+ for (i = 0; i < nunits; ++i)
+ fprintf (vect_dump, "%d ", mask[i]);
+ fprintf (vect_dump, "}\n");
+ }
+ return false;
+ }
+
while (--index >= 0)
{
tree t = build_int_cst (mask_element_type, mask[index]);
mask_vec = build_vector (mask_type, mask_vec);
index = 0;
- if (!targetm.vectorize.builtin_vec_perm_ok (vectype,
- mask_vec))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "unsupported vect permute ");
- print_generic_expr (vect_dump, mask_vec, 0);
- }
- free (mask);
- return false;
- }
-
if (!analyze_only)
{
if (need_next_vector)
vect_create_mask_and_perm (stmt, next_scalar_stmt,
mask_vec, first_vec_index, second_vec_index,
- gsi, node, builtin_decl, vectype, dr_chain,
+ gsi, node, vectype, dr_chain,
ncopies, vect_stmts_counter++);
}
}
}
}
- free (mask);
return true;
}
tree vectype;
int i;
slp_tree loads_node;
+ slp_void_p child;
if (!node)
return false;
- vect_schedule_slp_instance (SLP_TREE_LEFT (node), instance,
- vectorization_factor);
- vect_schedule_slp_instance (SLP_TREE_RIGHT (node), instance,
- vectorization_factor);
+ FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
+ vect_schedule_slp_instance ((slp_tree) child, instance,
+ vectorization_factor);
stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
stmt_info = vinfo_for_stmt (stmt);
/* Loads should be inserted before the first load. */
if (SLP_INSTANCE_FIRST_LOAD_STMT (instance)
&& STMT_VINFO_STRIDED_ACCESS (stmt_info)
- && !REFERENCE_CLASS_P (gimple_get_lhs (stmt)))
+ && !REFERENCE_CLASS_P (gimple_get_lhs (stmt))
+ && SLP_INSTANCE_LOAD_PERMUTATION (instance))
si = gsi_for_stmt (SLP_INSTANCE_FIRST_LOAD_STMT (instance));
else if (is_pattern_stmt_p (stmt_info))
- si = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
+ si = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
else
si = gsi_for_stmt (stmt);