#include "recog.h"
#include "optabs.h"
#include "diagnostic-core.h"
-#include "toplev.h"
#include "tree-vectorizer.h"
#include "langhooks.h"
gimple def_stmt;
stmt_vec_info def_stmt_info = NULL;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
- tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
- unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ unsigned int nunits;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
tree vec_inv;
tree vec_cst;
{
vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
gcc_assert (vector_type);
+ nunits = TYPE_VECTOR_SUBPARTS (vector_type);
if (scalar_def)
*scalar_def = op;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
- for (i = nunits - 1; i >= 0; --i)
- {
- t = tree_cons (NULL_TREE, op, t);
- }
- vec_cst = build_vector (vector_type, t);
+ vec_cst = build_vector_from_val (vector_type, op);
return vect_init_vector (stmt, vec_cst, vector_type, NULL);
}
/* Get the def from the vectorized stmt. */
def_stmt_info = vinfo_for_stmt (def_stmt);
vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
- gcc_assert (vec_stmt && gimple_code (vec_stmt) == GIMPLE_PHI);
- vec_oprnd = PHI_RESULT (vec_stmt);
+ if (gimple_code (vec_stmt) == GIMPLE_PHI)
+ vec_oprnd = PHI_RESULT (vec_stmt);
+ else
+ vec_oprnd = gimple_get_lhs (vec_stmt);
return vec_oprnd;
}
slp_tree slp_node)
{
if (slp_node)
- vect_get_slp_defs (slp_node, vec_oprnds0, vec_oprnds1, -1);
+ vect_get_slp_defs (op0, op1, slp_node, vec_oprnds0, vec_oprnds1, -1);
else
{
tree vec_oprnd;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree fndecl, new_temp, def, rhs_type;
gimple def_stmt;
- enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
+ enum vect_def_type dt[3]
+ = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
gimple new_stmt = NULL;
int ncopies, j;
VEC(tree, heap) *vargs = NULL;
vectype_in = NULL_TREE;
nargs = gimple_call_num_args (stmt);
- /* Bail out if the function has more than two arguments, we
- do not have interesting builtin functions to vectorize with
- more than two arguments. No arguments is also not good. */
- if (nargs == 0 || nargs > 2)
+ /* Bail out if the function has more than three arguments, we do not have
+ interesting builtin functions to vectorize with more than two arguments
+ except for fma. No arguments is also not good. */
+ if (nargs == 0 || nargs > 3)
return false;
for (i = 0; i < nargs; i++)
type = TREE_TYPE (scalar_dest);
new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
- fold_convert (type, integer_zero_node));
+ build_zero_cst (type));
set_vinfo_for_stmt (new_stmt, stmt_info);
set_vinfo_for_stmt (stmt, NULL);
STMT_VINFO_STMT (stmt_info) = new_stmt;
vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
}
- /* Arguments are ready. Create the new vector stmt. */
+ /* Arguments are ready. Create the new vector stmt. */
new_stmt = gimple_build_assign_with_ops (code1, vec_dest, vec_oprnd0,
vec_oprnd1);
new_temp = make_ssa_name (vec_dest, new_stmt);
return true;
}
-/* Function vectorizable_operation.
- Check if STMT performs a binary or unary operation that can be vectorized.
+/* Function vectorizable_shift.
+
+ Check if STMT performs a shift operation that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
static bool
-vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
- gimple *vec_stmt, slp_tree slp_node)
+vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi,
+ gimple *vec_stmt, slp_tree slp_node)
{
tree vec_dest;
tree scalar_dest;
enum tree_code code;
enum machine_mode vec_mode;
tree new_temp;
- int op_type;
optab optab;
int icode;
enum machine_mode optab_op2_mode;
tree vectype_out;
int ncopies;
int j, i;
- VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
+ VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
tree vop0, vop1;
unsigned int k;
bool scalar_shift_arg = false;
code = gimple_assign_rhs_code (stmt);
+ if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
+ || code == RROTATE_EXPR))
+ return false;
+
+ scalar_dest = gimple_assign_lhs (stmt);
+ vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+
+ op0 = gimple_assign_rhs1 (stmt);
+ if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
+ &def_stmt, &def, &dt[0], &vectype))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+ /* If op0 is an external or constant def use a vector type with
+ the same size as the output vector type. */
+ if (!vectype)
+ vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
+ if (vec_stmt)
+ gcc_assert (vectype);
+ if (!vectype)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "no vectype for scalar type ");
+ print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
+ }
+
+ return false;
+ }
+
+ nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
+ nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
+ if (nunits_out != nunits_in)
+ return false;
+
+ op1 = gimple_assign_rhs2 (stmt);
+ if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[1]))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+
+ if (loop_vinfo)
+ vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ else
+ vf = 1;
+
+ /* Multiple types in SLP are handled by creating the appropriate number of
+ vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
+ case of SLP. */
+ if (slp_node)
+ ncopies = 1;
+ else
+ ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+
+ gcc_assert (ncopies >= 1);
+
+ /* Determine whether the shift amount is a vector, or scalar. If the
+ shift/rotate amount is a vector, use the vector/vector shift optabs. */
+
+ /* Vector shifted by vector. */
+ if (dt[1] == vect_internal_def)
+ {
+ optab = optab_for_tree_code (code, vectype, optab_vector);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vector/vector shift/rotate found.");
+ }
+ /* See if the machine has a vector shifted by scalar insn and if not
+ then see if it has a vector shifted by vector insn. */
+ else if (dt[1] == vect_constant_def || dt[1] == vect_external_def)
+ {
+ optab = optab_for_tree_code (code, vectype, optab_scalar);
+ if (optab
+ && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
+ {
+ scalar_shift_arg = true;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vector/scalar shift/rotate found.");
+ }
+ else
+ {
+ optab = optab_for_tree_code (code, vectype, optab_vector);
+ if (optab
+ && (optab_handler (optab, TYPE_MODE (vectype))
+ != CODE_FOR_nothing))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vector/vector shift/rotate found.");
+
+ /* Unlike the other binary operators, shifts/rotates have
+ the rhs being int, instead of the same type as the lhs,
+ so make sure the scalar is the right type if we are
+ dealing with vectors of short/char. */
+ if (dt[1] == vect_constant_def)
+ op1 = fold_convert (TREE_TYPE (vectype), op1);
+ }
+ }
+ }
+ else
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "operand mode requires invariant argument.");
+ return false;
+ }
+
+ /* Supportable by target? */
+ if (!optab)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "no optab.");
+ return false;
+ }
+ vec_mode = TYPE_MODE (vectype);
+ icode = (int) optab_handler (optab, vec_mode);
+ if (icode == CODE_FOR_nothing)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "op not supported by target.");
+ /* Check only during analysis. */
+ if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
+ || (vf < vect_min_worthwhile_factor (code)
+ && !vec_stmt))
+ return false;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "proceeding using word mode.");
+ }
+
+ /* Worthwhile without SIMD support? Check only during analysis. */
+ if (!VECTOR_MODE_P (TYPE_MODE (vectype))
+ && vf < vect_min_worthwhile_factor (code)
+ && !vec_stmt)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not worthwhile without SIMD support.");
+ return false;
+ }
+
+ if (!vec_stmt) /* transformation not required. */
+ {
+ STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "=== vectorizable_shift ===");
+ vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+ return true;
+ }
+
+ /** Transform. **/
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "transform binary/unary operation.");
+
+ /* Handle def. */
+ vec_dest = vect_create_destination_var (scalar_dest, vectype);
+
+ /* Allocate VECs for vector operands. In case of SLP, vector operands are
+ created in the previous stages of the recursion, so no allocation is
+ needed, except for the case of shift with scalar shift argument. In that
+ case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
+ be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
+ In case of loop-based vectorization we allocate VECs of size 1. We
+ allocate VEC_OPRNDS1 only in case of binary operation. */
+ if (!slp_node)
+ {
+ vec_oprnds0 = VEC_alloc (tree, heap, 1);
+ vec_oprnds1 = VEC_alloc (tree, heap, 1);
+ }
+ else if (scalar_shift_arg)
+ vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
+
+ prev_stmt_info = NULL;
+ for (j = 0; j < ncopies; j++)
+ {
+ /* Handle uses. */
+ if (j == 0)
+ {
+ if (scalar_shift_arg)
+ {
+ /* Vector shl and shr insn patterns can be defined with scalar
+ operand 2 (shift operand). In this case, use constant or loop
+ invariant op1 directly, without extending it to vector mode
+ first. */
+ optab_op2_mode = insn_data[icode].operand[2].mode;
+ if (!VECTOR_MODE_P (optab_op2_mode))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "operand 1 using scalar mode.");
+ vec_oprnd1 = op1;
+ VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
+ if (slp_node)
+ {
+ /* Store vec_oprnd1 for every vector stmt to be created
+ for SLP_NODE. We check during the analysis that all
+ the shift arguments are the same.
+ TODO: Allow different constants for different vector
+ stmts generated for an SLP instance. */
+ for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
+ VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
+ }
+ }
+ }
+
+ /* vec_oprnd1 is available if operand 1 should be of a scalar-type
+ (a special case for certain kind of vector shifts); otherwise,
+ operand 1 should be of a vector type (the usual case). */
+ if (vec_oprnd1)
+ vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
+ slp_node);
+ else
+ vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
+ slp_node);
+ }
+ else
+ vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
+
+ /* Arguments are ready. Create the new vector stmt. */
+ FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
+ {
+ vop1 = VEC_index (tree, vec_oprnds1, i);
+ new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_assign_set_lhs (new_stmt, new_temp);
+ vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ if (slp_node)
+ VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
+ }
+
+ if (slp_node)
+ continue;
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
+
+ VEC_free (tree, heap, vec_oprnds0);
+ VEC_free (tree, heap, vec_oprnds1);
+
+ return true;
+}
+
+
+/* Function vectorizable_operation.
+
+ Check if STMT performs a binary, unary or ternary operation that can
+ be vectorized.
+ If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+ stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+ Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+static bool
+vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
+ gimple *vec_stmt, slp_tree slp_node)
+{
+ tree vec_dest;
+ tree scalar_dest;
+ tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree vectype;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ enum tree_code code;
+ enum machine_mode vec_mode;
+ tree new_temp;
+ int op_type;
+ optab optab;
+ int icode;
+ tree def;
+ gimple def_stmt;
+ enum vect_def_type dt[3]
+ = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
+ gimple new_stmt = NULL;
+ stmt_vec_info prev_stmt_info;
+ int nunits_in;
+ int nunits_out;
+ tree vectype_out;
+ int ncopies;
+ int j, i;
+ VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vec_oprnds2 = NULL;
+ tree vop0, vop1, vop2;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ int vf;
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
+ return false;
+
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
+ return false;
+
+ /* Is STMT a vectorizable binary/unary operation? */
+ if (!is_gimple_assign (stmt))
+ return false;
+
+ if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
+ return false;
+
+ code = gimple_assign_rhs_code (stmt);
+
/* For pointer addition, we should use the normal plus for
the vector addition. */
if (code == POINTER_PLUS_EXPR)
/* Support only unary or binary operations. */
op_type = TREE_CODE_LENGTH (code);
- if (op_type != unary_op && op_type != binary_op)
+ if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
{
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type);
+ fprintf (vect_dump, "num. args = %d (not unary/binary/ternary op).",
+ op_type);
return false;
}
if (nunits_out != nunits_in)
return false;
- if (op_type == binary_op)
+ if (op_type == binary_op || op_type == ternary_op)
{
op1 = gimple_assign_rhs2 (stmt);
if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def,
return false;
}
}
+ if (op_type == ternary_op)
+ {
+ op2 = gimple_assign_rhs3 (stmt);
+ if (!vect_is_simple_use (op2, loop_vinfo, bb_vinfo, &def_stmt, &def,
+ &dt[2]))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+ }
if (loop_vinfo)
vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
gcc_assert (ncopies >= 1);
- /* If this is a shift/rotate, determine whether the shift amount is a vector,
- or scalar. If the shift/rotate amount is a vector, use the vector/vector
- shift optabs. */
+ /* Shifts are handled in vectorizable_shift (). */
if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
|| code == RROTATE_EXPR)
- {
- /* vector shifted by vector */
- if (dt[1] == vect_internal_def)
- {
- optab = optab_for_tree_code (code, vectype, optab_vector);
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "vector/vector shift/rotate found.");
- }
+ return false;
- /* See if the machine has a vector shifted by scalar insn and if not
- then see if it has a vector shifted by vector insn */
- else if (dt[1] == vect_constant_def || dt[1] == vect_external_def)
- {
- optab = optab_for_tree_code (code, vectype, optab_scalar);
- if (optab
- && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
- {
- scalar_shift_arg = true;
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "vector/scalar shift/rotate found.");
- }
- else
- {
- optab = optab_for_tree_code (code, vectype, optab_vector);
- if (optab
- && (optab_handler (optab, TYPE_MODE (vectype))
- != CODE_FOR_nothing))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "vector/vector shift/rotate found.");
-
- /* Unlike the other binary operators, shifts/rotates have
- the rhs being int, instead of the same type as the lhs,
- so make sure the scalar is the right type if we are
- dealing with vectors of short/char. */
- if (dt[1] == vect_constant_def)
- op1 = fold_convert (TREE_TYPE (vectype), op1);
- }
- }
- }
-
- else
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "operand mode requires invariant argument.");
- return false;
- }
- }
- else
- optab = optab_for_tree_code (code, vectype, optab_default);
+ optab = optab_for_tree_code (code, vectype, optab_default);
/* Supportable by target? */
if (!optab)
if (!slp_node)
{
vec_oprnds0 = VEC_alloc (tree, heap, 1);
- if (op_type == binary_op)
+ if (op_type == binary_op || op_type == ternary_op)
vec_oprnds1 = VEC_alloc (tree, heap, 1);
+ if (op_type == ternary_op)
+ vec_oprnds2 = VEC_alloc (tree, heap, 1);
}
- else if (scalar_shift_arg)
- vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
/* In case the vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
/* Handle uses. */
if (j == 0)
{
- if (op_type == binary_op && scalar_shift_arg)
- {
- /* Vector shl and shr insn patterns can be defined with scalar
- operand 2 (shift operand). In this case, use constant or loop
- invariant op1 directly, without extending it to vector mode
- first. */
- optab_op2_mode = insn_data[icode].operand[2].mode;
- if (!VECTOR_MODE_P (optab_op2_mode))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "operand 1 using scalar mode.");
- vec_oprnd1 = op1;
- VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
- if (slp_node)
- {
- /* Store vec_oprnd1 for every vector stmt to be created
- for SLP_NODE. We check during the analysis that all
- the shift arguments are the same.
- TODO: Allow different constants for different vector
- stmts generated for an SLP instance. */
- for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
- VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
- }
- }
- }
-
- /* vec_oprnd1 is available if operand 1 should be of a scalar-type
- (a special case for certain kind of vector shifts); otherwise,
- operand 1 should be of a vector type (the usual case). */
- if (op_type == binary_op && !vec_oprnd1)
+ if (op_type == binary_op || op_type == ternary_op)
vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
slp_node);
else
vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
slp_node);
+ if (op_type == ternary_op)
+ {
+ vec_oprnds2 = VEC_alloc (tree, heap, 1);
+ VEC_quick_push (tree, vec_oprnds2,
+ vect_get_vec_def_for_operand (op2, stmt, NULL));
+ }
}
else
- vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
+ {
+ vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
+ if (op_type == ternary_op)
+ {
+ tree vec_oprnd = VEC_pop (tree, vec_oprnds2);
+ VEC_quick_push (tree, vec_oprnds2,
+ vect_get_vec_def_for_stmt_copy (dt[2],
+ vec_oprnd));
+ }
+ }
- /* Arguments are ready. Create the new vector stmt. */
+ /* Arguments are ready. Create the new vector stmt. */
FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
{
- vop1 = ((op_type == binary_op)
- ? VEC_index (tree, vec_oprnds1, i) : NULL);
- new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
+ vop1 = ((op_type == binary_op || op_type == ternary_op)
+ ? VEC_index (tree, vec_oprnds1, i) : NULL_TREE);
+ vop2 = ((op_type == ternary_op)
+ ? VEC_index (tree, vec_oprnds2, i) : NULL_TREE);
+ new_stmt = gimple_build_assign_with_ops3 (code, vec_dest,
+ vop0, vop1, vop2);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_assign_set_lhs (new_stmt, new_temp);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
VEC_free (tree, heap, vec_oprnds0);
if (vec_oprnds1)
VEC_free (tree, heap, vec_oprnds1);
+ if (vec_oprnds2)
+ VEC_free (tree, heap, vec_oprnds2);
return true;
}
{
/* Handle uses. */
if (slp_node)
- vect_get_slp_defs (slp_node, &vec_oprnds0, NULL, -1);
+ vect_get_slp_defs (op0, NULL_TREE, slp_node, &vec_oprnds0, NULL, -1);
else
{
VEC_free (tree, heap, vec_oprnds0);
vect_pow2 (multi_step_cvt) - 1);
}
- /* Arguments are ready. Create the new vector stmts. */
+ /* Arguments are ready. Create the new vector stmts. */
tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
vect_create_vectorized_demotion_stmts (&vec_oprnds0,
multi_step_cvt, stmt, tmp_vec_dsts,
if (j == 0)
{
if (slp_node)
- vect_get_slp_defs (slp_node, &vec_oprnds0, &vec_oprnds1, -1);
+ vect_get_slp_defs (op0, op1, slp_node, &vec_oprnds0,
+ &vec_oprnds1, -1);
else
{
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
}
}
- /* Arguments are ready. Create the new vector stmts. */
+ /* Arguments are ready. Create the new vector stmts. */
tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
vect_create_vectorized_promotion_stmts (&vec_oprnds0, &vec_oprnds1,
multi_step_cvt, stmt,
if (slp)
{
/* Get vectorized arguments for SLP_NODE. */
- vect_get_slp_defs (slp_node, &vec_oprnds, NULL, -1);
+ vect_get_slp_defs (NULL_TREE, NULL_TREE, slp_node, &vec_oprnds,
+ NULL, -1);
vec_oprnd = VEC_index (tree, vec_oprnds, 0);
}
pi->misalign = DR_MISALIGNMENT (first_dr);
}
- /* Arguments are ready. Create the new vector stmt. */
+ /* Arguments are ready. Create the new vector stmt. */
new_stmt = gimple_build_assign (data_ref, vec_oprnd);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
mark_symbols_for_renaming (new_stmt);
mask_type = get_vectype_for_scalar_type (mask_element_type);
nunits = TYPE_VECTOR_SUBPARTS (vectype);
- if (TYPE_VECTOR_SUBPARTS (vectype) != TYPE_VECTOR_SUBPARTS (mask_type))
+ if (!mask_type
+ || TYPE_VECTOR_SUBPARTS (vectype) != TYPE_VECTOR_SUBPARTS (mask_type))
return NULL;
for (i = 0; i < nunits; i++)
/* Generate the permute statement. */
perm_stmt = gimple_build_call (builtin_decl, 3, x, x, mask_vec);
+ if (!useless_type_conversion_p (vectype,
+ TREE_TYPE (TREE_TYPE (builtin_decl))))
+ {
+ tree tem = create_tmp_reg (TREE_TYPE (TREE_TYPE (builtin_decl)), NULL);
+ tem = make_ssa_name (tem, perm_stmt);
+ gimple_call_set_lhs (perm_stmt, tem);
+ vect_finish_stmt_generation (stmt, perm_stmt, gsi);
+ perm_stmt = gimple_build_assign (NULL_TREE,
+ build1 (VIEW_CONVERT_EXPR,
+ vectype, tem));
+ }
data_ref = make_ssa_name (perm_dest, perm_stmt);
- gimple_call_set_lhs (perm_stmt, data_ref);
+ gimple_set_lhs (perm_stmt, data_ref);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
return data_ref;
vec_else_clause);
}
- /* Arguments are ready. Create the new vector stmt. */
+ /* Arguments are ready. Create the new vector stmt. */
vec_compare = build2 (TREE_CODE (cond_expr), vectype,
vec_cond_lhs, vec_cond_rhs);
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
|| vectorizable_type_demotion (stmt, NULL, NULL, NULL)
|| vectorizable_conversion (stmt, NULL, NULL, NULL)
+ || vectorizable_shift (stmt, NULL, NULL, NULL)
|| vectorizable_operation (stmt, NULL, NULL, NULL)
|| vectorizable_assignment (stmt, NULL, NULL, NULL)
|| vectorizable_load (stmt, NULL, NULL, NULL, NULL)
else
{
if (bb_vinfo)
- ok = (vectorizable_operation (stmt, NULL, NULL, node)
+ ok = (vectorizable_shift (stmt, NULL, NULL, node)
+ || vectorizable_operation (stmt, NULL, NULL, node)
|| vectorizable_assignment (stmt, NULL, NULL, node)
|| vectorizable_load (stmt, NULL, NULL, node, NULL)
|| vectorizable_store (stmt, NULL, NULL, node));
gcc_assert (done);
break;
+ case shift_vec_info_type:
+ done = vectorizable_shift (stmt, gsi, &vec_stmt, slp_node);
+ gcc_assert (done);
+ break;
+
case op_vec_info_type:
done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
gcc_assert (done);
}
-/* Function get_vectype_for_scalar_type.
+/* Function get_vectype_for_scalar_type_and_size.
- Returns the vector type corresponding to SCALAR_TYPE as supported
+ Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
by the target. */
-tree
-get_vectype_for_scalar_type (tree scalar_type)
+static tree
+get_vectype_for_scalar_type_and_size (tree scalar_type, unsigned size)
{
enum machine_mode inner_mode = TYPE_MODE (scalar_type);
+ enum machine_mode simd_mode;
unsigned int nbytes = GET_MODE_SIZE (inner_mode);
int nunits;
tree vectype;
- if (nbytes == 0
- || (nbytes >= targetm.vectorize.units_per_simd_word (inner_mode)))
+ if (nbytes == 0)
return NULL_TREE;
/* We can't build a vector type of elements with alignment bigger than
&& GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type))
return NULL_TREE;
- /* FORNOW: Only a single vector size per mode
- (TARGET_VECTORIZE_UNITS_PER_SIMD_WORD) is expected. */
- nunits = targetm.vectorize.units_per_simd_word (inner_mode) / nbytes;
+ if (GET_MODE_CLASS (inner_mode) != MODE_INT
+ && GET_MODE_CLASS (inner_mode) != MODE_FLOAT)
+ return NULL_TREE;
+
+ /* If no size was supplied use the mode the target prefers. Otherwise
+ lookup a vector mode of the specified size. */
+ if (size == 0)
+ simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
+ else
+ simd_mode = mode_for_vector (inner_mode, size / nbytes);
+ nunits = GET_MODE_SIZE (simd_mode) / nbytes;
+ if (nunits <= 1)
+ return NULL_TREE;
vectype = build_vector_type (scalar_type, nunits);
if (vect_print_dump_info (REPORT_DETAILS))
return vectype;
}
+unsigned int current_vector_size;
+
+/* Function get_vectype_for_scalar_type.
+
+ Returns the vector type corresponding to SCALAR_TYPE as supported
+ by the target. */
+
+tree
+get_vectype_for_scalar_type (tree scalar_type)
+{
+ tree vectype;
+ vectype = get_vectype_for_scalar_type_and_size (scalar_type,
+ current_vector_size);
+ if (vectype
+ && current_vector_size == 0)
+ current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
+ return vectype;
+}
+
/* Function get_same_sized_vectype
Returns a vector type corresponding to SCALAR_TYPE of size
VECTOR_TYPE if supported by the target. */
tree
-get_same_sized_vectype (tree scalar_type, tree vector_type ATTRIBUTE_UNUSED)
+get_same_sized_vectype (tree scalar_type, tree vector_type)
{
- return get_vectype_for_scalar_type (scalar_type);
+ return get_vectype_for_scalar_type_and_size
+ (scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
}
/* Function vect_is_simple_use.
OSDN Git Service
