have to scan the RHS or function arguments instead. */
if (is_gimple_assign (stmt))
{
- for (i = 1; i < gimple_num_ops (stmt); i++)
+ enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
+ tree op = gimple_assign_rhs1 (stmt);
+
+ i = 1;
+ if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
+ {
+ if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo,
+ live_p, relevant, &worklist)
+ || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo,
+ live_p, relevant, &worklist))
+ {
+ VEC_free (gimple, heap, worklist);
+ return false;
+ }
+ i = 2;
+ }
+ for (; i < gimple_num_ops (stmt); i++)
{
- tree op = gimple_op (stmt, i);
+ op = gimple_op (stmt, i);
if (!process_use (stmt, op, loop_vinfo, live_p, relevant,
&worklist))
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- gimple_stmt_iterator si;
gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
print_gimple_stmt (vect_dump, vec_stmt, 0, TDF_SLIM);
}
- si = *gsi;
- if (is_gimple_debug (gsi_stmt (si)))
- {
- gsi_next_nondebug (&si);
- gcc_assert (!gsi_end_p (si));
- }
- gimple_set_location (vec_stmt, gimple_location (gsi_stmt (si)));
+ gimple_set_location (vec_stmt, gimple_location (stmt));
}
/* Checks if CALL can be vectorized in type VECTYPE. Returns
}
else
{
- vec_oprnd1 = gimple_call_arg (new_stmt, 2*i);
+ vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1);
vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
vec_oprnd1
VEC (tree, heap) *vec_oprnds0 = NULL;
VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
tree last_oprnd, intermediate_type;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
- /* FORNOW: not supported by basic block SLP vectorization. */
- gcc_assert (loop_vinfo);
-
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
&& CONVERT_EXPR_CODE_P (code))))
return false;
- if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
+ if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
&def_stmt, &def, &dt[0], &vectype_in))
{
if (vect_print_dump_info (REPORT_DETAILS))
int multi_step_cvt = 0;
VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ unsigned int k;
- /* FORNOW: not supported by basic block SLP vectorization. */
- gcc_assert (loop_vinfo);
-
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
code = gimple_assign_rhs_code (stmt);
if (!CONVERT_EXPR_CODE_P (code)
- && code != WIDEN_MULT_EXPR)
+ && code != WIDEN_MULT_EXPR
+ && code != WIDEN_LSHIFT_EXPR)
return false;
scalar_dest = gimple_assign_lhs (stmt);
&& SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
&& CONVERT_EXPR_CODE_P (code))))
return false;
- if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
+ if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
&def_stmt, &def, &dt[0], &vectype_in))
{
if (vect_print_dump_info (REPORT_DETAILS))
bool ok;
op1 = gimple_assign_rhs2 (stmt);
- if (code == WIDEN_MULT_EXPR)
+ if (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR)
{
/* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
OP1. */
fprintf (vect_dump, "transform type promotion operation. ncopies = %d.",
ncopies);
- if (code == WIDEN_MULT_EXPR)
+ if (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR)
{
if (CONSTANT_CLASS_P (op0))
op0 = fold_convert (TREE_TYPE (op1), op0);
if (op_type == binary_op)
vec_oprnds1 = VEC_alloc (tree, heap, 1);
}
+ else if (code == WIDEN_LSHIFT_EXPR)
+ vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
/* In case the vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
if (j == 0)
{
if (slp_node)
- vect_get_slp_defs (op0, op1, slp_node, &vec_oprnds0,
- &vec_oprnds1, -1);
- else
+ {
+ if (code == WIDEN_LSHIFT_EXPR)
+ {
+ vec_oprnd1 = op1;
+ /* Store vec_oprnd1 for every vector stmt to be created
+ for SLP_NODE. We check during the analysis that all
+ the shift arguments are the same. */
+ for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
+ VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
+
+ vect_get_slp_defs (op0, NULL_TREE, slp_node, &vec_oprnds0, NULL,
+ -1);
+ }
+ else
+ vect_get_slp_defs (op0, op1, slp_node, &vec_oprnds0,
+ &vec_oprnds1, -1);
+ }
+ else
{
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
VEC_quick_push (tree, vec_oprnds0, vec_oprnd0);
if (op_type == binary_op)
{
- vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt, NULL);
+ if (code == WIDEN_LSHIFT_EXPR)
+ vec_oprnd1 = op1;
+ else
+ vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt, NULL);
VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
}
}
VEC_replace (tree, vec_oprnds0, 0, vec_oprnd0);
if (op_type == binary_op)
{
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd1);
+ if (code == WIDEN_LSHIFT_EXPR)
+ vec_oprnd1 = op1;
+ else
+ vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd1);
VEC_replace (tree, vec_oprnds1, 0, vec_oprnd1);
}
}
}
/* Given a vector type VECTYPE returns a builtin DECL to be used
- for vector permutation and stores a mask into *MASK that implements
- reversal of the vector elements. If that is impossible to do
- returns NULL (and *MASK is unchanged). */
+ for vector permutation and returns the mask that implements
+ reversal of the vector elements. If that is impossible to do,
+ returns NULL. */
static tree
-perm_mask_for_reverse (tree vectype, tree *mask)
+perm_mask_for_reverse (tree vectype)
{
- tree builtin_decl;
- tree mask_element_type, mask_type;
- tree mask_vec = NULL;
- int i;
- int nunits;
- if (!targetm.vectorize.builtin_vec_perm)
- return NULL;
+ tree mask_element_type, mask_type, mask_vec = NULL;
+ int i, nunits;
- builtin_decl = targetm.vectorize.builtin_vec_perm (vectype,
- &mask_element_type);
- if (!builtin_decl || !mask_element_type)
+ if (!can_vec_perm_expr_p (vectype, NULL_TREE))
return NULL;
+ mask_element_type
+ = lang_hooks.types.type_for_size
+ (TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (vectype))), 1);
mask_type = get_vectype_for_scalar_type (mask_element_type);
nunits = TYPE_VECTOR_SUBPARTS (vectype);
- if (!mask_type
- || TYPE_VECTOR_SUBPARTS (vectype) != TYPE_VECTOR_SUBPARTS (mask_type))
- return NULL;
for (i = 0; i < nunits; i++)
mask_vec = tree_cons (NULL, build_int_cst (mask_element_type, i), mask_vec);
mask_vec = build_vector (mask_type, mask_vec);
- if (!targetm.vectorize.builtin_vec_perm_ok (vectype, mask_vec))
+ if (!can_vec_perm_expr_p (vectype, mask_vec))
return NULL;
- if (mask)
- *mask = mask_vec;
- return builtin_decl;
+
+ return mask_vec;
}
/* Given a vector variable X, that was generated for the scalar LHS of
reverse_vec_elements (tree x, gimple stmt, gimple_stmt_iterator *gsi)
{
tree vectype = TREE_TYPE (x);
- tree mask_vec, builtin_decl;
- tree perm_dest, data_ref;
+ tree mask_vec, perm_dest, data_ref;
gimple perm_stmt;
- builtin_decl = perm_mask_for_reverse (vectype, &mask_vec);
+ mask_vec = perm_mask_for_reverse (vectype);
perm_dest = vect_create_destination_var (gimple_assign_lhs (stmt), vectype);
/* Generate the permute statement. */
- perm_stmt = gimple_build_call (builtin_decl, 3, x, x, mask_vec);
- if (!useless_type_conversion_p (vectype,
- TREE_TYPE (TREE_TYPE (builtin_decl))))
- {
- tree tem = create_tmp_reg (TREE_TYPE (TREE_TYPE (builtin_decl)), NULL);
- tem = make_ssa_name (tem, perm_stmt);
- gimple_call_set_lhs (perm_stmt, tem);
- vect_finish_stmt_generation (stmt, perm_stmt, gsi);
- perm_stmt = gimple_build_assign (NULL_TREE,
- build1 (VIEW_CONVERT_EXPR,
- vectype, tem));
- }
+ perm_stmt = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, perm_dest,
+ x, x, mask_vec);
data_ref = make_ssa_name (perm_dest, perm_stmt);
gimple_set_lhs (perm_stmt, data_ref);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
fprintf (vect_dump, "negative step but alignment required.");
return false;
}
- if (!perm_mask_for_reverse (vectype, NULL))
+ if (!perm_mask_for_reverse (vectype))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "negative step and reversing not supported.");
if (strided_load)
{
first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
+ if (slp
+ && !SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance)
+ && first_stmt != VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0))
+ first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
+
/* Check if the chain of loads is already vectorized. */
if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt)))
{
LOOP - the loop that is being vectorized.
COND - Condition that is checked for simple use.
+ Output:
+ *COMP_VECTYPE - the vector type for the comparison.
+
Returns whether a COND can be vectorized. Checks whether
condition operands are supportable using vec_is_simple_use. */
static bool
-vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
+vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo, tree *comp_vectype)
{
tree lhs, rhs;
tree def;
enum vect_def_type dt;
+ tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
if (!COMPARISON_CLASS_P (cond))
return false;
if (TREE_CODE (lhs) == SSA_NAME)
{
gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
- if (!vect_is_simple_use (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
- &dt))
+ if (!vect_is_simple_use_1 (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
+ &dt, &vectype1))
return false;
}
else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
if (TREE_CODE (rhs) == SSA_NAME)
{
gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
- if (!vect_is_simple_use (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
- &dt))
+ if (!vect_is_simple_use_1 (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
+ &dt, &vectype2))
return false;
}
else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
&& TREE_CODE (rhs) != FIXED_CST)
return false;
+ *comp_vectype = vectype1 ? vectype1 : vectype2;
return true;
}
{
tree scalar_dest = NULL_TREE;
tree vec_dest = NULL_TREE;
- tree op = NULL_TREE;
tree cond_expr, then_clause, else_clause;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree comp_vectype;
tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
tree vec_compare, vec_cond_expr;
tree new_temp;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- enum machine_mode vec_mode;
tree def;
enum vect_def_type dt, dts[4];
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
if (code != COND_EXPR)
return false;
- gcc_assert (gimple_assign_single_p (stmt));
- op = gimple_assign_rhs1 (stmt);
- cond_expr = TREE_OPERAND (op, 0);
- then_clause = TREE_OPERAND (op, 1);
- else_clause = TREE_OPERAND (op, 2);
-
- if (!vect_is_simple_cond (cond_expr, loop_vinfo))
- return false;
+ cond_expr = gimple_assign_rhs1 (stmt);
+ then_clause = gimple_assign_rhs2 (stmt);
+ else_clause = gimple_assign_rhs3 (stmt);
- /* We do not handle two different vector types for the condition
- and the values. */
- if (!types_compatible_p (TREE_TYPE (TREE_OPERAND (cond_expr, 0)),
- TREE_TYPE (vectype)))
+ if (!vect_is_simple_cond (cond_expr, loop_vinfo, &comp_vectype)
+ || !comp_vectype)
return false;
if (TREE_CODE (then_clause) == SSA_NAME)
&& TREE_CODE (else_clause) != FIXED_CST)
return false;
-
- vec_mode = TYPE_MODE (vectype);
-
if (!vec_stmt)
{
STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
- return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
+ return expand_vec_cond_expr_p (vectype, comp_vectype);
}
/* Transform */
else
{
if (bb_vinfo)
- ok = (vectorizable_shift (stmt, NULL, NULL, node)
+ ok = (vectorizable_type_promotion (stmt, NULL, NULL, node)
+ || vectorizable_type_demotion (stmt, NULL, NULL, node)
+ || vectorizable_shift (stmt, NULL, NULL, node)
|| vectorizable_operation (stmt, NULL, NULL, node)
|| vectorizable_assignment (stmt, NULL, NULL, node)
|| vectorizable_load (stmt, NULL, NULL, node, NULL)
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
+ struct loop *vect_loop = NULL;
bool ordered_p;
enum machine_mode vec_mode;
enum insn_code icode1, icode2;
tree wide_vectype = vectype_out;
enum tree_code c1, c2;
+ if (loop_info)
+ vect_loop = LOOP_VINFO_LOOP (loop_info);
+
/* The result of a vectorized widening operation usually requires two vectors
(because the widened results do not fit int one vector). The generated
vector results would normally be expected to be generated in the same
iterations in parallel). We therefore don't allow to change the order
of the computation in the inner-loop during outer-loop vectorization. */
- if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
+ if (vect_loop
+ && STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
&& !nested_in_vect_loop_p (vect_loop, stmt))
ordered_p = false;
else
}
break;
+ case WIDEN_LSHIFT_EXPR:
+ if (BYTES_BIG_ENDIAN)
+ {
+ c1 = VEC_WIDEN_LSHIFT_HI_EXPR;
+ c2 = VEC_WIDEN_LSHIFT_LO_EXPR;
+ }
+ else
+ {
+ c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
+ c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
+ }
+ break;
+
CASE_CONVERT:
if (BYTES_BIG_ENDIAN)
{
OSDN Git Service
