if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "vec_stmt_relevant_p: used out of loop.");
+ if (is_gimple_debug (USE_STMT (use_p)))
+ continue;
+
/* We expect all such uses to be in the loop exit phis
(because of loop closed form) */
gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
{
tree operand;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
-
+
/* USE corresponds to some operand in STMT. If there is no data
reference in STMT, then any operand that corresponds to USE
is not indexing an array. */
if (!STMT_VINFO_DATA_REF (stmt_info))
return true;
-
+
/* STMT has a data_ref. FORNOW this means that its of one of
the following forms:
-1- ARRAY_REF = var
Therefore, all we need to check is if STMT falls into the
first case, and whether var corresponds to USE. */
-
- if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
- return false;
if (!gimple_assign_copy_p (stmt))
return false;
+ if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
+ return false;
operand = gimple_assign_rhs1 (stmt);
-
if (TREE_CODE (operand) != SSA_NAME)
return false;
if (!exist_non_indexing_operands_for_use_p (use, stmt))
return true;
- if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (use, loop_vinfo, NULL, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
+
switch (relevant)
{
case vect_unused_in_scope:
- relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ?
- vect_used_by_reduction : vect_unused_in_scope;
+ relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
+ vect_used_in_scope : vect_unused_in_scope;
break;
+
case vect_used_in_outer_by_reduction:
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
relevant = vect_used_by_reduction;
break;
+
case vect_used_in_outer:
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
relevant = vect_used_in_scope;
break;
- case vect_used_by_reduction:
+
case vect_used_in_scope:
break;
...
inner-loop:
d = def_stmt
- outer-loop-tail-bb:
+ outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
stmt # use (d) */
else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
+
switch (relevant)
{
case vect_unused_in_scope:
- relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ?
- vect_used_in_outer_by_reduction : vect_unused_in_scope;
- break;
-
- case vect_used_in_outer_by_reduction:
- case vect_used_in_outer:
+ relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
+ || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
+ vect_used_in_outer_by_reduction : vect_unused_in_scope;
break;
case vect_used_by_reduction:
basic_block bb;
gimple phi;
bool live_p;
- enum vect_relevant relevant;
+ enum vect_relevant relevant, tmp_relevant;
+ enum vect_def_type def_type;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
those that are used by a reduction computation, and those that are
(also) used by a regular computation. This allows us later on to
identify stmts that are used solely by a reduction, and therefore the
- order of the results that they produce does not have to be kept.
-
- Reduction phis are expected to be used by a reduction stmt, or by
- in an outer loop; Other reduction stmts are expected to be
- in the loop, and possibly used by a stmt in an outer loop.
- Here are the expected values of "relevant" for reduction phis/stmts:
+ order of the results that they produce does not have to be kept. */
- relevance: phi stmt
- vect_unused_in_scope ok
- vect_used_in_outer_by_reduction ok ok
- vect_used_in_outer ok ok
- vect_used_by_reduction ok
- vect_used_in_scope */
-
- if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
+ def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
+ tmp_relevant = relevant;
+ switch (def_type)
{
- enum vect_relevant tmp_relevant = relevant;
- switch (tmp_relevant)
- {
- case vect_unused_in_scope:
- gcc_assert (gimple_code (stmt) != GIMPLE_PHI);
- relevant = vect_used_by_reduction;
- break;
+ case vect_reduction_def:
+ switch (tmp_relevant)
+ {
+ case vect_unused_in_scope:
+ relevant = vect_used_by_reduction;
+ break;
- case vect_used_in_outer_by_reduction:
- case vect_used_in_outer:
- gcc_assert (gimple_code (stmt) != GIMPLE_ASSIGN
- || (gimple_assign_rhs_code (stmt) != WIDEN_SUM_EXPR
- && (gimple_assign_rhs_code (stmt)
- != DOT_PROD_EXPR)));
- break;
+ case vect_used_by_reduction:
+ if (gimple_code (stmt) == GIMPLE_PHI)
+ break;
+ /* fall through */
- case vect_used_by_reduction:
- if (gimple_code (stmt) == GIMPLE_PHI)
- break;
- /* fall through */
- case vect_used_in_scope:
- default:
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "unsupported use of reduction.");
- VEC_free (gimple, heap, worklist);
- return false;
- }
- live_p = false;
- }
+ default:
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported use of reduction.");
+ VEC_free (gimple, heap, worklist);
+ return false;
+ }
+
+ live_p = false;
+ break;
+
+ case vect_nested_cycle:
+ if (tmp_relevant != vect_unused_in_scope
+ && tmp_relevant != vect_used_in_outer_by_reduction
+ && tmp_relevant != vect_used_in_outer)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported use of nested cycle.");
+
+ VEC_free (gimple, heap, worklist);
+ return false;
+ }
+
+ live_p = false;
+ break;
+
+ case vect_double_reduction_def:
+ if (tmp_relevant != vect_unused_in_scope
+ && tmp_relevant != vect_used_by_reduction)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported use of double reduction.");
+
+ VEC_free (gimple, heap, worklist);
+ return false;
+ }
+
+ live_p = false;
+ break;
+
+ default:
+ break;
+ }
+
FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
{
tree op = USE_FROM_PTR (use_p);
else
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-
- if (nested_in_vect_loop_p (loop, stmt))
- loop = loop->inner;
- pe = loop_preheader_edge (loop);
- new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
- gcc_assert (!new_bb);
+
+ if (loop_vinfo)
+ {
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+
+ if (nested_in_vect_loop_p (loop, stmt))
+ loop = loop->inner;
+
+ pe = loop_preheader_edge (loop);
+ new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
+ gcc_assert (!new_bb);
+ }
+ else
+ {
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ basic_block bb;
+ gimple_stmt_iterator gsi_bb_start;
+
+ gcc_assert (bb_vinfo);
+ bb = BB_VINFO_BB (bb_vinfo);
+ gsi_bb_start = gsi_after_labels (bb);
+ gsi_insert_before (&gsi_bb_start, init_stmt, GSI_SAME_STMT);
+ }
}
if (vect_print_dump_info (REPORT_DETAILS))
return vec_oprnd;
}
+
/* Function vect_get_vec_def_for_operand.
OP is an operand in STMT. This function returns a (vector) def that will be
print_generic_expr (vect_dump, op, TDF_SLIM);
}
- is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
+ is_simple_use = vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def,
+ &dt);
gcc_assert (is_simple_use);
if (vect_print_dump_info (REPORT_DETAILS))
{
/* Case 4: operand is defined by a loop header phi - reduction */
case vect_reduction_def:
+ case vect_double_reduction_def:
+ case vect_nested_cycle:
{
struct loop *loop;
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
- set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo));
+ set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo,
+ bb_vinfo));
if (vect_print_dump_info (REPORT_DETAILS))
{
tree fndecl, new_temp, def, rhs_type, lhs_type;
gimple def_stmt;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
- gimple new_stmt;
+ gimple new_stmt = NULL;
int ncopies, j;
VEC(tree, heap) *vargs = NULL;
enum { NARROW, NONE, WIDEN } modifier;
size_t i, nargs;
+ /* FORNOW: unsupported in basic block SLP. */
+ gcc_assert (loop_vinfo);
+
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
}
rhs_type = TREE_TYPE (op);
- if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt[i]))
+ if (!vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt[i]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
vec_oprnd0
= vect_get_vec_def_for_operand (op, stmt, NULL);
else
- vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (dt[nargs], vec_oprnd0);
+ {
+ vec_oprnd0 = gimple_call_arg (new_stmt, i);
+ vec_oprnd0
+ = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
+ }
VEC_quick_push (tree, vargs, vec_oprnd0);
}
gimple_call_set_lhs (new_stmt, new_temp);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ mark_symbols_for_renaming (new_stmt);
if (j == 0)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
vec_oprnd0
= vect_get_vec_def_for_operand (op, stmt, NULL);
vec_oprnd1
- = vect_get_vec_def_for_stmt_copy (dt[nargs], vec_oprnd0);
+ = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
}
else
{
+ vec_oprnd1 = gimple_call_arg (new_stmt, 2*i);
vec_oprnd0
- = vect_get_vec_def_for_stmt_copy (dt[nargs], vec_oprnd1);
+ = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
vec_oprnd1
- = vect_get_vec_def_for_stmt_copy (dt[nargs], vec_oprnd0);
+ = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
}
VEC_quick_push (tree, vargs, vec_oprnd0);
gimple_call_set_lhs (new_stmt, new_temp);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ mark_symbols_for_renaming (new_stmt);
if (j == 0)
STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
/* Is STMT a vectorizable conversion? */
+ /* FORNOW: unsupported in basic block SLP. */
+ gcc_assert (loop_vinfo);
+
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
gcc_assert (ncopies >= 1);
/* Check the operands of the operation. */
- if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt[0]))
+ if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
int i;
VEC(tree,heap) *vec_oprnds = NULL;
tree vop;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
/* Multiple types in SLP are handled by creating the appropriate number of
vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
if (ncopies > 1)
return false; /* FORNOW */
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
else
return false;
- if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt[0]))
+ if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[0]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
unsigned int k;
bool shift_p = false;
bool scalar_shift_arg = false;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ int vf;
+
+ if (loop_vinfo)
+ vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ else
+ /* FORNOW: multiple types are not supported in basic block SLP. */
+ vf = nunits_in;
/* Multiple types in SLP are handled by creating the appropriate number of
vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
gcc_assert (ncopies >= 1);
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
}
op0 = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt[0]))
+ if (!vect_is_simple_use (op0, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[0]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
if (op_type == binary_op)
{
op1 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use (op1, loop_vinfo, &def_stmt, &def, &dt[1]))
+ if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def,
+ &dt[1]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
fprintf (vect_dump, "op not supported by target.");
/* Check only during analysis. */
if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
- || (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
- < vect_min_worthwhile_factor (code)
+ || (vf < vect_min_worthwhile_factor (code)
&& !vec_stmt))
return false;
if (vect_print_dump_info (REPORT_DETAILS))
/* Worthwhile without SIMD support? Check only during analysis. */
if (!VECTOR_MODE_P (TYPE_MODE (vectype))
- && LOOP_VINFO_VECT_FACTOR (loop_vinfo)
- < vect_min_worthwhile_factor (code)
+ && vf < vect_min_worthwhile_factor (code)
&& !vec_stmt)
{
if (vect_print_dump_info (REPORT_DETAILS))
VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
tree last_oprnd, intermediate_type;
+ /* FORNOW: not supported by basic block SLP vectorization. */
+ gcc_assert (loop_vinfo);
+
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
ncopies = 1;
else
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
-
gcc_assert (ncopies >= 1);
if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
return false;
/* Check the operands of the operation. */
- if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt[0]))
+ if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
prev_stmt_info);
}
}
-
+
/* Function vectorizable_type_promotion
VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
+ /* FORNOW: not supported by basic block SLP vectorization. */
+ gcc_assert (loop_vinfo);
+
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
return false;
/* Check the operands of the operation. */
- if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt[0]))
+ if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
if (op_type == binary_op)
{
op1 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use (op1, loop_vinfo, &def_stmt, &def, &dt[1]))
+ if (!vect_is_simple_use (op1, loop_vinfo, NULL, &def_stmt, &def, &dt[1]))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
enum machine_mode vec_mode;
tree dummy;
enum dr_alignment_support alignment_support_scheme;
bool slp = (slp_node != NULL);
stmt_vec_info first_stmt_vinfo;
unsigned int vec_num;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
/* Multiple types in SLP are handled by creating the appropriate number of
vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
gcc_assert (ncopies >= 1);
/* FORNOW. This restriction should be relaxed. */
- if (nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
+ if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "multiple types in nested loop.");
return false;
}
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
scalar_dest = gimple_assign_lhs (stmt);
if (TREE_CODE (scalar_dest) != ARRAY_REF
&& TREE_CODE (scalar_dest) != INDIRECT_REF
- && !STMT_VINFO_STRIDED_ACCESS (stmt_info))
+ && TREE_CODE (scalar_dest) != COMPONENT_REF
+ && TREE_CODE (scalar_dest) != IMAGPART_EXPR
+ && TREE_CODE (scalar_dest) != REALPART_EXPR)
return false;
gcc_assert (gimple_assign_single_p (stmt));
op = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
{
gcc_assert (gimple_assign_single_p (next_stmt));
op = gimple_assign_rhs1 (next_stmt);
- if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt,
+ &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
/* FORNOW */
- gcc_assert (!nested_in_vect_loop_p (loop, stmt));
+ gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
/* We vectorize all the stmts of the interleaving group when we
reach the last stmt in the group. */
alignment_support_scheme = vect_supportable_dr_alignment (first_dr);
gcc_assert (alignment_support_scheme);
- gcc_assert (alignment_support_scheme == dr_aligned); /* FORNOW */
/* In case the vectorization factor (VF) is bigger than the number
of elements that we can fit in a vectype (nunits), we have to generate
dataref_ptr = vect_create_data_ref_ptr (first_stmt, NULL, NULL_TREE,
&dummy, &ptr_incr, false,
&inv_p);
- gcc_assert (!inv_p);
+ gcc_assert (bb_vinfo || !inv_p);
}
else
{
for (i = 0; i < group_size; i++)
{
op = VEC_index (tree, oprnds, i);
- vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
+ vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def,
+ &dt);
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
VEC_replace(tree, dr_chain, i, vec_oprnd);
VEC_replace(tree, oprnds, i, vec_oprnd);
vect_permute_store_chain(). */
vec_oprnd = VEC_index (tree, result_chain, i);
- data_ref = build_fold_indirect_ref (dataref_ptr);
+ if (aligned_access_p (first_dr))
+ data_ref = build_fold_indirect_ref (dataref_ptr);
+ else
+ {
+ int mis = DR_MISALIGNMENT (first_dr);
+ tree tmis = (mis == -1 ? size_zero_node : size_int (mis));
+ tmis = size_binop (MULT_EXPR, tmis, size_int (BITS_PER_UNIT));
+ data_ref = build2 (MISALIGNED_INDIRECT_REF, vectype, dataref_ptr, tmis);
+ }
+
/* If accesses through a pointer to vectype do not alias the original
memory reference we have a problem. This should never happen. */
gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref),
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info prev_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
- bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
+ bool nested_in_vect_loop = false;
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree new_temp;
bool slp = (slp_node != NULL);
bool slp_perm = false;
enum tree_code code;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ int vf;
+
+ if (loop_vinfo)
+ {
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
+ vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ }
+ else
+ /* FORNOW: multiple types are not supported in basic block SLP. */
+ vf = nunits;
/* Multiple types in SLP are handled by creating the appropriate number of
vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
return false;
}
- if (slp && SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance))
- slp_perm = true;
-
- if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
code = gimple_assign_rhs_code (stmt);
if (code != ARRAY_REF
&& code != INDIRECT_REF
- && !STMT_VINFO_STRIDED_ACCESS (stmt_info))
+ && code != COMPONENT_REF
+ && code != IMAGPART_EXPR
+ && code != REALPART_EXPR)
return false;
if (!STMT_VINFO_DATA_REF (stmt_info))
{
strided_load = false;
vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
- }
+ if (SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance))
+ slp_perm = true;
+ }
else
vec_num = group_size;
This can only occur when vectorizing memory accesses in the inner-loop
nested within an outer-loop that is being vectorized. */
- if (nested_in_vect_loop_p (loop, stmt)
+ if (loop && nested_in_vect_loop_p (loop, stmt)
&& (TREE_INT_CST_LOW (DR_STEP (dr))
% GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
{
{
gcc_assert (phi);
if (i == vec_num - 1 && j == ncopies - 1)
- add_phi_arg (phi, lsq, loop_latch_edge (containing_loop));
+ add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
+ UNKNOWN_LOCATION);
msq = lsq;
}
}
/* 4. Handle invariant-load. */
- if (inv_p)
+ if (inv_p && !bb_vinfo)
{
gcc_assert (!strided_load);
gcc_assert (nested_in_vect_loop_p (loop, stmt));
if (slp_perm)
{
- if (!vect_transform_slp_perm_load (stmt, dr_chain, gsi,
- LOOP_VINFO_VECT_FACTOR (loop_vinfo),
+ if (!vect_transform_slp_perm_load (stmt, dr_chain, gsi, vf,
slp_node_instance, false))
{
VEC_free (tree, heap, dr_chain);
if (TREE_CODE (lhs) == SSA_NAME)
{
gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
- if (!vect_is_simple_use (lhs, loop_vinfo, &lhs_def_stmt, &def, &dt))
+ if (!vect_is_simple_use (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
+ &dt))
return false;
}
else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
if (TREE_CODE (rhs) == SSA_NAME)
{
gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
- if (!vect_is_simple_use (rhs, loop_vinfo, &rhs_def_stmt, &def, &dt))
+ if (!vect_is_simple_use (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
+ &dt))
return false;
}
else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
Check if STMT is conditional modify expression that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
- at BSI.
+ at GSI.
+
+ When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
+ to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
+ else caluse if it is 2).
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
-static bool
+bool
vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
- gimple *vec_stmt)
+ gimple *vec_stmt, tree reduc_def, int reduc_index)
{
tree scalar_dest = NULL_TREE;
tree vec_dest = NULL_TREE;
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
enum tree_code code;
+ /* FORNOW: unsupported in basic block SLP. */
+ gcc_assert (loop_vinfo);
+
gcc_assert (ncopies >= 1);
if (ncopies > 1)
return false; /* FORNOW */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
+ && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+ && reduc_def))
return false;
/* FORNOW: SLP not supported. */
return false;
/* FORNOW: not yet supported. */
- if (STMT_VINFO_LIVE_P (stmt_info))
+ if (STMT_VINFO_LIVE_P (stmt_info))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "value used after loop.");
if (TREE_CODE (then_clause) == SSA_NAME)
{
gimple then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
- if (!vect_is_simple_use (then_clause, loop_vinfo,
+ if (!vect_is_simple_use (then_clause, loop_vinfo, NULL,
&then_def_stmt, &def, &dt))
return false;
}
if (TREE_CODE (else_clause) == SSA_NAME)
{
gimple else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
- if (!vect_is_simple_use (else_clause, loop_vinfo,
+ if (!vect_is_simple_use (else_clause, loop_vinfo, NULL,
&else_def_stmt, &def, &dt))
return false;
}
if (!vec_stmt)
{
STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
- return expand_vec_cond_expr_p (op, vec_mode);
+ return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
}
/* Transform */
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt, NULL);
vec_cond_rhs =
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt, NULL);
- vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL);
- vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL);
+ if (reduc_index == 1)
+ vec_then_clause = reduc_def;
+ else
+ vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL);
+ if (reduc_index == 2)
+ vec_else_clause = reduc_def;
+ else
+ vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL);
/* Arguments are ready. Create the new vector stmt. */
vec_compare = build2 (TREE_CODE (cond_expr), vectype,
/* Make sure the statement is vectorizable. */
bool
-vect_analyze_stmt (gimple stmt, bool *need_to_vectorize)
+vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
+ HOST_WIDE_INT dummy;
+ tree scalar_type, vectype;
if (vect_print_dump_info (REPORT_DETAILS))
{
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
- /* Skip stmts that do not need to be vectorized. In loops this is expected
+ if (gimple_has_volatile_ops (stmt))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: stmt has volatile operands");
+
+ return false;
+ }
+
+ /* Skip stmts that do not need to be vectorized. In loops this is expected
to include:
- the COND_EXPR which is the loop exit condition
- any LABEL_EXPRs in the loop
- - computations that are used only for array indexing or loop control.
+ - computations that are used only for array indexing or loop control.
In basic blocks we only analyze statements that are a part of some SLP
instance, therefore, all the statements are relevant. */
- if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
{
if (vect_print_dump_info (REPORT_DETAILS))
break;
case vect_reduction_def:
- gcc_assert (relevance == vect_used_in_outer
+ case vect_nested_cycle:
+ gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
|| relevance == vect_used_in_outer_by_reduction
- || relevance == vect_unused_in_scope);
+ || relevance == vect_unused_in_scope));
break;
case vect_induction_def:
gcc_unreachable ();
}
+ if (bb_vinfo)
+ {
+ gcc_assert (PURE_SLP_STMT (stmt_info));
+
+ scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "get vectype for scalar type: ");
+ print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
+ }
+
+ vectype = get_vectype_for_scalar_type (scalar_type);
+ if (!vectype)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "not SLPed: unsupported data-type ");
+ print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
+ }
+ return false;
+ }
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "vectype: ");
+ print_generic_expr (vect_dump, vectype, TDF_SLIM);
+ }
+
+ STMT_VINFO_VECTYPE (stmt_info) = vectype;
+ }
+
if (STMT_VINFO_RELEVANT_P (stmt_info))
{
gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
}
ok = true;
- if (STMT_VINFO_RELEVANT_P (stmt_info)
- || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+ if (!bb_vinfo
+ && (STMT_VINFO_RELEVANT_P (stmt_info)
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
|| vectorizable_type_demotion (stmt, NULL, NULL, NULL)
|| vectorizable_conversion (stmt, NULL, NULL, NULL)
|| vectorizable_load (stmt, NULL, NULL, NULL, NULL)
|| vectorizable_call (stmt, NULL, NULL)
|| vectorizable_store (stmt, NULL, NULL, NULL)
- || vectorizable_condition (stmt, NULL, NULL)
- || vectorizable_reduction (stmt, NULL, NULL));
+ || vectorizable_reduction (stmt, NULL, NULL)
+ || vectorizable_condition (stmt, NULL, NULL, NULL, 0));
+ else
+ {
+ if (bb_vinfo)
+ ok = (vectorizable_operation (stmt, NULL, NULL, node)
+ || vectorizable_assignment (stmt, NULL, NULL, node)
+ || vectorizable_load (stmt, NULL, NULL, node, NULL)
+ || vectorizable_store (stmt, NULL, NULL, node));
+ }
if (!ok)
{
fprintf (vect_dump, "supported: ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
-
+
return false;
}
+ if (bb_vinfo)
+ return true;
+
/* Stmts that are (also) "live" (i.e. - that are used out of the loop)
need extra handling, except for vectorizable reductions. */
if (STMT_VINFO_LIVE_P (stmt_info)
fprintf (vect_dump, "supported: ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
-
+
return false;
}
if (!PURE_SLP_STMT (stmt_info))
{
- /* Groups of strided accesses whose size is not a power of 2 are not
- vectorizable yet using loop-vectorization. Therefore, if this stmt
- feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
- loop-based vectorized), the loop cannot be vectorized. */
+ /* Groups of strided accesses whose size is not a power of 2 are not
+ vectorizable yet using loop-vectorization. Therefore, if this stmt
+ feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
+ loop-based vectorized), the loop cannot be vectorized. */
if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
&& exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
DR_GROUP_FIRST_DR (stmt_info)))) == -1)
return false;
}
}
-
+
return true;
}
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
gimple orig_stmt_in_pattern;
bool done;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
switch (STMT_VINFO_TYPE (stmt_info))
{
case condition_vec_info_type:
gcc_assert (!slp_node);
- done = vectorizable_condition (stmt, gsi, &vec_stmt);
+ done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0);
gcc_assert (done);
break;
/* Handle inner-loop stmts whose DEF is used in the loop-nest that
is being vectorized, but outside the immediately enclosing loop. */
if (vec_stmt
- && nested_in_vect_loop_p (loop, stmt)
+ && STMT_VINFO_LOOP_VINFO (stmt_info)
+ && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
+ STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
&& STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
&& (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
- || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer_by_reduction))
+ || STMT_VINFO_RELEVANT (stmt_info) ==
+ vect_used_in_outer_by_reduction))
{
- struct loop *innerloop = loop->inner;
+ struct loop *innerloop = LOOP_VINFO_LOOP (
+ STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
imm_use_iterator imm_iter;
use_operand_p use_p;
tree scalar_dest;
gimple exit_phi;
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "Record the vdef for outer-loop vectorization.");
+ fprintf (vect_dump, "Record the vdef for outer-loop vectorization.");
/* Find the relevant loop-exit phi-node, and reord the vec_stmt there
(to be used when vectorizing outer-loop stmts that use the DEF of
Create and initialize a new stmt_vec_info struct for STMT. */
stmt_vec_info
-new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo)
+new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
+ bb_vec_info bb_vinfo)
{
stmt_vec_info res;
res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
STMT_VINFO_TYPE (res) = undef_vec_info_type;
STMT_VINFO_STMT (res) = stmt;
STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
+ STMT_VINFO_BB_VINFO (res) = bb_vinfo;
STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
STMT_VINFO_LIVE_P (res) = false;
STMT_VINFO_VECTYPE (res) = NULL;
/* Function vect_is_simple_use.
Input:
- LOOP - the loop that is being vectorized.
- OPERAND - operand of a stmt in LOOP.
+ LOOP_VINFO - the vect info of the loop that is being vectorized.
+ BB_VINFO - the vect info of the basic block that is being vectorized.
+ OPERAND - operand of a stmt in the loop or bb.
DEF - the defining stmt in case OPERAND is an SSA_NAME.
Returns whether a stmt with OPERAND can be vectorized.
- Supportable operands are constants, loop invariants, and operands that are
- defined by the current iteration of the loop. Unsupportable operands are
- those that are defined by a previous iteration of the loop (as is the case
- in reduction/induction computations). */
+ For loops, supportable operands are constants, loop invariants, and operands
+ that are defined by the current iteration of the loop. Unsupportable
+ operands are those that are defined by a previous iteration of the loop (as
+ is the case in reduction/induction computations).
+ For basic blocks, supportable operands are constants and bb invariants.
+ For now, operands defined outside the basic block are not supported. */
bool
-vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, gimple *def_stmt,
+vect_is_simple_use (tree operand, loop_vec_info loop_vinfo,
+ bb_vec_info bb_vinfo, gimple *def_stmt,
tree *def, enum vect_def_type *dt)
{
basic_block bb;
stmt_vec_info stmt_vinfo;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
+
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
*def_stmt = NULL;
*def = NULL_TREE;
*dt = vect_constant_def;
return true;
}
+
if (is_gimple_min_invariant (operand))
{
*def = operand;
fprintf (vect_dump, "non-associatable copy.");
operand = TREE_OPERAND (operand, 0);
}
+
if (TREE_CODE (operand) != SSA_NAME)
{
if (vect_print_dump_info (REPORT_DETAILS))
}
bb = gimple_bb (*def_stmt);
- if (!flow_bb_inside_loop_p (loop, bb))
+
+ if ((loop && !flow_bb_inside_loop_p (loop, bb))
+ || (!loop && bb != BB_VINFO_BB (bb_vinfo))
+ || (!loop && gimple_code (*def_stmt) == GIMPLE_PHI))
*dt = vect_external_def;
else
{
OSDN Git Service
