+2009-04-28 Ira Rosen <irar@il.ibm.com>
+
+ * tree-vect-loop-manip.c (vect_create_cond_for_alias_checks):
+ Use REPORT_VECTORIZED_LOCATIONS instead
+ REPORT_VECTORIZED_LOOPS.
+ * tree-vectorizer.c (vect_verbosity_level): Make static.
+ (vect_loop_location): Rename to vect_location.
+ (vect_set_verbosity_level): Update comment.
+ (vect_set_dump_settings): Use REPORT_VECTORIZED_LOCATIONS
+ and vect_location.
+ (vectorize_loops): Fix comment. Use REPORT_VECTORIZED_LOCATIONS
+ and vect_location. Use REPORT_UNVECTORIZED_LOCATIONS
+ instead REPORT_UNVECTORIZED_LOOPS.
+ * tree-vectorizer.h (enum vect_def_type): Rename vect_invariant_def and
+ vect_loop_def to vect_external_def and vect_internal_def.
+ (enum verbosity_levels): Rename REPORT_VECTORIZED_LOOPS
+ and REPORT_UNVECTORIZED_LOOPS to REPORT_VECTORIZED_LOCATIONS and
+ REPORT_UNVECTORIZED_LOCATIONS.
+ (enum vect_relevant): Update comment. Rename vect_unused_in_loop
+ and vect_used_in_loop and to vect_unused_in_scope and
+ vect_used_in_scope.
+ (STMT_VINFO_RELEVANT_P): Use vect_unused_in_scope.
+ (vect_verbosity_level): Remove declaration.
+ (vect_analyze_operations): Likewise.
+ (vect_analyze_stmt): Declare.
+ * tree-vect-loop.c (vect_determine_vectorization_factor): Use
+ REPORT_UNVECTORIZED_LOCATIONS.
+ (vect_get_loop_niters): Fix indentation.
+ (vect_analyze_loop_form): Use REPORT_UNVECTORIZED_LOCATIONS.
+ (vect_analyze_loop_operations): New function.
+ (vect_analyze_loop): Call vect_analyze_loop_operations instead of
+ vect_analyze_operations.
+ (vect_is_simple_reduction): Use new names.
+ (vectorizable_live_operation, vect_transform_loop): Likewise.
+ * tree-vect-data-refs.c (vect_check_interleaving): Add a return value to
+ specify whether the data references can be a part of interleaving chain.
+ (vect_analyze_data_ref_dependence): Use new names.
+ (vect_analyze_data_refs_alignment, vect_analyze_data_refs): Likewise.
+ (vect_create_addr_base_for_vector_ref): Remove redundant code.
+ * tree-vect-patterns.c (widened_name_p): Use new names.
+ (vect_recog_dot_prod_pattern): Likewise.
+ * tree-vect-stmts.c (vect_stmt_relevant_p): Use new names.
+ (process_use, vect_mark_stmts_to_be_vectorized,
+ vect_model_simple_cost, vect_model_store_cost,
+ vect_get_vec_def_for_operand, vect_get_vec_def_for_stmt_copy,
+ vectorizable_call, vectorizable_conversion, vectorizable_assignment,
+ vectorizable_operation, vectorizable_type_demotion,
+ vectorizable_type_promotion, vectorizable_store, vectorizable_load,
+ vectorizable_condition): Likewise.
+ (vect_analyze_operations): Split into vect_analyze_loop_operations
+ and ...
+ (vect_analyze_stmt): ... new function.
+ (new_stmt_vec_info): Use new names.
+ (vect_is_simple_use): Use new names and fix comment.
+ * tree-vect-slp.c (vect_get_and_check_slp_defs): Use new names.
+ (vect_build_slp_tree, vect_analyze_slp, vect_schedule_slp): Likewise.
+
2009-04-28 Uros Bizjak <ubizjak@gmail.com>
PR target/39911
Check if DRA and DRB are a part of interleaving. In case they are, insert
DRA and DRB in an interleaving chain. */
-static void
+static bool
vect_check_interleaving (struct data_reference *dra,
struct data_reference *drb)
{
|| !vect_equal_offsets (DR_OFFSET (dra), DR_OFFSET (drb))
|| !tree_int_cst_compare (DR_INIT (dra), DR_INIT (drb))
|| DR_IS_READ (dra) != DR_IS_READ (drb))
- return;
+ return false;
/* Check:
1. data-refs are of the same type
2. their steps are equal
- 3. the step is greater than the difference between data-refs' inits */
+ 3. the step (if greater than zero) is greater than the difference between
+ data-refs' inits. */
type_size_a = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra))));
type_size_b = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb))));
|| tree_int_cst_compare (DR_STEP (dra), DR_STEP (drb))
|| !types_compatible_p (TREE_TYPE (DR_REF (dra)),
TREE_TYPE (DR_REF (drb))))
- return;
+ return false;
init_a = TREE_INT_CST_LOW (DR_INIT (dra));
init_b = TREE_INT_CST_LOW (DR_INIT (drb));
diff_mod_size = (init_a - init_b) % type_size_a;
if ((init_a - init_b) > step)
- return;
+ return false;
if (diff_mod_size == 0)
{
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- return;
+ return true;
}
}
else
diff_mod_size = (init_b - init_a) % type_size_a;
if ((init_b - init_a) > step)
- return;
+ return false;
if (diff_mod_size == 0)
{
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- return;
+ return true;
}
}
+
+ return false;
}
/* Check if data references pointed by DR_I and DR_J are same or
continue;
}
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized, possible dependence "
supportable_dr_alignment = vect_supportable_dr_alignment (dr);
if (!supportable_dr_alignment)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
if (DR_IS_READ (dr))
fprintf (vect_dump,
Return FALSE if a data reference is found that cannot be vectorized. */
bool
-vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
+vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_data_refs_alignment ===");
if (!vect_compute_data_refs_alignment (loop_vinfo))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump,
"not vectorized: can't calculate alignment for data ref.");
return false;
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
if (!vect_analyze_data_ref_access (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: complicated access pattern.");
return false;
}
if (!dr || !DR_REF (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unhandled data-ref ");
return false;
}
if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
|| !DR_STEP (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump, "not vectorized: data ref analysis failed ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: base addr of dr is a "
"constant");
return false;
if (STMT_VINFO_DATA_REF (stmt_info))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: more than one data ref in stmt: ");
}
return false;
}
+
STMT_VINFO_DATA_REF (stmt_info) = dr;
/* Set vectype for STMT. */
get_vectype_for_scalar_type (scalar_type);
if (!STMT_VINFO_VECTYPE (stmt_info))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: no vectype for stmt: ");
gimple_seq seq = NULL;
tree base_offset = unshare_expr (DR_OFFSET (dr));
tree init = unshare_expr (DR_INIT (dr));
- tree vect_ptr_type, addr_expr2;
+ tree vect_ptr_type;
tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
gcc_assert (loop);
vect_ptr_type = build_pointer_type (STMT_VINFO_VECTYPE (stmt_info));
- /* addr_expr = addr_base */
+ vec_stmt = fold_convert (vect_ptr_type, addr_base);
addr_expr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
get_name (base_name));
+
add_referenced_var (addr_expr);
- vec_stmt = fold_convert (vect_ptr_type, addr_base);
- addr_expr2 = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
- get_name (base_name));
- add_referenced_var (addr_expr2);
- vec_stmt = force_gimple_operand (vec_stmt, &seq, false, addr_expr2);
+ vec_stmt = force_gimple_operand (vec_stmt, &seq, false, addr_expr);
gimple_seq_add_seq (new_stmt_list, seq);
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "created ");
print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
}
+
return vec_stmt;
}
else
*cond_expr = part_cond_expr;
}
- if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
fprintf (vect_dump, "created %u versioning for alias checks.\n",
VEC_length (ddr_p, may_alias_ddrs));
vectype = get_vectype_for_scalar_type (scalar_type);
if (!vectype)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: unsupported data-type ");
if (gimple_get_lhs (stmt) == NULL_TREE)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump, "not vectorized: irregular stmt.");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
if (VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump, "not vectorized: vector stmt in loop:");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
vectype = get_vectype_for_scalar_type (scalar_type);
if (!vectype)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: unsupported data-type ");
fprintf (vect_dump, "vectorization factor = %d", vectorization_factor);
if (vectorization_factor <= 1)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unsupported data-type");
return false;
}
vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
}
-
/* Function vect_get_loop_niters.
Determine how many iterations the loop is executed.
*number_of_iterations = niters;
if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "==> get_loop_niters:" );
- print_generic_expr (vect_dump, *number_of_iterations, TDF_SLIM);
- }
+ {
+ fprintf (vect_dump, "==> get_loop_niters:" );
+ print_generic_expr (vect_dump, *number_of_iterations, TDF_SLIM);
+ }
}
return get_loop_exit_condition (loop);
}
else if (TREE_INT_CST_LOW (number_of_iterations) == 0)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: number of iterations = 0.");
if (inner_loop_vinfo)
destroy_loop_vec_info (inner_loop_vinfo, false);
return loop_vinfo;
}
+
+/* Function vect_analyze_loop_operations.
+
+ Scan the loop stmts and make sure they are all vectorizable. */
+
+static bool
+vect_analyze_loop_operations (loop_vec_info loop_vinfo)
+{
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
+ int nbbs = loop->num_nodes;
+ gimple_stmt_iterator si;
+ unsigned int vectorization_factor = 0;
+ int i;
+ gimple phi;
+ stmt_vec_info stmt_info;
+ bool need_to_vectorize = false;
+ int min_profitable_iters;
+ int min_scalar_loop_bound;
+ unsigned int th;
+ bool only_slp_in_loop = true, ok;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "=== vect_analyze_loop_operations ===");
+
+ gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+ vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+
+ for (i = 0; i < nbbs; i++)
+ {
+ basic_block bb = bbs[i];
+
+ for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
+ {
+ phi = gsi_stmt (si);
+ ok = true;
+
+ stmt_info = vinfo_for_stmt (phi);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "examining phi: ");
+ print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
+ }
+
+ if (! is_loop_header_bb_p (bb))
+ {
+ /* inner-loop loop-closed exit phi in outer-loop vectorization
+ (i.e. a phi in the tail of the outer-loop).
+ FORNOW: we currently don't support the case that these phis
+ are not used in the outerloop, cause this case requires
+ to actually do something here. */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ || STMT_VINFO_LIVE_P (stmt_info))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,
+ "Unsupported loop-closed phi in outer-loop.");
+ return false;
+ }
+ continue;
+ }
+
+ gcc_assert (stmt_info);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: value used after loop.");
+ return false;
+ }
+
+ if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
+ && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
+ {
+ /* A scalar-dependence cycle that we don't support. */
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: scalar dependence cycle.");
+ return false;
+ }
+
+ if (STMT_VINFO_RELEVANT_P (stmt_info))
+ {
+ need_to_vectorize = true;
+ if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
+ ok = vectorizable_induction (phi, NULL, NULL);
+ }
+
+ if (!ok)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ {
+ fprintf (vect_dump,
+ "not vectorized: relevant phi not supported: ");
+ print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
+ }
+ return false;
+ }
+ }
+
+ for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+ {
+ gimple stmt = gsi_stmt (si);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+
+ gcc_assert (stmt_info);
+
+ if (!vect_analyze_stmt (stmt, &need_to_vectorize))
+ return false;
+
+ if (STMT_VINFO_RELEVANT_P (stmt_info) && !PURE_SLP_STMT (stmt_info))
+ /* STMT needs both SLP and loop-based vectorization. */
+ only_slp_in_loop = false;
+ }
+ } /* bbs */
+
+ /* All operations in the loop are either irrelevant (deal with loop
+ control, or dead), or only used outside the loop and can be moved
+ out of the loop (e.g. invariants, inductions). The loop can be
+ optimized away by scalar optimizations. We're better off not
+ touching this loop. */
+ if (!need_to_vectorize)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,
+ "All the computation can be taken out of the loop.");
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump,
+ "not vectorized: redundant loop. no profit to vectorize.");
+ return false;
+ }
+
+ /* If all the stmts in the loop can be SLPed, we perform only SLP, and
+ vectorization factor of the loop is the unrolling factor required by the
+ SLP instances. If that unrolling factor is 1, we say, that we perform
+ pure SLP on loop - cross iteration parallelism is not exploited. */
+ if (only_slp_in_loop)
+ vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
+ else
+ vectorization_factor = least_common_multiple (vectorization_factor,
+ LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
+
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,
+ "vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC,
+ vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: iteration count too small.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,"not vectorized: iteration count smaller than "
+ "vectorization factor.");
+ return false;
+ }
+
+ /* Analyze cost. Decide if worth while to vectorize. */
+
+ /* Once VF is set, SLP costs should be updated since the number of created
+ vector stmts depends on VF. */
+ vect_update_slp_costs_according_to_vf (loop_vinfo);
+
+ min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
+ LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
+
+ if (min_profitable_iters < 0)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: vectorization not profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: vector version will never be "
+ "profitable.");
+ return false;
+ }
+
+ min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
+ * vectorization_factor) - 1);
+
+ /* Use the cost model only if it is more conservative than user specified
+ threshold. */
+
+ th = (unsigned) min_scalar_loop_bound;
+ if (min_profitable_iters
+ && (!min_scalar_loop_bound
+ || min_profitable_iters > min_scalar_loop_bound))
+ th = (unsigned) min_profitable_iters;
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && LOOP_VINFO_INT_NITERS (loop_vinfo) <= th)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump, "not vectorized: vectorization not "
+ "profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: iteration count smaller than "
+ "user specified loop bound parameter or minimum "
+ "profitable iterations (whichever is more conservative).");
+ return false;
+ }
+
+ if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ || LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
+ || LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "epilog loop required.");
+ if (!vect_can_advance_ivs_p (loop_vinfo))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump,
+ "not vectorized: can't create epilog loop 1.");
+ return false;
+ }
+ if (!slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ fprintf (vect_dump,
+ "not vectorized: can't create epilog loop 2.");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
/* Function vect_analyze_loop.
Apply a set of analyses on LOOP, and create a loop_vec_info struct
/* Scan all the operations in the loop and make sure they are
vectorizable. */
- ok = vect_analyze_operations (loop_vinfo);
+ ok = vect_analyze_loop_operations (loop_vinfo);
if (!ok)
{
if (vect_print_dump_info (REPORT_DETAILS))
/* Check that one def is the reduction def, defined by PHI,
- the other def is either defined in the loop ("vect_loop_def"),
+ the other def is either defined in the loop ("vect_internal_def"),
or it's an induction (defined by a loop-header phi-node). */
if (def2 == phi
&& (is_gimple_assign (def1)
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_induction_def
|| (gimple_code (def1) == GIMPLE_PHI
- && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_loop_def
+ && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_internal_def
&& !is_loop_header_bb_p (gimple_bb (def1)))))
{
if (vect_print_dump_info (REPORT_DETAILS))
&& (is_gimple_assign (def2)
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_induction_def
|| (gimple_code (def2) == GIMPLE_PHI
- && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_loop_def
+ && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_internal_def
&& !is_loop_header_bb_p (gimple_bb (def2)))))
{
/* Swap operands (just for simplicity - so that the rest of the code
/* Reductions that are not used even in an enclosing outer-loop,
are expected to be "live" (used out of the loop). */
- if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_loop
+ if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
&& !STMT_VINFO_LIVE_P (stmt_info))
return false;
is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, &def_stmt,
&def, &dt);
gcc_assert (is_simple_use);
- if (dt != vect_loop_def
- && dt != vect_invariant_def
+ if (dt != vect_internal_def
+ && dt != vect_external_def
&& dt != vect_constant_def
&& dt != vect_induction_def)
return false;
}
- is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, &def_stmt, &def, &dt);
+ is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, &def_stmt, &def,
+ &dt);
gcc_assert (is_simple_use);
gcc_assert (dt == vect_reduction_def);
gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
from the vectorized reduction operation generated in the previous iteration.
*/
- if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_loop)
+ if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope)
{
single_defuse_cycle = true;
epilog_copies = 1;
return false;
}
- if (dt != vect_invariant_def && dt != vect_constant_def)
+ if (dt != vect_external_def && dt != vect_constant_def)
return false;
}
until all the loops have been transformed? */
update_ssa (TODO_update_ssa);
- if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
fprintf (vect_dump, "LOOP VECTORIZED.");
- if (loop->inner && vect_print_dump_info (REPORT_VECTORIZED_LOOPS))
+ if (loop->inner && vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
fprintf (vect_dump, "OUTER LOOP VECTORIZED.");
}
if (!vect_is_simple_use (name, loop_vinfo, def_stmt, &def, &dt))
return false;
- if (dt != vect_loop_def
- && dt != vect_invariant_def && dt != vect_constant_def)
+ if (dt != vect_internal_def
+ && dt != vect_external_def && dt != vect_constant_def)
return false;
if (! *def_stmt)
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
- if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_internal_def)
return NULL;
if (gimple_assign_rhs_code (stmt) != MULT_EXPR)
return NULL;
return NULL;
stmt_vinfo = vinfo_for_stmt (stmt);
gcc_assert (stmt_vinfo);
- gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_loop_def);
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_internal_def);
oprnd00 = gimple_assign_rhs1 (stmt);
oprnd01 = gimple_assign_rhs2 (stmt);
}
switch (dt[i])
{
case vect_constant_def:
- case vect_invariant_def:
+ case vect_external_def:
break;
- case vect_loop_def:
+ case vect_internal_def:
if (i == 0)
VEC_safe_push (gimple, heap, *def_stmts0, def_stmt);
else
}
/* Create SLP_TREE nodes for the definition node/s. */
- if (first_stmt_dt0 == vect_loop_def)
+ if (first_stmt_dt0 == vect_internal_def)
{
slp_tree left_node = XNEW (struct _slp_tree);
SLP_TREE_SCALAR_STMTS (left_node) = def_stmts0;
SLP_TREE_LEFT (*node) = left_node;
}
- if (first_stmt_dt1 == vect_loop_def)
+ if (first_stmt_dt1 == vect_internal_def)
{
slp_tree right_node = XNEW (struct _slp_tree);
SLP_TREE_SCALAR_STMTS (right_node) = def_stmts1;
if (!vect_analyze_slp_instance (loop_vinfo, store))
{
/* SLP failed. No instance can be SLPed in the loop. */
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "SLP failed.");
return false;
is_store = vect_schedule_slp_instance (SLP_INSTANCE_TREE (instance),
instance, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
- if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS)
- || vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS)
+ || vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "vectorizing stmts using SLP.");
}
use_operand_p use_p;
def_operand_p def_p;
- *relevant = vect_unused_in_loop;
+ *relevant = vect_unused_in_scope;
*live_p = false;
/* cond stmt other than loop exit cond. */
if (is_ctrl_stmt (stmt)
- && STMT_VINFO_TYPE (vinfo_for_stmt (stmt)) != loop_exit_ctrl_vec_info_type)
- *relevant = vect_used_in_loop;
+ && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
+ != loop_exit_ctrl_vec_info_type)
+ *relevant = vect_used_in_scope;
/* changing memory. */
if (gimple_code (stmt) != GIMPLE_PHI)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "vec_stmt_relevant_p: stmt has vdefs.");
- *relevant = vect_used_in_loop;
+ *relevant = vect_used_in_scope;
}
/* uses outside the loop. */
if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
return false;
}
dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
- || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_loop);
+ || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
return true;
}
fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
switch (relevant)
{
- case vect_unused_in_loop:
+ case vect_unused_in_scope:
relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ?
- vect_used_by_reduction : vect_unused_in_loop;
+ vect_used_by_reduction : vect_unused_in_scope;
break;
case vect_used_in_outer_by_reduction:
relevant = vect_used_by_reduction;
break;
case vect_used_in_outer:
- relevant = vect_used_in_loop;
+ relevant = vect_used_in_scope;
break;
case vect_used_by_reduction:
- case vect_used_in_loop:
+ case vect_used_in_scope:
break;
default:
fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
switch (relevant)
{
- case vect_unused_in_loop:
+ case vect_unused_in_scope:
relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ?
- vect_used_in_outer_by_reduction : vect_unused_in_loop;
+ vect_used_in_outer_by_reduction : vect_unused_in_scope;
break;
case vect_used_in_outer_by_reduction:
relevant = vect_used_in_outer_by_reduction;
break;
- case vect_used_in_loop:
+ case vect_used_in_scope:
relevant = vect_used_in_outer;
break;
Here are the expected values of "relevant" for reduction phis/stmts:
relevance: phi stmt
- vect_unused_in_loop ok
+ vect_unused_in_scope ok
vect_used_in_outer_by_reduction ok ok
vect_used_in_outer ok ok
vect_used_by_reduction ok
- vect_used_in_loop */
+ vect_used_in_scope */
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
{
enum vect_relevant tmp_relevant = relevant;
switch (tmp_relevant)
{
- case vect_unused_in_loop:
+ case vect_unused_in_scope:
gcc_assert (gimple_code (stmt) != GIMPLE_PHI);
relevant = vect_used_by_reduction;
break;
if (gimple_code (stmt) == GIMPLE_PHI)
break;
/* fall through */
- case vect_used_in_loop:
+ case vect_used_in_scope:
default:
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "unsupported use of reduction.");
/* FORNOW: Assuming maximum 2 args per stmts. */
for (i = 0; i < 2; i++)
{
- if (dt[i] == vect_constant_def || dt[i] == vect_invariant_def)
+ if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
outside_cost += TARG_SCALAR_TO_VEC_COST;
}
if (PURE_SLP_STMT (stmt_info))
return;
- if (dt == vect_constant_def || dt == vect_invariant_def)
+ if (dt == vect_constant_def || dt == vect_external_def)
outside_cost = TARG_SCALAR_TO_VEC_COST;
/* Strided access? */
}
/* Case 2: operand is defined outside the loop - loop invariant. */
- case vect_invariant_def:
+ case vect_external_def:
{
vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
gcc_assert (vector_type);
}
/* Case 3: operand is defined inside the loop. */
- case vect_loop_def:
+ case vect_internal_def:
{
if (scalar_def)
*scalar_def = NULL/* FIXME tuples: def_stmt*/;
stmt_vec_info def_stmt_info;
/* Do nothing; can reuse same def. */
- if (dt == vect_invariant_def || dt == vect_constant_def )
+ if (dt == vect_external_def || dt == vect_constant_def )
return vec_oprnd;
vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* FORNOW: SLP not supported. */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
if (!is_gimple_assign (stmt))
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is vectorizable assignment? */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is STMT a vectorizable binary/unary operation? */
shift_p = true;
/* vector shifted by vector */
- if (dt[1] == vect_loop_def)
+ if (dt[1] == vect_internal_def)
{
optab = optab_for_tree_code (code, vectype, optab_vector);
if (vect_print_dump_info (REPORT_DETAILS))
/* See if the machine has a vector shifted by scalar insn and if not
then see if it has a vector shifted by vector insn */
- else if (dt[1] == vect_constant_def || dt[1] == vect_invariant_def)
+ else if (dt[1] == vect_constant_def || dt[1] == vect_external_def)
{
optab = optab_for_tree_code (code, vectype, optab_scalar);
if (optab
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is STMT a vectorizable type-demotion operation? */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is STMT a vectorizable type-promotion operation? */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is vectorizable store? */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* Is vectorizable load? */
if (!STMT_VINFO_RELEVANT_P (stmt_info))
return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
/* FORNOW: SLP not supported. */
}
-/* Function vect_analyze_operations.
-
- Scan the loop stmts and make sure they are all vectorizable. */
+/* Make sure the statement is vectorizable. */
bool
-vect_analyze_operations (loop_vec_info loop_vinfo)
+vect_analyze_stmt (gimple stmt, bool *need_to_vectorize)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
- int nbbs = loop->num_nodes;
- gimple_stmt_iterator si;
- unsigned int vectorization_factor = 0;
- int i;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
- gimple phi;
- stmt_vec_info stmt_info;
- bool need_to_vectorize = false;
- int min_profitable_iters;
- int min_scalar_loop_bound;
- unsigned int th;
- bool only_slp_in_loop = true;
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "=== vect_analyze_operations ===");
-
- gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
- vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-
- for (i = 0; i < nbbs; i++)
{
- basic_block bb = bbs[i];
-
- for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
- {
- phi = gsi_stmt (si);
- ok = true;
-
- stmt_info = vinfo_for_stmt (phi);
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "examining phi: ");
- print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
- }
-
- if (! is_loop_header_bb_p (bb))
- {
- /* inner-loop loop-closed exit phi in outer-loop vectorization
- (i.e. a phi in the tail of the outer-loop).
- FORNOW: we currently don't support the case that these phis
- are not used in the outerloop, cause this case requires
- to actually do something here. */
- if (!STMT_VINFO_RELEVANT_P (stmt_info)
- || STMT_VINFO_LIVE_P (stmt_info))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump,
- "Unsupported loop-closed phi in outer-loop.");
- return false;
- }
- continue;
- }
-
- gcc_assert (stmt_info);
-
- if (STMT_VINFO_LIVE_P (stmt_info))
- {
- /* FORNOW: not yet supported. */
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: value used after loop.");
- return false;
- }
-
- if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_loop
- && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
- {
- /* A scalar-dependence cycle that we don't support. */
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: scalar dependence cycle.");
- return false;
- }
-
- if (STMT_VINFO_RELEVANT_P (stmt_info))
- {
- need_to_vectorize = true;
- if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
- ok = vectorizable_induction (phi, NULL, NULL);
- }
-
- if (!ok)
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump,
- "not vectorized: relevant phi not supported: ");
- print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
- }
- return false;
- }
- }
-
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
- {
- gimple stmt = gsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
-
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "==> examining statement: ");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
-
- gcc_assert (stmt_info);
-
- /* skip stmts which do not need to be vectorized.
- this is expected to include:
- - the COND_EXPR which is the loop exit condition
- - any LABEL_EXPRs in the loop
- - computations that are used only for array indexing or loop
- control */
-
- if (!STMT_VINFO_RELEVANT_P (stmt_info)
- && !STMT_VINFO_LIVE_P (stmt_info))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "irrelevant.");
- continue;
- }
-
- switch (STMT_VINFO_DEF_TYPE (stmt_info))
- {
- case vect_loop_def:
- break;
-
- case vect_reduction_def:
- gcc_assert (relevance == vect_used_in_outer
- || relevance == vect_used_in_outer_by_reduction
- || relevance == vect_unused_in_loop);
- break;
-
- case vect_induction_def:
- case vect_constant_def:
- case vect_invariant_def:
- case vect_unknown_def_type:
- default:
- gcc_unreachable ();
- }
-
- if (STMT_VINFO_RELEVANT_P (stmt_info))
- {
- gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
- gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
- need_to_vectorize = true;
- }
-
- ok = true;
- if (STMT_VINFO_RELEVANT_P (stmt_info)
- || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
- ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
- || vectorizable_type_demotion (stmt, NULL, NULL, NULL)
- || vectorizable_conversion (stmt, NULL, NULL, NULL)
- || vectorizable_operation (stmt, NULL, NULL, NULL)
- || vectorizable_assignment (stmt, NULL, NULL, NULL)
- || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
- || vectorizable_call (stmt, NULL, NULL)
- || vectorizable_store (stmt, NULL, NULL, NULL)
- || vectorizable_condition (stmt, NULL, NULL)
- || vectorizable_reduction (stmt, NULL, NULL));
-
- if (!ok)
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump, "not vectorized: relevant stmt not ");
- fprintf (vect_dump, "supported: ");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
- return false;
- }
-
- /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
- need extra handling, except for vectorizable reductions. */
- if (STMT_VINFO_LIVE_P (stmt_info)
- && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
- ok = vectorizable_live_operation (stmt, NULL, NULL);
+ fprintf (vect_dump, "==> examining statement: ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
- if (!ok)
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump, "not vectorized: live stmt not ");
- fprintf (vect_dump, "supported: ");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
- return false;
- }
+ /* Skip stmts that do not need to be vectorized. In loops this is expected
+ to include:
+ - the COND_EXPR which is the loop exit condition
+ - any LABEL_EXPRs in the loop
+ - computations that are used only for array indexing or loop control.
+ In basic blocks we only analyze statements that are a part of some SLP
+ instance, therefore, all the statements are relevant. */
- if (!PURE_SLP_STMT (stmt_info))
- {
- /* STMT needs loop-based vectorization. */
- only_slp_in_loop = false;
-
- /* Groups of strided accesses whose size is not a power of 2 are
- not vectorizable yet using loop-vectorization. Therefore, if
- this stmt feeds non-SLP-able stmts (i.e., this stmt has to be
- both SLPed and loop-based vectorized), the loop cannot be
- vectorized. */
- if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
- && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
- DR_GROUP_FIRST_DR (stmt_info)))) == -1)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "not vectorized: the size of group "
- "of strided accesses is not a power of 2");
- print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
- }
- return false;
- }
- }
- } /* stmts in bb */
- } /* bbs */
-
- /* All operations in the loop are either irrelevant (deal with loop
- control, or dead), or only used outside the loop and can be moved
- out of the loop (e.g. invariants, inductions). The loop can be
- optimized away by scalar optimizations. We're better off not
- touching this loop. */
- if (!need_to_vectorize)
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ && !STMT_VINFO_LIVE_P (stmt_info))
{
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump,
- "All the computation can be taken out of the loop.");
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump,
- "not vectorized: redundant loop. no profit to vectorize.");
- return false;
- }
+ fprintf (vect_dump, "irrelevant.");
- /* If all the stmts in the loop can be SLPed, we perform only SLP, and
- vectorization factor of the loop is the unrolling factor required by the
- SLP instances. If that unrolling factor is 1, we say, that we perform
- pure SLP on loop - cross iteration parallelism is not exploited. */
- if (only_slp_in_loop)
- vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
- else
- vectorization_factor = least_common_multiple (vectorization_factor,
- LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
+ return true;
+ }
- LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
+ switch (STMT_VINFO_DEF_TYPE (stmt_info))
+ {
+ case vect_internal_def:
+ break;
- if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump,
- "vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC,
- vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
+ case vect_reduction_def:
+ gcc_assert (relevance == vect_used_in_outer
+ || relevance == vect_used_in_outer_by_reduction
+ || relevance == vect_unused_in_scope);
+ break;
+
+ case vect_induction_def:
+ case vect_constant_def:
+ case vect_external_def:
+ case vect_unknown_def_type:
+ default:
+ gcc_unreachable ();
+ }
- if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
+ if (STMT_VINFO_RELEVANT_P (stmt_info))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: iteration count too small.");
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump,"not vectorized: iteration count smaller than "
- "vectorization factor.");
- return false;
+ gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
+ gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
+ *need_to_vectorize = true;
}
- /* Analyze cost. Decide if worth while to vectorize. */
-
- /* Once VF is set, SLP costs should be updated since the number of created
- vector stmts depends on VF. */
- vect_update_slp_costs_according_to_vf (loop_vinfo);
-
- min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
- LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
-
- if (min_profitable_iters < 0)
+ ok = true;
+ if (STMT_VINFO_RELEVANT_P (stmt_info)
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+ ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
+ || vectorizable_type_demotion (stmt, NULL, NULL, NULL)
+ || vectorizable_conversion (stmt, NULL, NULL, NULL)
+ || vectorizable_operation (stmt, NULL, NULL, NULL)
+ || vectorizable_assignment (stmt, NULL, NULL, NULL)
+ || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
+ || vectorizable_call (stmt, NULL, NULL)
+ || vectorizable_store (stmt, NULL, NULL, NULL)
+ || vectorizable_condition (stmt, NULL, NULL)
+ || vectorizable_reduction (stmt, NULL, NULL));
+
+ if (!ok)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: vectorization not profitable.");
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "not vectorized: vector version will never be "
- "profitable.");
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ {
+ fprintf (vect_dump, "not vectorized: relevant stmt not ");
+ fprintf (vect_dump, "supported: ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
return false;
}
- min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
- * vectorization_factor) - 1);
-
- /* Use the cost model only if it is more conservative than user specified
- threshold. */
-
- th = (unsigned) min_scalar_loop_bound;
- if (min_profitable_iters
- && (!min_scalar_loop_bound
- || min_profitable_iters > min_scalar_loop_bound))
- th = (unsigned) min_profitable_iters;
+ /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
+ need extra handling, except for vectorizable reductions. */
+ if (STMT_VINFO_LIVE_P (stmt_info)
+ && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
+ ok = vectorizable_live_operation (stmt, NULL, NULL);
- if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && LOOP_VINFO_INT_NITERS (loop_vinfo) <= th)
+ if (!ok)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: vectorization not "
- "profitable.");
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "not vectorized: iteration count smaller than "
- "user specified loop bound parameter or minimum "
- "profitable iterations (whichever is more conservative).");
- return false;
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ {
+ fprintf (vect_dump, "not vectorized: live stmt not ");
+ fprintf (vect_dump, "supported: ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ return false;
}
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- || LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
- || LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
+ if (!PURE_SLP_STMT (stmt_info))
{
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "epilog loop required.");
- if (!vect_can_advance_ivs_p (loop_vinfo))
+ /* Groups of strided accesses whose size is not a power of 2 are not
+ vectorizable yet using loop-vectorization. Therefore, if this stmt
+ feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
+ loop-based vectorized), the loop cannot be vectorized. */
+ if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
+ && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
+ DR_GROUP_FIRST_DR (stmt_info)))) == -1)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump,
- "not vectorized: can't create epilog loop 1.");
- return false;
- }
- if (!slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump,
- "not vectorized: can't create epilog loop 2.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "not vectorized: the size of group "
+ "of strided accesses is not a power of 2");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
return false;
}
}
STMT_VINFO_TYPE (res) = undef_vec_info_type;
STMT_VINFO_STMT (res) = stmt;
STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
- STMT_VINFO_RELEVANT (res) = vect_unused_in_loop;
+ STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
STMT_VINFO_LIVE_P (res) = false;
STMT_VINFO_VECTYPE (res) = NULL;
STMT_VINFO_VEC_STMT (res) = NULL;
&& is_loop_header_bb_p (gimple_bb (stmt)))
STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
else
- STMT_VINFO_DEF_TYPE (res) = vect_loop_def;
+ STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
+
STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5);
STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0;
STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0;
if (is_gimple_min_invariant (operand))
{
*def = operand;
- *dt = vect_invariant_def;
+ *dt = vect_external_def;
return true;
}
print_gimple_stmt (vect_dump, *def_stmt, 0, TDF_SLIM);
}
- /* empty stmt is expected only in case of a function argument.
+ /* Empty stmt is expected only in case of a function argument.
(Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
if (gimple_nop_p (*def_stmt))
{
*def = operand;
- *dt = vect_invariant_def;
+ *dt = vect_external_def;
return true;
}
bb = gimple_bb (*def_stmt);
if (!flow_bb_inside_loop_p (loop, bb))
- *dt = vect_invariant_def;
+ *dt = vect_external_def;
else
{
stmt_vinfo = vinfo_for_stmt (*def_stmt);
/* vect_verbosity_level set to an invalid value
to mark that it's uninitialized. */
-enum verbosity_levels vect_verbosity_level = MAX_VERBOSITY_LEVEL;
+static enum verbosity_levels vect_verbosity_level = MAX_VERBOSITY_LEVEL;
/* Loop location. */
-LOC vect_loop_location;
+LOC vect_location;
/* Bitmap of virtual variables to be renamed. */
bitmap vect_memsyms_to_rename;
/* Function vect_set_verbosity_level.
- Called from toplev.c upon detection of the
+ Called from opts.c upon detection of the
-ftree-vectorizer-verbose=N option. */
void
if (dump_file && (dump_flags & TDF_DETAILS))
vect_verbosity_level = REPORT_DETAILS;
else if (dump_file && (dump_flags & TDF_STATS))
- vect_verbosity_level = REPORT_UNVECTORIZED_LOOPS;
+ vect_verbosity_level = REPORT_UNVECTORIZED_LOCATIONS;
else
vect_verbosity_level = REPORT_NONE;
if (!current_function_decl || !vect_dump)
return false;
- if (vect_loop_location == UNKNOWN_LOC)
+ if (vect_location == UNKNOWN_LOC)
fprintf (vect_dump, "\n%s:%d: note: ",
DECL_SOURCE_FILE (current_function_decl),
DECL_SOURCE_LINE (current_function_decl));
else
fprintf (vect_dump, "\n%s:%d: note: ",
- LOC_FILE (vect_loop_location), LOC_LINE (vect_loop_location));
+ LOC_FILE (vect_location), LOC_LINE (vect_location));
return true;
}
/* Function vectorize_loops.
- Entry Point to loop vectorization phase. */
+ Entry point to loop vectorization phase. */
unsigned
vectorize_loops (void)
/* Fix the verbosity level if not defined explicitly by the user. */
vect_set_dump_settings ();
- /* Allocate the bitmap that records which virtual variables that
+ /* Allocate the bitmap that records which virtual variables
need to be renamed. */
vect_memsyms_to_rename = BITMAP_ALLOC (NULL);
{
loop_vec_info loop_vinfo;
- vect_loop_location = find_loop_location (loop);
+ vect_location = find_loop_location (loop);
loop_vinfo = vect_analyze_loop (loop);
loop->aux = loop_vinfo;
vect_transform_loop (loop_vinfo);
num_vectorized_loops++;
}
- vect_loop_location = UNKNOWN_LOC;
+
+ vect_location = UNKNOWN_LOC;
statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)
- || (vect_print_dump_info (REPORT_VECTORIZED_LOOPS)
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)
+ || (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS)
&& num_vectorized_loops > 0))
fprintf (vect_dump, "vectorized %u loops in function.\n",
num_vectorized_loops);
/* Define type of def-use cross-iteration cycle. */
enum vect_def_type {
vect_uninitialized_def = 0,
- vect_constant_def,
- vect_invariant_def,
- vect_loop_def,
+ vect_constant_def = 1,
+ vect_external_def,
+ vect_internal_def,
vect_induction_def,
vect_reduction_def,
vect_unknown_def_type
/* Define verbosity levels. */
enum verbosity_levels {
REPORT_NONE,
- REPORT_VECTORIZED_LOOPS,
- REPORT_UNVECTORIZED_LOOPS,
+ REPORT_VECTORIZED_LOCATIONS,
+ REPORT_UNVECTORIZED_LOCATIONS,
REPORT_COST,
REPORT_ALIGNMENT,
REPORT_DR_DETAILS,
loop_exit_ctrl_vec_info_type
};
-/* Indicates whether/how a variable is used in the loop. */
+/* Indicates whether/how a variable is used in the scope of loop/basic
+ block. */
enum vect_relevant {
- vect_unused_in_loop = 0,
+ vect_unused_in_scope = 0,
vect_used_in_outer_by_reduction,
vect_used_in_outer,
computed. */
vect_used_by_reduction,
- vect_used_in_loop
+ vect_used_in_scope
};
/* The type of vectorization that can be applied to the stmt: regular loop-based
#define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
#define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
-#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_loop)
+#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
#define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
#define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
extern FILE *vect_dump;
extern LOC vect_loop_location;
-extern enum verbosity_levels vect_verbosity_level;
-
/* Bitmap of virtual variables to be renamed. */
extern bitmap vect_memsyms_to_rename;
-
/*-----------------------------------------------------------------*/
/* Function prototypes. */
/*-----------------------------------------------------------------*/
extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *,
bool *, slp_tree, slp_instance);
extern void vect_remove_stores (gimple);
-extern bool vect_analyze_operations (loop_vec_info);
+extern bool vect_analyze_stmt (gimple, bool *);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
OSDN Git Service
