/* Analysis Utilities for Loop Vectorization.
- Copyright (C) 2003,2004,2005,2006 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
static bool exist_non_indexing_operands_for_use_p (tree, tree);
static tree vect_get_loop_niters (struct loop *, tree *);
static bool vect_analyze_data_ref_dependence
- (struct data_dependence_relation *, loop_vec_info, bool);
+ (struct data_dependence_relation *, loop_vec_info);
static bool vect_compute_data_ref_alignment (struct data_reference *);
static bool vect_analyze_data_ref_access (struct data_reference *);
static bool vect_can_advance_ivs_p (loop_vec_info);
int nbbs = loop->num_nodes;
block_stmt_iterator si;
unsigned int vectorization_factor = 0;
- int i;
tree scalar_type;
+ tree phi;
+ tree vectype;
+ unsigned int nunits;
+ stmt_vec_info stmt_info;
+ int i;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
{
basic_block bb = bbs[i];
+ for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
+ {
+ stmt_info = vinfo_for_stmt (phi);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "==> examining phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+
+ gcc_assert (stmt_info);
+
+ if (STMT_VINFO_RELEVANT_P (stmt_info))
+ {
+ gcc_assert (!STMT_VINFO_VECTYPE (stmt_info));
+ scalar_type = TREE_TYPE (PHI_RESULT (phi));
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "get vectype for scalar type: ");
+ print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
+ }
+
+ vectype = get_vectype_for_scalar_type (scalar_type);
+ if (!vectype)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ {
+ fprintf (vect_dump,
+ "not vectorized: unsupported data-type ");
+ print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
+ }
+ return false;
+ }
+ STMT_VINFO_VECTYPE (stmt_info) = vectype;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "vectype: ");
+ print_generic_expr (vect_dump, vectype, TDF_SLIM);
+ }
+
+ nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "nunits = %d", nunits);
+
+ if (!vectorization_factor
+ || (nunits > vectorization_factor))
+ vectorization_factor = nunits;
+ }
+ }
+
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
{
- tree stmt = bsi_stmt (si);
- unsigned int nunits;
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- tree vectype;
+ tree stmt = bsi_stmt (si);
+ stmt_info = vinfo_for_stmt (stmt);
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "==> examining statement: ");
- print_generic_expr (vect_dump, stmt, TDF_SLIM);
- }
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "==> examining statement: ");
+ print_generic_expr (vect_dump, stmt, TDF_SLIM);
+ }
+
+ gcc_assert (stmt_info);
- gcc_assert (stmt_info);
- /* skip stmts which do not need to be vectorized. */
- if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ /* skip stmts which do not need to be vectorized. */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "skip.");
- continue;
- }
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "skip.");
+ continue;
+ }
- if (VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (stmt))))
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump, "not vectorized: vector stmt in loop:");
- print_generic_expr (vect_dump, stmt, TDF_SLIM);
- }
- return false;
- }
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ {
+ fprintf (vect_dump, "not vectorized: irregular stmt.");
+ print_generic_expr (vect_dump, stmt, TDF_SLIM);
+ }
+ return false;
+ }
+
+ if (!GIMPLE_STMT_P (stmt)
+ && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (stmt))))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ {
+ fprintf (vect_dump, "not vectorized: vector stmt in loop:");
+ print_generic_expr (vect_dump, stmt, TDF_SLIM);
+ }
+ return false;
+ }
if (STMT_VINFO_VECTYPE (stmt_info))
{
+ /* The only case when a vectype had been already set is for stmts
+ that contain a dataref, or for "pattern-stmts" (stmts generated
+ by the vectorizer to represent/replace a certain idiom). */
+ gcc_assert (STMT_VINFO_DATA_REF (stmt_info)
+ || is_pattern_stmt_p (stmt_info));
vectype = STMT_VINFO_VECTYPE (stmt_info);
- scalar_type = TREE_TYPE (vectype);
}
else
{
- if (STMT_VINFO_DATA_REF (stmt_info))
- scalar_type =
- TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
- else if (TREE_CODE (stmt) == MODIFY_EXPR)
- scalar_type = TREE_TYPE (TREE_OPERAND (stmt, 0));
- else
- scalar_type = TREE_TYPE (stmt);
+ gcc_assert (! STMT_VINFO_DATA_REF (stmt_info)
+ && !is_pattern_stmt_p (stmt_info));
+
+ /* We set the vectype according to the type of the result (lhs).
+ For stmts whose result-type is different than the type of the
+ arguments (e.g. demotion, promotion), vectype will be reset
+ appropriately (later). Note that we have to visit the smallest
+ datatype in this function, because that determines the VF.
+ If the smallest datatype in the loop is present only as the
+ rhs of a promotion operation - we'd miss it here.
+ However, in such a case, that a variable of this datatype
+ does not appear in the lhs anywhere in the loop, it shouldn't
+ affect the vectorization factor. */
+ scalar_type = TREE_TYPE (GIMPLE_STMT_OPERAND (stmt, 0));
if (vect_print_dump_info (REPORT_DETAILS))
{
STMT_VINFO_VECTYPE (stmt_info) = vectype;
}
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "vectype: ");
- print_generic_expr (vect_dump, vectype, TDF_SLIM);
- }
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "vectype: ");
+ print_generic_expr (vect_dump, vectype, TDF_SLIM);
+ }
- nunits = TYPE_VECTOR_SUBPARTS (vectype);
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "nunits = %d", nunits);
+ nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "nunits = %d", nunits);
- if (!vectorization_factor
+ if (!vectorization_factor
|| (nunits > vectorization_factor))
vectorization_factor = nunits;
}
/* TODO: Analyze cost. Decide if worth while to vectorize. */
-
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vectorization factor = %d", vectorization_factor);
if (vectorization_factor <= 1)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
tree phi;
stmt_vec_info stmt_info;
bool need_to_vectorize = false;
+ int min_profitable_iters;
+ int min_scalar_loop_bound;
+ unsigned int th;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_operations ===");
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
{
+ ok = true;
+
stmt_info = vinfo_for_stmt (phi);
if (vect_print_dump_info (REPORT_DETAILS))
{
/* FORNOW: not yet supported. */
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
fprintf (vect_dump, "not vectorized: value used after loop.");
- return false;
- }
+ return false;
+ }
+
+ if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_loop
+ && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
+ {
+ /* A scalar-dependence cycle that we don't support. */
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: scalar dependence cycle.");
+ return false;
+ }
if (STMT_VINFO_RELEVANT_P (stmt_info))
{
- /* Most likely a reduction-like computation that is used
- in the loop. */
+ need_to_vectorize = true;
+ if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
+ ok = vectorizable_induction (phi, NULL, NULL);
+ }
+
+ if (!ok)
+ {
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: unsupported pattern.");
- return false;
+ {
+ fprintf (vect_dump,
+ "not vectorized: relevant phi not supported: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+ return false;
}
}
{
tree stmt = bsi_stmt (si);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ enum vect_def_type relevance = STMT_VINFO_RELEVANT (stmt_info);
if (vect_print_dump_info (REPORT_DETAILS))
{
continue;
}
- if (STMT_VINFO_RELEVANT_P (stmt_info))
- {
- gcc_assert (!VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (stmt))));
- gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
-
- ok = (vectorizable_type_promotion (stmt, NULL, NULL)
- || vectorizable_type_demotion (stmt, NULL, NULL)
- || vectorizable_operation (stmt, NULL, NULL)
- || vectorizable_assignment (stmt, NULL, NULL)
- || vectorizable_load (stmt, NULL, NULL)
- || vectorizable_store (stmt, NULL, NULL)
- || vectorizable_condition (stmt, NULL, NULL));
-
- if (!ok)
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump,
- "not vectorized: relevant stmt not supported: ");
- print_generic_expr (vect_dump, stmt, TDF_SLIM);
- }
- return false;
- }
- need_to_vectorize = true;
- }
-
- if (STMT_VINFO_LIVE_P (stmt_info))
+ switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
- ok = vectorizable_reduction (stmt, NULL, NULL);
+ case vect_loop_def:
+ break;
+
+ case vect_reduction_def:
+ gcc_assert (relevance == vect_unused_in_loop);
+ break;
+
+ case vect_induction_def:
+ case vect_constant_def:
+ case vect_invariant_def:
+ case vect_unknown_def_type:
+ default:
+ gcc_unreachable ();
+ }
- if (ok)
- need_to_vectorize = true;
- else
- ok = vectorizable_live_operation (stmt, NULL, NULL);
+ if (STMT_VINFO_RELEVANT_P (stmt_info))
+ {
+ gcc_assert (GIMPLE_STMT_P (stmt)
+ || !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (stmt))));
+ gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
+ need_to_vectorize = true;
+ }
- if (!ok)
+ ok = (vectorizable_type_promotion (stmt, NULL, NULL)
+ || vectorizable_type_demotion (stmt, NULL, NULL)
+ || vectorizable_conversion (stmt, NULL, NULL)
+ || vectorizable_operation (stmt, NULL, NULL)
+ || vectorizable_assignment (stmt, NULL, NULL)
+ || vectorizable_load (stmt, NULL, NULL)
+ || vectorizable_call (stmt, NULL, NULL)
+ || vectorizable_store (stmt, NULL, NULL)
+ || vectorizable_condition (stmt, NULL, NULL)
+ || vectorizable_reduction (stmt, NULL, NULL));
+
+ /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
+ need extra handling, except for vectorizable reductions. */
+ if (STMT_VINFO_LIVE_P (stmt_info)
+ && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
+ ok |= vectorizable_live_operation (stmt, NULL, NULL);
+
+ if (!ok)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- {
- fprintf (vect_dump,
- "not vectorized: live stmt not supported: ");
- print_generic_expr (vect_dump, stmt, TDF_SLIM);
- }
- return false;
+ fprintf (vect_dump, "not vectorized: stmt not supported: ");
+ print_generic_expr (vect_dump, stmt, TDF_SLIM);
}
- }
+ return false;
+ }
} /* stmts in bb */
} /* bbs */
- /* TODO: Analyze cost. Decide if worth while to vectorize. */
-
/* All operations in the loop are either irrelevant (deal with loop
control, or dead), or only used outside the loop and can be moved
out of the loop (e.g. invariants, inductions). The loop can be
vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor)
+ && (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: iteration count too small.");
+ fprintf (vect_dump, "not vectorized: iteration count too small.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,"not vectorized: iteration count smaller than "
+ "vectorization factor.");
return false;
}
+ /* Analyze cost. Decide if worth while to vectorize. */
+
+ min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
+ LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
+ if (min_profitable_iters < 0)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: vectorization not profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: vector version will never be "
+ "profitable.");
+ return false;
+ }
+
+ min_scalar_loop_bound = (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND))
+ * vectorization_factor;
+
+ /* Use the cost model only if it is more conservative than user specified
+ threshold. */
+
+ th = (unsigned) min_scalar_loop_bound;
+ if (min_profitable_iters
+ && (!min_scalar_loop_bound
+ || min_profitable_iters > min_scalar_loop_bound))
+ th = (unsigned) min_profitable_iters;
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: vectorization not "
+ "profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: iteration count smaller than "
+ "user specified loop bound parameter or minimum "
+ "profitable iterations (whichever is more conservative).");
+ return false;
+ }
+
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|| LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
|| LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
Therefore, all we need to check is if STMT falls into the
first case, and whether var corresponds to USE. */
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) == SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) == SSA_NAME)
return false;
- operand = TREE_OPERAND (stmt, 1);
+ operand = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (operand) != SSA_NAME)
return false;
tree phi;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block bb = loop->header;
- tree dummy;
+ tree dumy;
+ VEC(tree,heap) *worklist = VEC_alloc (tree, heap, 64);
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
+ /* First - identify all inductions. */
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
{
tree access_fn = NULL;
tree def = PHI_RESULT (phi);
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
- tree reduc_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
{
- fprintf (vect_dump, "Analyze phi: ");
- print_generic_expr (vect_dump, phi, TDF_SLIM);
+ fprintf (vect_dump, "Analyze phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
}
/* Skip virtual phi's. The data dependences that are associated with
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
-
if (!is_gimple_reg (SSA_NAME_VAR (def)))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "virtual phi. skip.");
- continue;
- }
+ continue;
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
/* Analyze the evolution function. */
-
access_fn = analyze_scalar_evolution (loop, def);
+ if (access_fn && vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Access function of PHI: ");
+ print_generic_expr (vect_dump, access_fn, TDF_SLIM);
+ }
- if (!access_fn)
- continue;
+ if (!access_fn
+ || !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy))
+ {
+ VEC_safe_push (tree, heap, worklist, phi);
+ continue;
+ }
if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "Access function of PHI: ");
- print_generic_expr (vect_dump, access_fn, TDF_SLIM);
- }
+ fprintf (vect_dump, "Detected induction.");
+ STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
+ }
- if (vect_is_simple_iv_evolution (loop->num, access_fn, &dummy, &dummy))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "Detected induction.");
- STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
- continue;
- }
- /* TODO: handle invariant phis */
+ /* Second - identify all reductions. */
+ while (VEC_length (tree, worklist) > 0)
+ {
+ tree phi = VEC_pop (tree, worklist);
+ tree def = PHI_RESULT (phi);
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
+ tree reduc_stmt;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Analyze phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+
+ gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
reduc_stmt = vect_is_simple_reduction (loop, phi);
if (reduc_stmt)
else
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Unknown def-use cycle pattern.");
-
}
+ VEC_free (tree, heap, worklist);
return;
}
static bool
vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
- loop_vec_info loop_vinfo,
- bool check_interleaving)
+ loop_vec_info loop_vinfo)
{
unsigned int i;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
{
/* Independent data accesses. */
- if (check_interleaving)
- vect_check_interleaving (dra, drb);
+ vect_check_interleaving (dra, drb);
return false;
}
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- continue;
+
+ /* For interleaving, mark that there is a read-write dependency if
+ necessary. We check before that one of the data-refs is store. */
+ if (DR_IS_READ (dra))
+ DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_a) = true;
+ else
+ {
+ if (DR_IS_READ (drb))
+ DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true;
+ }
+
+ continue;
}
if (abs (dist) >= vectorization_factor)
}
-/* Function vect_check_dependences.
-
- Return TRUE if there is a store-store or load-store dependence between
- data-refs in DDR, otherwise return FALSE. */
-
-static bool
-vect_check_dependences (struct data_dependence_relation *ddr)
-{
- struct data_reference *dra = DDR_A (ddr);
- struct data_reference *drb = DDR_B (ddr);
-
- if (DDR_ARE_DEPENDENT (ddr) == chrec_known || dra == drb)
- /* Independent or same data accesses. */
- return false;
-
- if (DR_IS_READ (dra) == DR_IS_READ (drb) && DR_IS_READ (dra))
- /* Two loads. */
- return false;
-
- if (vect_print_dump_info (REPORT_DR_DETAILS))
- {
- fprintf (vect_dump, "possible store or store/load dependence between ");
- print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
- fprintf (vect_dump, " and ");
- print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
- }
- return true;
-}
-
-
/* Function vect_analyze_data_ref_dependences.
Examine all the data references in the loop, and make sure there do not
unsigned int i;
VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo);
struct data_dependence_relation *ddr;
- bool check_interleaving = true;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_dependences ===");
- /* We allow interleaving only if there are no store-store and load-store
- dependencies in the loop. */
for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
- {
- if (vect_check_dependences (ddr))
- {
- check_interleaving = false;
- break;
- }
- }
-
- for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
- if (vect_analyze_data_ref_dependence (ddr, loop_vinfo, check_interleaving))
+ if (vect_analyze_data_ref_dependence (ddr, loop_vinfo))
return false;
return true;
fprintf (vect_dump, "vect_compute_data_ref_alignment:");
/* Initialize misalignment to unknown. */
- DR_MISALIGNMENT (dr) = -1;
+ SET_DR_MISALIGNMENT (dr, -1);
- misalign = DR_OFFSET_MISALIGNMENT (dr);
+ misalign = DR_INIT (dr);
aligned_to = DR_ALIGNED_TO (dr);
base_addr = DR_BASE_ADDRESS (dr);
base = build_fold_indirect_ref (base_addr);
vectype = STMT_VINFO_VECTYPE (stmt_info);
alignment = ssize_int (TYPE_ALIGN (vectype)/BITS_PER_UNIT);
- if ((aligned_to && tree_int_cst_compare (aligned_to, alignment) < 0)
- || !misalign)
+ if (tree_int_cst_compare (aligned_to, alignment) < 0)
{
if (vect_print_dump_info (REPORT_DETAILS))
{
return false;
}
- DR_MISALIGNMENT (dr) = TREE_INT_CST_LOW (misalign);
+ SET_DR_MISALIGNMENT (dr, TREE_INT_CST_LOW (misalign));
if (vect_print_dump_info (REPORT_DETAILS))
{
if (DR_GROUP_FIRST_DR (peel_stmt_info))
dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info);
- if (known_alignment_for_access_p (dr)
- && known_alignment_for_access_p (dr_peel)
- && (DR_MISALIGNMENT (dr) / dr_size ==
- DR_MISALIGNMENT (dr_peel) / dr_peel_size))
- {
- DR_MISALIGNMENT (dr) = 0;
- return;
- }
-
/* It can be assumed that the data refs with the same alignment as dr_peel
are aligned in the vector loop. */
same_align_drs
continue;
gcc_assert (DR_MISALIGNMENT (dr) / dr_size ==
DR_MISALIGNMENT (dr_peel) / dr_peel_size);
- DR_MISALIGNMENT (dr) = 0;
+ SET_DR_MISALIGNMENT (dr, 0);
return;
}
if (known_alignment_for_access_p (dr)
&& known_alignment_for_access_p (dr_peel))
{
- DR_MISALIGNMENT (dr) += npeel * dr_size;
- DR_MISALIGNMENT (dr) %= UNITS_PER_SIMD_WORD;
+ int misal = DR_MISALIGNMENT (dr);
+ misal += npeel * dr_size;
+ misal %= UNITS_PER_SIMD_WORD;
+ SET_DR_MISALIGNMENT (dr, misal);
return;
}
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Setting misalignment to -1.");
- DR_MISALIGNMENT (dr) = -1;
+ SET_DR_MISALIGNMENT (dr, -1);
}
vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
{
VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum dr_alignment_support supportable_dr_alignment;
struct data_reference *dr0 = NULL;
struct data_reference *dr;
can make all data references satisfy vect_supportable_dr_alignment.
If so, update data structures as needed and return true. Note that
at this time vect_supportable_dr_alignment is known to return false
- for a a misaligned write.
+ for a misaligned write.
B) If peeling wasn't possible and there is a data reference with an
unknown misalignment that does not satisfy vect_supportable_dr_alignment
{
/* For interleaved access we peel only if number of iterations in
the prolog loop ({VF - misalignment}), is a multiple of the
- number of the interelaved accesses. */
+ number of the interleaved accesses. */
int elem_size, mis_in_elements;
- int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ int nelements = TYPE_VECTOR_SUBPARTS (vectype);
/* FORNOW: handle only known alignment. */
if (!known_alignment_for_access_p (dr))
break;
}
- elem_size = UNITS_PER_SIMD_WORD / vf;
+ elem_size = UNITS_PER_SIMD_WORD / nelements;
mis_in_elements = DR_MISALIGNMENT (dr) / elem_size;
- if ((vf - mis_in_elements) % DR_GROUP_SIZE (stmt_info))
+ if ((nelements - mis_in_elements) % DR_GROUP_SIZE (stmt_info))
{
do_peeling = false;
break;
/* Often peeling for alignment will require peeling for loop-bound, which in
turn requires that we know how to adjust the loop ivs after the loop. */
- if (!vect_can_advance_ivs_p (loop_vinfo))
+ if (!vect_can_advance_ivs_p (loop_vinfo)
+ || !slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
do_peeling = false;
if (do_peeling)
{
int mis;
int npeel = 0;
+ tree stmt = DR_STMT (dr0);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ int nelements = TYPE_VECTOR_SUBPARTS (vectype);
if (known_alignment_for_access_p (dr0))
{
factor minus the misalignment as an element count. */
mis = DR_MISALIGNMENT (dr0);
mis /= GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr0))));
- npeel = LOOP_VINFO_VECT_FACTOR (loop_vinfo) - mis;
+ npeel = nelements - mis;
/* For interleaved data access every iteration accesses all the
members of the group, therefore we divide the number of iterations
save_misalignment = DR_MISALIGNMENT (dr);
vect_update_misalignment_for_peel (dr, dr0, npeel);
supportable_dr_alignment = vect_supportable_dr_alignment (dr);
- DR_MISALIGNMENT (dr) = save_misalignment;
+ SET_DR_MISALIGNMENT (dr, save_misalignment);
if (!supportable_dr_alignment)
{
LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0;
LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) = DR_MISALIGNMENT (dr0);
- DR_MISALIGNMENT (dr0) = 0;
+ SET_DR_MISALIGNMENT (dr0, 0);
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "Alignment of access forced using peeling.");
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
dr = STMT_VINFO_DATA_REF (stmt_info);
- DR_MISALIGNMENT (dr) = 0;
+ SET_DR_MISALIGNMENT (dr, 0);
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "Alignment of access forced using versioning.");
}
{
/* Skip same data-refs. In case that two or more stmts share data-ref
(supported only for loads), we vectorize only the first stmt, and
- the rest get their vectorized loads from the the first one. */
+ the rest get their vectorized loads from the first one. */
if (!tree_int_cst_compare (DR_INIT (data_ref),
DR_INIT (STMT_VINFO_DATA_REF (
vinfo_for_stmt (next)))))
{
- /* For load use the same data-ref load. (We check in
- vect_check_dependences() that there are no two stores to the
- same location). */
+ if (!DR_IS_READ (data_ref))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Two store stmts share the same dr.");
+ return false;
+ }
+
+ /* Check that there is no load-store dependencies for this loads
+ to prevent a case of load-store-load to the same location. */
+ if (DR_GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (next))
+ || DR_GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (prev)))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,
+ "READ_WRITE dependence in interleaving.");
+ return false;
+ }
+
+ /* For load use the same data-ref load. */
DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next)) = prev;
prev = next;
/* COUNT is the number of accesses found, we multiply it by the size of
the type to get COUNT_IN_BYTES. */
count_in_bytes = type_size * count;
- /* Check the size of the interleaving is not greater than STEP. */
+
+ /* Check that the size of the interleaving is not greater than STEP. */
if (dr_step < count_in_bytes)
{
if (vect_print_dump_info (REPORT_DETAILS))
return false;
}
+ /* Check that the size of the interleaving is equal to STEP for stores,
+ i.e., that there are no gaps. */
+ if (!DR_IS_READ (dr) && dr_step != count_in_bytes)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "interleaved store with gaps");
+ return false;
+ }
+
/* Check that STEP is a multiple of type size. */
if ((dr_step % type_size) != 0)
{
tree scalar_type;
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "=== vect_analyze_data_refs ===");
+ fprintf (vect_dump, "=== vect_analyze_data_refs ===\n");
compute_data_dependences_for_loop (loop, true,
&LOOP_VINFO_DATAREFS (loop_vinfo),
/* Update DR field in stmt_vec_info struct. */
stmt = DR_STMT (dr);
stmt_info = vinfo_for_stmt (stmt);
-
+
if (STMT_VINFO_DATA_REF (stmt_info))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
}
return false;
}
- if (!DR_MEMTAG (dr))
+
+ if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: base addr of dr is a "
+ "constant");
+ return false;
+ }
+
+ if (!DR_SYMBOL_TAG (dr))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
{
if (relevant > STMT_VINFO_RELEVANT (stmt_info))
STMT_VINFO_RELEVANT (stmt_info) = relevant;
- if (TREE_CODE (stmt) == PHI_NODE)
- /* Don't put phi-nodes in the worklist. Phis that are marked relevant
- or live will fail vectorization later on. */
- return;
-
if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
&& STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
{
}
+/*
+ Function process_use.
+
+ Inputs:
+ - a USE in STMT in a loop represented by LOOP_VINFO
+ - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
+ that defined USE. This is dont by calling mark_relevant and passing it
+ the WORKLIST (to add DEF_STMT to the WORKlist in case itis relevant).
+
+ Outputs:
+ Generally, LIVE_P and RELEVANT are used to define the liveness and
+ relevance info of the DEF_STMT of this USE:
+ STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
+ STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
+ Exceptions:
+ - case 1: If USE is used only for address computations (e.g. array indexing),
+ which does not need to be directly vectorized, then the liveness/relevance
+ of the respective DEF_STMT is left unchanged.
+ - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
+ skip DEF_STMT cause it had already been processed.
+
+ Return true if everything is as expected. Return false otherwise. */
+
+static bool
+process_use (tree stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
+ enum vect_relevant relevant, VEC(tree,heap) **worklist)
+{
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ stmt_vec_info dstmt_vinfo;
+ basic_block def_bb;
+ tree def, def_stmt;
+ enum vect_def_type dt;
+
+ /* case 1: we are only interested in uses that need to be vectorized. Uses
+ that are used for address computation are not considered relevant. */
+ if (!exist_non_indexing_operands_for_use_p (use, stmt))
+ return true;
+
+ if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
+ return false;
+ }
+
+ if (!def_stmt || IS_EMPTY_STMT (def_stmt))
+ return true;
+
+ def_bb = bb_for_stmt (def_stmt);
+ if (!flow_bb_inside_loop_p (loop, def_bb))
+ return true;
+
+ /* case 2: A reduction phi defining a reduction stmt (DEF_STMT). DEF_STMT
+ must have already been processed, so we just check that everything is as
+ expected, and we are done. */
+ dstmt_vinfo = vinfo_for_stmt (def_stmt);
+ if (TREE_CODE (stmt) == PHI_NODE
+ && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
+ && TREE_CODE (def_stmt) != PHI_NODE
+ && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def)
+ {
+ if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
+ dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
+ gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
+ gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
+ || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_loop);
+ return true;
+ }
+
+ vect_mark_relevant (worklist, def_stmt, relevant, live_p);
+ return true;
+}
+
+
/* Function vect_mark_stmts_to_be_vectorized.
Not all stmts in the loop need to be vectorized. For example:
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
unsigned int nbbs = loop->num_nodes;
block_stmt_iterator si;
- tree stmt, use;
+ tree stmt;
stmt_ann_t ann;
- ssa_op_iter iter;
unsigned int i;
stmt_vec_info stmt_vinfo;
basic_block bb;
tree phi;
bool live_p;
enum vect_relevant relevant;
- tree def, def_stmt;
- enum vect_def_type dt;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
worklist = VEC_alloc (tree, heap, 64);
/* 1. Init worklist. */
-
- bb = loop->header;
- for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "init: phi relevant? ");
- print_generic_expr (vect_dump, phi, TDF_SLIM);
- }
-
- if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
- vect_mark_relevant (&worklist, phi, relevant, live_p);
- }
-
for (i = 0; i < nbbs; i++)
{
bb = bbs[i];
+ for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "init: phi relevant? ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+
+ if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
+ vect_mark_relevant (&worklist, phi, relevant, live_p);
+ }
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
{
stmt = bsi_stmt (si);
-
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "init: stmt relevant? ");
}
}
-
/* 2. Process_worklist */
-
while (VEC_length (tree, worklist) > 0)
{
- stmt = VEC_pop (tree, worklist);
+ use_operand_p use_p;
+ ssa_op_iter iter;
+ stmt = VEC_pop (tree, worklist);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "worklist: examine stmt: ");
print_generic_expr (vect_dump, stmt, TDF_SLIM);
}
- /* Examine the USEs of STMT. For each ssa-name USE that is defined
- in the loop, mark the stmt that defines it (DEF_STMT) as
- relevant/irrelevant and live/dead according to the liveness and
- relevance properties of STMT.
- */
-
- gcc_assert (TREE_CODE (stmt) != PHI_NODE);
-
+ /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
+ (DEF_STMT) as relevant/irrelevant and live/dead according to the
+ liveness and relevance properties of STMT. */
ann = stmt_ann (stmt);
stmt_vinfo = vinfo_for_stmt (stmt);
-
relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
/* Generally, the liveness and relevance properties of STMT are
- propagated to the DEF_STMTs of its USEs:
- STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
- STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
-
- Exceptions:
-
- (case 1)
- If USE is used only for address computations (e.g. array indexing),
- which does not need to be directly vectorized, then the
- liveness/relevance of the respective DEF_STMT is left unchanged.
-
- (case 2)
- If STMT has been identified as defining a reduction variable, then
- we have two cases:
- (case 2.1)
- The last use of STMT is the reduction-variable, which is defined
- by a loop-header-phi. We don't want to mark the phi as live or
- relevant (because it does not need to be vectorized, it is handled
- as part of the vectorization of the reduction), so in this case we
- skip the call to vect_mark_relevant.
- (case 2.2)
- The rest of the uses of STMT are defined in the loop body. For
- the def_stmt of these uses we want to set liveness/relevance
- as follows:
- STMT_VINFO_LIVE_P (DEF_STMT_info) <-- false
- STMT_VINFO_RELEVANT (DEF_STMT_info) <-- vect_used_by_reduction
- because even though STMT is classified as live (since it defines a
- value that is used across loop iterations) and irrelevant (since it
- is not used inside the loop), it will be vectorized, and therefore
- the corresponding DEF_STMTs need to marked as relevant.
- We distinguish between two kinds of relevant stmts - those that are
- used by a reduction conputation, and those that are (also) used by a regular computation. This allows us later on to identify stmts
- that are used solely by a reduction, and therefore the order of
- the results that they produce does not have to be kept.
- */
-
- /* case 2.2: */
+ propagated as is to the DEF_STMTs of its USEs:
+ live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
+ relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
+
+ One exception is when STMT has been identified as defining a reduction
+ variable; in this case we set the liveness/relevance as follows:
+ live_p = false
+ relevant = vect_used_by_reduction
+ This is because we distinguish between two kinds of relevant stmts -
+ those that are used by a reduction computation, and those that are
+ (also) used by a regular computation. This allows us later on to
+ identify stmts that are used solely by a reduction, and therefore the
+ order of the results that they produce does not have to be kept.
+
+ Reduction phis are expected to be used by a reduction stmt; Other
+ reduction stmts are expected to be unused in the loop. These are the
+ expected values of "relevant" for reduction phis/stmts in the loop:
+
+ relevance: phi stmt
+ vect_unused_in_loop ok
+ vect_used_by_reduction ok
+ vect_used_in_loop */
+
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
{
- gcc_assert (relevant == vect_unused_in_loop && live_p);
- relevant = vect_used_by_reduction;
- live_p = false;
- }
+ switch (relevant)
+ {
+ case vect_unused_in_loop:
+ gcc_assert (TREE_CODE (stmt) != PHI_NODE);
+ break;
+ case vect_used_by_reduction:
+ if (TREE_CODE (stmt) == PHI_NODE)
+ break;
+ case vect_used_in_loop:
+ default:
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported use of reduction.");
+ VEC_free (tree, heap, worklist);
+ return false;
+ }
+ relevant = vect_used_by_reduction;
+ live_p = false;
+ }
- FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
+ FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
{
- /* case 1: we are only interested in uses that need to be vectorized.
- Uses that are used for address computation are not considered
- relevant.
- */
- if (!exist_non_indexing_operands_for_use_p (use, stmt))
- continue;
-
- if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
+ tree op = USE_FROM_PTR (use_p);
+ if (!process_use (stmt, op, loop_vinfo, live_p, relevant, &worklist))
+ {
VEC_free (tree, heap, worklist);
- return false;
- }
-
- if (!def_stmt || IS_EMPTY_STMT (def_stmt))
- continue;
-
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "worklist: examine use %d: ", i);
- print_generic_expr (vect_dump, use, TDF_SLIM);
- }
-
- bb = bb_for_stmt (def_stmt);
- if (!flow_bb_inside_loop_p (loop, bb))
- continue;
-
- /* case 2.1: the reduction-use does not mark the defining-phi
- as relevant. */
- if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
- && TREE_CODE (def_stmt) == PHI_NODE)
- continue;
-
- vect_mark_relevant (&worklist, def_stmt, relevant, live_p);
+ return false;
+ }
}
- } /* while worklist */
+ } /* while worklist */
VEC_free (tree, heap, worklist);
return true;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== get_loop_niters ===");
- niters = number_of_iterations_in_loop (loop);
+ niters = number_of_exit_cond_executions (loop);
if (niters != NULL_TREE
&& niters != chrec_dont_know)
return false;
}
- loop_vinfo = new_loop_vec_info (loop);
- LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
-
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
+ if (!NITERS_KNOWN_P (number_of_iterations))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
print_generic_expr (vect_dump, number_of_iterations, TDF_DETAILS);
}
}
- else
- if (LOOP_VINFO_INT_NITERS (loop_vinfo) == 0)
+ else if (TREE_INT_CST_LOW (number_of_iterations) == 0)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
fprintf (vect_dump, "not vectorized: number of iterations = 0.");
return NULL;
}
+ loop_vinfo = new_loop_vec_info (loop);
+ LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
LOOP_VINFO_EXIT_COND (loop_vinfo) = loop_cond;
+ gcc_assert (!loop->aux);
+ loop->aux = loop_vinfo;
return loop_vinfo;
}
OSDN Git Service
