/* Analysis Utilities for Loop Vectorization.
- Copyright (C) 2003,2004,2005,2006 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
print_generic_expr (vect_dump, stmt, TDF_SLIM);
}
- if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
- continue;
-
gcc_assert (stmt_info);
/* skip stmts which do not need to be vectorized. */
continue;
}
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ {
+ fprintf (vect_dump, "not vectorized: irregular stmt.");
+ print_generic_expr (vect_dump, stmt, TDF_SLIM);
+ }
+ return false;
+ }
+
if (!GIMPLE_STMT_P (stmt)
&& VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (stmt))))
{
}
/* TODO: Analyze cost. Decide if worth while to vectorize. */
-
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vectorization factor = %d", vectorization_factor);
if (vectorization_factor <= 1)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
tree phi;
stmt_vec_info stmt_info;
bool need_to_vectorize = false;
+ int min_profitable_iters;
+ int min_scalar_loop_bound;
+ unsigned int th;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_operations ===");
} /* stmts in bb */
} /* bbs */
- /* TODO: Analyze cost. Decide if worth while to vectorize. */
-
/* All operations in the loop are either irrelevant (deal with loop
control, or dead), or only used outside the loop and can be moved
out of the loop (e.g. invariants, inductions). The loop can be
vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && ((LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor)
- || (LOOP_VINFO_INT_NITERS (loop_vinfo) <=
- ((unsigned) (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND))
- * vectorization_factor))))
+ && (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: iteration count too small.");
+ fprintf (vect_dump, "not vectorized: iteration count too small.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump,"not vectorized: iteration count smaller than "
+ "vectorization factor.");
return false;
}
+ /* Analyze cost. Decide if worth while to vectorize. */
+
+ min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
+ LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
+ if (min_profitable_iters < 0)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: vectorization not profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: vector version will never be "
+ "profitable.");
+ return false;
+ }
+
+ min_scalar_loop_bound = (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND))
+ * vectorization_factor;
+
+ /* Use the cost model only if it is more conservative than user specified
+ threshold. */
+
+ th = (unsigned) min_scalar_loop_bound;
+ if (min_profitable_iters
+ && (!min_scalar_loop_bound
+ || min_profitable_iters > min_scalar_loop_bound))
+ th = (unsigned) min_profitable_iters;
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && LOOP_VINFO_INT_NITERS (loop_vinfo) < th)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: vectorization not "
+ "profitable.");
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "not vectorized: iteration count smaller than "
+ "user specified loop bound parameter or minimum "
+ "profitable iterations (whichever is more conservative).");
+ return false;
+ }
+
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|| LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
|| LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
fprintf (vect_dump, "vect_compute_data_ref_alignment:");
/* Initialize misalignment to unknown. */
- DR_MISALIGNMENT (dr) = -1;
+ SET_DR_MISALIGNMENT (dr, -1);
- misalign = DR_OFFSET_MISALIGNMENT (dr);
+ misalign = DR_INIT (dr);
aligned_to = DR_ALIGNED_TO (dr);
base_addr = DR_BASE_ADDRESS (dr);
base = build_fold_indirect_ref (base_addr);
vectype = STMT_VINFO_VECTYPE (stmt_info);
alignment = ssize_int (TYPE_ALIGN (vectype)/BITS_PER_UNIT);
- if ((aligned_to && tree_int_cst_compare (aligned_to, alignment) < 0)
- || !misalign)
+ if (tree_int_cst_compare (aligned_to, alignment) < 0)
{
if (vect_print_dump_info (REPORT_DETAILS))
{
return false;
}
- DR_MISALIGNMENT (dr) = TREE_INT_CST_LOW (misalign);
+ SET_DR_MISALIGNMENT (dr, TREE_INT_CST_LOW (misalign));
if (vect_print_dump_info (REPORT_DETAILS))
{
if (DR_GROUP_FIRST_DR (peel_stmt_info))
dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info);
- if (known_alignment_for_access_p (dr)
- && known_alignment_for_access_p (dr_peel)
- && (DR_MISALIGNMENT (dr) / dr_size ==
- DR_MISALIGNMENT (dr_peel) / dr_peel_size))
- {
- DR_MISALIGNMENT (dr) = 0;
- return;
- }
-
/* It can be assumed that the data refs with the same alignment as dr_peel
are aligned in the vector loop. */
same_align_drs
continue;
gcc_assert (DR_MISALIGNMENT (dr) / dr_size ==
DR_MISALIGNMENT (dr_peel) / dr_peel_size);
- DR_MISALIGNMENT (dr) = 0;
+ SET_DR_MISALIGNMENT (dr, 0);
return;
}
if (known_alignment_for_access_p (dr)
&& known_alignment_for_access_p (dr_peel))
{
- DR_MISALIGNMENT (dr) += npeel * dr_size;
- DR_MISALIGNMENT (dr) %= UNITS_PER_SIMD_WORD;
+ int misal = DR_MISALIGNMENT (dr);
+ misal += npeel * dr_size;
+ misal %= UNITS_PER_SIMD_WORD;
+ SET_DR_MISALIGNMENT (dr, misal);
return;
}
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Setting misalignment to -1.");
- DR_MISALIGNMENT (dr) = -1;
+ SET_DR_MISALIGNMENT (dr, -1);
}
vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
{
VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum dr_alignment_support supportable_dr_alignment;
struct data_reference *dr0 = NULL;
struct data_reference *dr;
the prolog loop ({VF - misalignment}), is a multiple of the
number of the interleaved accesses. */
int elem_size, mis_in_elements;
- int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ int nelements = TYPE_VECTOR_SUBPARTS (vectype);
/* FORNOW: handle only known alignment. */
if (!known_alignment_for_access_p (dr))
break;
}
- elem_size = UNITS_PER_SIMD_WORD / vf;
+ elem_size = UNITS_PER_SIMD_WORD / nelements;
mis_in_elements = DR_MISALIGNMENT (dr) / elem_size;
- if ((vf - mis_in_elements) % DR_GROUP_SIZE (stmt_info))
+ if ((nelements - mis_in_elements) % DR_GROUP_SIZE (stmt_info))
{
do_peeling = false;
break;
/* Often peeling for alignment will require peeling for loop-bound, which in
turn requires that we know how to adjust the loop ivs after the loop. */
- if (!vect_can_advance_ivs_p (loop_vinfo))
+ if (!vect_can_advance_ivs_p (loop_vinfo)
+ || !slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
do_peeling = false;
if (do_peeling)
{
int mis;
int npeel = 0;
+ tree stmt = DR_STMT (dr0);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ int nelements = TYPE_VECTOR_SUBPARTS (vectype);
if (known_alignment_for_access_p (dr0))
{
factor minus the misalignment as an element count. */
mis = DR_MISALIGNMENT (dr0);
mis /= GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr0))));
- npeel = LOOP_VINFO_VECT_FACTOR (loop_vinfo) - mis;
+ npeel = nelements - mis;
/* For interleaved data access every iteration accesses all the
members of the group, therefore we divide the number of iterations
save_misalignment = DR_MISALIGNMENT (dr);
vect_update_misalignment_for_peel (dr, dr0, npeel);
supportable_dr_alignment = vect_supportable_dr_alignment (dr);
- DR_MISALIGNMENT (dr) = save_misalignment;
+ SET_DR_MISALIGNMENT (dr, save_misalignment);
if (!supportable_dr_alignment)
{
LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0;
LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) = DR_MISALIGNMENT (dr0);
- DR_MISALIGNMENT (dr0) = 0;
+ SET_DR_MISALIGNMENT (dr0, 0);
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "Alignment of access forced using peeling.");
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
dr = STMT_VINFO_DATA_REF (stmt_info);
- DR_MISALIGNMENT (dr) = 0;
+ SET_DR_MISALIGNMENT (dr, 0);
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "Alignment of access forced using versioning.");
}
}
return false;
}
- if (!DR_MEMTAG (dr))
+
+ if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: base addr of dr is a "
+ "constant");
+ return false;
+ }
+
+ if (!DR_SYMBOL_TAG (dr))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
{
}
+/*
+ Function process_use.
+
+ Inputs:
+ - a USE in STMT in a loop represented by LOOP_VINFO
+ - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
+ that defined USE. This is dont by calling mark_relevant and passing it
+ the WORKLIST (to add DEF_STMT to the WORKlist in case itis relevant).
+
+ Outputs:
+ Generally, LIVE_P and RELEVANT are used to define the liveness and
+ relevance info of the DEF_STMT of this USE:
+ STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
+ STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
+ Exceptions:
+ - case 1: If USE is used only for address computations (e.g. array indexing),
+ which does not need to be directly vectorized, then the liveness/relevance
+ of the respective DEF_STMT is left unchanged.
+ - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
+ skip DEF_STMT cause it had already been processed.
+
+ Return true if everything is as expected. Return false otherwise. */
+
+static bool
+process_use (tree stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
+ enum vect_relevant relevant, VEC(tree,heap) **worklist)
+{
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ stmt_vec_info dstmt_vinfo;
+ basic_block def_bb;
+ tree def, def_stmt;
+ enum vect_def_type dt;
+
+ /* case 1: we are only interested in uses that need to be vectorized. Uses
+ that are used for address computation are not considered relevant. */
+ if (!exist_non_indexing_operands_for_use_p (use, stmt))
+ return true;
+
+ if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
+ return false;
+ }
+
+ if (!def_stmt || IS_EMPTY_STMT (def_stmt))
+ return true;
+
+ def_bb = bb_for_stmt (def_stmt);
+ if (!flow_bb_inside_loop_p (loop, def_bb))
+ return true;
+
+ /* case 2: A reduction phi defining a reduction stmt (DEF_STMT). DEF_STMT
+ must have already been processed, so we just check that everything is as
+ expected, and we are done. */
+ dstmt_vinfo = vinfo_for_stmt (def_stmt);
+ if (TREE_CODE (stmt) == PHI_NODE
+ && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
+ && TREE_CODE (def_stmt) != PHI_NODE
+ && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def)
+ {
+ if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
+ dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
+ gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
+ gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
+ || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_loop);
+ return true;
+ }
+
+ vect_mark_relevant (worklist, def_stmt, relevant, live_p);
+ return true;
+}
+
+
/* Function vect_mark_stmts_to_be_vectorized.
Not all stmts in the loop need to be vectorized. For example:
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
unsigned int nbbs = loop->num_nodes;
block_stmt_iterator si;
- tree stmt, use;
+ tree stmt;
stmt_ann_t ann;
- ssa_op_iter iter;
unsigned int i;
stmt_vec_info stmt_vinfo;
basic_block bb;
tree phi;
bool live_p;
enum vect_relevant relevant;
- tree def, def_stmt;
- enum vect_def_type dt;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
}
}
-
/* 2. Process_worklist */
-
while (VEC_length (tree, worklist) > 0)
{
- stmt = VEC_pop (tree, worklist);
+ use_operand_p use_p;
+ ssa_op_iter iter;
+ stmt = VEC_pop (tree, worklist);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "worklist: examine stmt: ");
print_generic_expr (vect_dump, stmt, TDF_SLIM);
}
- /* Examine the USEs of STMT. For each ssa-name USE that is defined
- in the loop, mark the stmt that defines it (DEF_STMT) as
- relevant/irrelevant and live/dead according to the liveness and
- relevance properties of STMT.
- */
-
+ /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
+ (DEF_STMT) as relevant/irrelevant and live/dead according to the
+ liveness and relevance properties of STMT. */
ann = stmt_ann (stmt);
stmt_vinfo = vinfo_for_stmt (stmt);
-
relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
/* Generally, the liveness and relevance properties of STMT are
- propagated to the DEF_STMTs of its USEs:
- STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
- STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
-
- Exceptions:
-
- (case 1)
- If USE is used only for address computations (e.g. array indexing),
- which does not need to be directly vectorized, then the
- liveness/relevance of the respective DEF_STMT is left unchanged.
-
- (case 2)
- If STMT has been identified as defining a reduction variable, then
- we want to set liveness/relevance as follows:
- STMT_VINFO_LIVE_P (DEF_STMT_info) <-- false
- STMT_VINFO_RELEVANT (DEF_STMT_info) <-- vect_used_by_reduction
- because even though STMT is classified as live (since it defines a
- value that is used across loop iterations) and irrelevant (since it
- is not used inside the loop), it will be vectorized, and therefore
- the corresponding DEF_STMTs need to marked as relevant.
- We distinguish between two kinds of relevant stmts - those that are
- used by a reduction computation, and those that are (also) used by
- a regular computation. This allows us later on to identify stmts
- that are used solely by a reduction, and therefore the order of
- the results that they produce does not have to be kept.
- */
-
- /* case 2.2: */
+ propagated as is to the DEF_STMTs of its USEs:
+ live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
+ relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
+
+ One exception is when STMT has been identified as defining a reduction
+ variable; in this case we set the liveness/relevance as follows:
+ live_p = false
+ relevant = vect_used_by_reduction
+ This is because we distinguish between two kinds of relevant stmts -
+ those that are used by a reduction computation, and those that are
+ (also) used by a regular computation. This allows us later on to
+ identify stmts that are used solely by a reduction, and therefore the
+ order of the results that they produce does not have to be kept.
+
+ Reduction phis are expected to be used by a reduction stmt; Other
+ reduction stmts are expected to be unused in the loop. These are the
+ expected values of "relevant" for reduction phis/stmts in the loop:
+
+ relevance: phi stmt
+ vect_unused_in_loop ok
+ vect_used_by_reduction ok
+ vect_used_in_loop */
+
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
- {
+ {
+ switch (relevant)
+ {
+ case vect_unused_in_loop:
+ gcc_assert (TREE_CODE (stmt) != PHI_NODE);
+ break;
+ case vect_used_by_reduction:
+ if (TREE_CODE (stmt) == PHI_NODE)
+ break;
+ case vect_used_in_loop:
+ default:
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "unsupported use of reduction.");
+ VEC_free (tree, heap, worklist);
+ return false;
+ }
relevant = vect_used_by_reduction;
- live_p = false;
+ live_p = false;
}
- i = 0;
- FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
+ FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ tree op = USE_FROM_PTR (use_p);
+ if (!process_use (stmt, op, loop_vinfo, live_p, relevant, &worklist))
{
- fprintf (vect_dump, "worklist: examine use %d: ", i++);
- print_generic_expr (vect_dump, use, TDF_SLIM);
- }
-
- /* case 1: we are only interested in uses that need to be vectorized.
- Uses that are used for address computation are not considered
- relevant.
- */
- if (!exist_non_indexing_operands_for_use_p (use, stmt))
- continue;
-
- if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
VEC_free (tree, heap, worklist);
return false;
- }
-
- if (!def_stmt || IS_EMPTY_STMT (def_stmt))
- continue;
-
- bb = bb_for_stmt (def_stmt);
- if (!flow_bb_inside_loop_p (loop, bb))
- continue;
- vect_mark_relevant (&worklist, def_stmt, relevant, live_p);
+ }
}
- } /* while worklist */
+ } /* while worklist */
VEC_free (tree, heap, worklist);
return true;
return false;
}
- loop_vinfo = new_loop_vec_info (loop);
- LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
-
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
+ if (!NITERS_KNOWN_P (number_of_iterations))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
print_generic_expr (vect_dump, number_of_iterations, TDF_DETAILS);
}
}
- else
- if (LOOP_VINFO_INT_NITERS (loop_vinfo) == 0)
+ else if (TREE_INT_CST_LOW (number_of_iterations) == 0)
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
fprintf (vect_dump, "not vectorized: number of iterations = 0.");
return NULL;
}
+ loop_vinfo = new_loop_vec_info (loop);
+ LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
LOOP_VINFO_EXIT_COND (loop_vinfo) = loop_cond;
+ gcc_assert (!loop->aux);
+ loop->aux = loop_vinfo;
return loop_vinfo;
}
OSDN Git Service
