/* Vectorizer
- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free
- Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
vect_internal_def,
vect_induction_def,
vect_reduction_def,
+ vect_double_reduction_def,
+ vect_nested_cycle,
vect_unknown_def_type
};
+#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
+ || ((D) == vect_double_reduction_def) \
+ || ((D) == vect_nested_cycle))
+
/* Define verbosity levels. */
enum verbosity_levels {
REPORT_NONE,
VEC (gimple, heap) *stmts;
/* Vectorized stmt/s. */
VEC (gimple, heap) *vec_stmts;
- /* Number of vector stmts that are created to replace the group of scalar
- stmts. It is calculated during the transformation phase as the number of
- scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
+ /* Number of vector stmts that are created to replace the group of scalar
+ stmts. It is calculated during the transformation phase as the number of
+ scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
divided by vector size. */
unsigned int vec_stmts_size;
/* Vectorization costs associated with SLP node. */
unsigned int unrolling_factor;
/* Vectorization costs associated with SLP instance. */
- struct
+ struct
{
int outside_of_loop; /* Statements generated outside loop. */
int inside_of_loop; /* Statements generated inside loop. */
} cost;
- /* Loads permutation relatively to the stores, NULL if there is no
+ /* Loads permutation relatively to the stores, NULL if there is no
permutation. */
VEC (int, heap) *load_permutation;
tree num_iters_unchanged;
/* Minimum number of iterations below which vectorization is expected to
- not be profitable (as estimated by the cost model).
+ not be profitable (as estimated by the cost model).
-1 indicates that vectorization will not be profitable.
FORNOW: This field is an int. Will be a tree in the future, to represent
- values unknown at compile time. */
- int min_profitable_iters;
-
+ values unknown at compile time. */
+ int min_profitable_iters;
+
/* Is the loop vectorizable? */
bool vectorizable;
of the loop. */
VEC(slp_instance, heap) *slp_instances;
- /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
+ /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
applied to the loop, i.e., no unrolling is needed, this is 1. */
unsigned slp_unrolling_factor;
} *loop_vec_info;
/* Access Functions. */
-#define LOOP_VINFO_LOOP(L) (L)->loop
-#define LOOP_VINFO_BBS(L) (L)->bbs
-#define LOOP_VINFO_NITERS(L) (L)->num_iters
+#define LOOP_VINFO_LOOP(L) (L)->loop
+#define LOOP_VINFO_BBS(L) (L)->bbs
+#define LOOP_VINFO_NITERS(L) (L)->num_iters
/* Since LOOP_VINFO_NITERS can change after prologue peeling
retain total unchanged scalar loop iterations for cost model. */
-#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
-#define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
-#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
-#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
-#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
-#define LOOP_VINFO_DATAREFS(L) (L)->datarefs
-#define LOOP_VINFO_DDRS(L) (L)->ddrs
-#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
-#define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
-#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
-#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
-#define LOOP_VINFO_LOC(L) (L)->loop_line_number
-#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
-#define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
-#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
+#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
+#define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
+#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
+#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
+#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
+#define LOOP_VINFO_DATAREFS(L) (L)->datarefs
+#define LOOP_VINFO_DDRS(L) (L)->ddrs
+#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
+#define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
+#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
+#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
+#define LOOP_VINFO_LOC(L) (L)->loop_line_number
+#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
+#define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
+#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
+#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
+VEC_length (gimple, (L)->may_misalign_stmts) > 0
+#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
+VEC_length (ddr_p, (L)->may_alias_ddrs) > 0
+
#define NITERS_KNOWN_P(n) \
(host_integerp ((n),0) \
&& TREE_INT_CST_LOW ((n)) > 0)
-#define LOOP_VINFO_NITERS_KNOWN_P(L) \
+#define LOOP_VINFO_NITERS_KNOWN_P(L) \
NITERS_KNOWN_P((L)->num_iters)
static inline loop_vec_info
static inline bool
nested_in_vect_loop_p (struct loop *loop, gimple stmt)
{
- return (loop->inner
+ return (loop->inner
&& (loop->inner == (gimple_bb (stmt))->loop_father));
}
typedef struct _bb_vec_info {
basic_block bb;
- /* All interleaving chains of stores in the basic block, represented by the
+ /* All interleaving chains of stores in the basic block, represented by the
first stmt in the chain. */
VEC(gimple, heap) *strided_stores;
- /* All SLP instances in the basic block. This is a subset of the set of
+ /* All SLP instances in the basic block. This is a subset of the set of
STRIDED_STORES of the basic block. */
VEC(slp_instance, heap) *slp_instances;
loop_exit_ctrl_vec_info_type
};
-/* Indicates whether/how a variable is used in the scope of loop/basic
+/* Indicates whether/how a variable is used in the scope of loop/basic
block. */
enum vect_relevant {
vect_unused_in_scope = 0,
+ /* The def is in the inner loop, and the use is in the outer loop, and the
+ use is a reduction stmt. */
vect_used_in_outer_by_reduction,
+ /* The def is in the inner loop, and the use is in the outer loop (and is
+ not part of reduction). */
vect_used_in_outer,
/* defs that feed computations that end up (only) in a reduction. These
- defs may be used by non-reduction stmts, but eventually, any
- computations/values that are affected by these defs are used to compute
- a reduction (i.e. don't get stored to memory, for example). We use this
- to identify computations that we can change the order in which they are
+ defs may be used by non-reduction stmts, but eventually, any
+ computations/values that are affected by these defs are used to compute
+ a reduction (i.e. don't get stored to memory, for example). We use this
+ to identify computations that we can change the order in which they are
computed. */
vect_used_by_reduction,
- vect_used_in_scope
+ vect_used_in_scope
};
/* The type of vectorization that can be applied to the stmt: regular loop-based
vectorization; pure SLP - the stmt is a part of SLP instances and does not
have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
a part of SLP instance and also must be loop-based vectorized, since it has
- uses outside SLP sequences.
-
- In the loop context the meanings of pure and hybrid SLP are slightly
- different. By saying that pure SLP is applied to the loop, we mean that we
- exploit only intra-iteration parallelism in the loop; i.e., the loop can be
- vectorized without doing any conceptual unrolling, cause we don't pack
- together stmts from different iterations, only within a single iteration.
- Loop hybrid SLP means that we exploit both intra-iteration and
+ uses outside SLP sequences.
+
+ In the loop context the meanings of pure and hybrid SLP are slightly
+ different. By saying that pure SLP is applied to the loop, we mean that we
+ exploit only intra-iteration parallelism in the loop; i.e., the loop can be
+ vectorized without doing any conceptual unrolling, cause we don't pack
+ together stmts from different iterations, only within a single iteration.
+ Loop hybrid SLP means that we exploit both intra-iteration and
inter-iteration parallelism (e.g., number of elements in the vector is 4
- and the slp-group-size is 2, in which case we don't have enough parallelism
- within an iteration, so we obtain the rest of the parallelism from subsequent
+ and the slp-group-size is 2, in which case we don't have enough parallelism
+ within an iteration, so we obtain the rest of the parallelism from subsequent
iterations by unrolling the loop by 2). */
-enum slp_vect_type {
+enum slp_vect_type {
loop_vect = 0,
pure_slp,
hybrid
used outside the loop. */
bool live;
- /* The vector type to be used. */
+ /* The vector type to be used for the LHS of this statement. */
tree vectype;
/* The vectorized version of the stmt. */
/** The following is relevant only for stmts that contain a non-scalar
- data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
+ data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
at most one such data-ref. **/
/* Information about the data-ref (access function, etc),
/* Stmt is part of some pattern (computation idiom) */
bool in_pattern_p;
- /* Used for various bookkeeping purposes, generally holding a pointer to
- some other stmt S that is in some way "related" to this stmt.
+ /* Used for various bookkeeping purposes, generally holding a pointer to
+ some other stmt S that is in some way "related" to this stmt.
Current use of this field is:
- If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
- true): S is the "pattern stmt" that represents (and replaces) the
- sequence of stmts that constitutes the pattern. Similarly, the
- related_stmt of the "pattern stmt" points back to this stmt (which is
- the last stmt in the original sequence of stmts that constitutes the
+ If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
+ true): S is the "pattern stmt" that represents (and replaces) the
+ sequence of stmts that constitutes the pattern. Similarly, the
+ related_stmt of the "pattern stmt" points back to this stmt (which is
+ the last stmt in the original sequence of stmts that constitutes the
pattern). */
gimple related_stmt;
bool read_write_dep;
/* Vectorization costs associated with statement. */
- struct
+ struct
{
int outside_of_loop; /* Statements generated outside loop. */
int inside_of_loop; /* Statements generated inside loop. */
/* Whether the stmt is SLPed, loop-based vectorized, or both. */
enum slp_vect_type slp_type;
-
+
/* The bb_vec_info with respect to which STMT is vectorized. */
bb_vec_info bb_vinfo;
} *stmt_vec_info;
#endif
/* Cost of any vector operation, excluding load, store or vector to scalar
- operation. */
+ operation. */
#ifndef TARG_VEC_STMT_COST
#define TARG_VEC_STMT_COST 1
#endif
return false;
}
-static inline void
-stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
+static inline void
+stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
int cost)
{
if (slp_node)
SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost;
else
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost;
-}
+}
-static inline void
-stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
+static inline void
+stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
int cost)
{
if (slp_node)
SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost;
else
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost;
-}
+}
static inline int
vect_pow2 (int x)
extern FILE *vect_dump;
extern LOC vect_loop_location;
-/* Bitmap of virtual variables to be renamed. */
-extern bitmap vect_memsyms_to_rename;
-
/*-----------------------------------------------------------------*/
/* Function prototypes. */
/*-----------------------------------------------------------------*/
-/* Simple loop peeling and versioning utilities for vectorizer's purposes -
+/* Simple loop peeling and versioning utilities for vectorizer's purposes -
in tree-vect-loop-manip.c. */
extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
/* In tree-vect-stmts.c. */
extern tree get_vectype_for_scalar_type (tree);
+extern tree get_same_sized_vectype (tree, tree);
extern bool vect_is_simple_use (tree, loop_vec_info, bb_vec_info, gimple *,
tree *, enum vect_def_type *);
-extern bool supportable_widening_operation (enum tree_code, gimple, tree,
- tree *, tree *, enum tree_code *,
- enum tree_code *, int *,
+extern bool vect_is_simple_use_1 (tree, loop_vec_info, bb_vec_info, gimple *,
+ tree *, enum vect_def_type *, tree *);
+extern bool supportable_widening_operation (enum tree_code, gimple, tree, tree,
+ tree *, tree *, enum tree_code *,
+ enum tree_code *, int *,
VEC (tree, heap) **);
-extern bool supportable_narrowing_operation (enum tree_code, const_gimple,
- tree, enum tree_code *, int *,
- VEC (tree, heap) **);
-extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info,
+extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
+ enum tree_code *,
+ int *, VEC (tree, heap) **);
+extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info,
bb_vec_info);
extern void free_stmt_vec_info (gimple stmt);
extern tree vectorizable_function (gimple, tree, tree);
bool *, slp_tree, slp_instance);
extern void vect_remove_stores (gimple);
extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
+extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
+ tree, int);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info);
extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *,
- gimple *, bool, bool *);
+ gimple *, bool, bool *);
extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);
extern tree vect_create_destination_var (tree, tree);
extern bool vect_strided_store_supported (tree);
extern bool vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
gimple_stmt_iterator *, VEC(tree,heap) **);
extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *,
- enum dr_alignment_support, tree,
+ enum dr_alignment_support, tree,
struct loop **);
extern bool vect_permute_load_chain (VEC(tree,heap) *,unsigned int, gimple,
gimple_stmt_iterator *, VEC(tree,heap) **);
/* In tree-vect-loop.c. */
/* FORNOW: Used in tree-parloops.c. */
extern void destroy_loop_vec_info (loop_vec_info, bool);
-extern gimple vect_is_simple_reduction (loop_vec_info, gimple);
+extern gimple vect_is_simple_reduction (loop_vec_info, gimple, bool, bool *);
/* Drive for loop analysis stage. */
extern loop_vec_info vect_analyze_loop (struct loop *);
/* Drive for loop transformation stage. */
/* In tree-vect-slp.c. */
extern void vect_free_slp_instance (slp_instance);
extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *,
- gimple_stmt_iterator *, int,
+ gimple_stmt_iterator *, int,
slp_instance, bool);
extern bool vect_schedule_slp (loop_vec_info, bb_vec_info);
extern void vect_update_slp_costs_according_to_vf (loop_vec_info);
#define NUM_PATTERNS 4
void vect_pattern_recog (loop_vec_info);
-/* Vectorization debug information - in tree-vectorizer.c. */
+/* In tree-vectorizer.c. */
+unsigned vectorize_loops (void);
+/* Vectorization debug information */
extern bool vect_print_dump_info (enum verbosity_levels);
-extern void vect_set_verbosity_level (const char *);
#endif /* GCC_TREE_VECTORIZER_H */
OSDN Git Service
