/* Reduction cycles detected in the loop. Used in loop-aware SLP. */
VEC (gimple, heap) *reductions;
+ /* All reduction chains in the loop, represented by the first
+ stmt in the chain. */
+ VEC (gimple, heap) *reduction_chains;
+
/* Hash table used to choose the best peeling option. */
htab_t peeling_htab;
+ /* When we have strided data accesses with gaps, we may introduce invalid
+ memory accesses. We peel the last iteration of the loop to prevent
+ this. */
+ bool peeling_for_gaps;
+
} *loop_vec_info;
/* Access Functions. */
#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
+#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
#define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab
+#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
VEC_length (gimple, (L)->may_misalign_stmts) > 0
pattern). */
gimple related_stmt;
+ /* Used to keep a def stmt of a pattern stmt if such exists. */
+ gimple pattern_def_stmt;
+
/* List of datarefs that are known to have the same alignment as the dataref
of this stmt. */
VEC(dr_p,heap) *same_align_refs;
/* Whether the stmt is SLPed, loop-based vectorized, or both. */
enum slp_vect_type slp_type;
- /* Interleaving info. */
- /* First data-ref in the interleaving group. */
- gimple first_dr;
- /* Pointer to the next data-ref in the group. */
- gimple next_dr;
- /* In case that two or more stmts share data-ref, this is the pointer to the
- previously detected stmt with the same dr. */
+ /* Interleaving and reduction chains info. */
+ /* First element in the group. */
+ gimple first_element;
+ /* Pointer to the next element in the group. */
+ gimple next_element;
+ /* For data-refs, in case that two or more stmts share data-ref, this is the
+ pointer to the previously detected stmt with the same dr. */
gimple same_dr_stmt;
- /* The size of the interleaving group. */
+ /* The size of the group. */
unsigned int size;
/* For stores, number of stores from this group seen. We vectorize the last
one. */
#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
+#define STMT_VINFO_PATTERN_DEF_STMT(S) (S)->pattern_def_stmt
#define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
-#define STMT_VINFO_DR_GROUP_FIRST_DR(S) (S)->first_dr
-#define STMT_VINFO_DR_GROUP_NEXT_DR(S) (S)->next_dr
-#define STMT_VINFO_DR_GROUP_SIZE(S) (S)->size
-#define STMT_VINFO_DR_GROUP_STORE_COUNT(S) (S)->store_count
-#define STMT_VINFO_DR_GROUP_GAP(S) (S)->gap
-#define STMT_VINFO_DR_GROUP_SAME_DR_STMT(S)(S)->same_dr_stmt
-#define STMT_VINFO_DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
-#define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_dr != NULL)
-
-#define DR_GROUP_FIRST_DR(S) (S)->first_dr
-#define DR_GROUP_NEXT_DR(S) (S)->next_dr
-#define DR_GROUP_SIZE(S) (S)->size
-#define DR_GROUP_STORE_COUNT(S) (S)->store_count
-#define DR_GROUP_GAP(S) (S)->gap
-#define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
-#define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
+#define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element
+#define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element
+#define STMT_VINFO_GROUP_SIZE(S) (S)->size
+#define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count
+#define STMT_VINFO_GROUP_GAP(S) (S)->gap
+#define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
+#define STMT_VINFO_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
+#define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info)
+
+#define GROUP_FIRST_ELEMENT(S) (S)->first_element
+#define GROUP_NEXT_ELEMENT(S) (S)->next_element
+#define GROUP_SIZE(S) (S)->size
+#define GROUP_STORE_COUNT(S) (S)->store_count
+#define GROUP_GAP(S) (S)->gap
+#define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
+#define GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
#define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
extern tree vectorizable_function (gimple, tree, tree);
extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
slp_tree);
-extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type,
- slp_tree);
-extern void vect_model_load_cost (stmt_vec_info, int, slp_tree);
+extern void vect_model_store_cost (stmt_vec_info, int, bool,
+ enum vect_def_type, slp_tree);
+extern void vect_model_load_cost (stmt_vec_info, int, bool, slp_tree);
extern void vect_finish_stmt_generation (gimple, gimple,
gimple_stmt_iterator *);
extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
extern void vect_get_load_cost (struct data_reference *, int, bool,
unsigned int *, unsigned int *);
extern void vect_get_store_cost (struct data_reference *, int, unsigned int *);
+extern bool vect_supportable_shift (enum tree_code, tree);
/* In tree-vect-data-refs.c. */
extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info);
extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *);
-extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *,
- gimple *, bool, bool *);
+extern tree vect_create_data_ref_ptr (gimple, tree, struct loop *, tree,
+ tree *, gimple_stmt_iterator *,
+ gimple *, bool, bool *);
extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);
extern tree vect_create_destination_var (tree, tree);
-extern bool vect_strided_store_supported (tree);
-extern bool vect_strided_load_supported (tree);
-extern bool vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
+extern bool vect_strided_store_supported (tree, unsigned HOST_WIDE_INT);
+extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT);
+extern bool vect_strided_load_supported (tree, unsigned HOST_WIDE_INT);
+extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT);
+extern void vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
gimple_stmt_iterator *, VEC(tree,heap) **);
extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *,
enum dr_alignment_support, tree,
struct loop **);
-extern bool vect_permute_load_chain (VEC(tree,heap) *,unsigned int, gimple,
- gimple_stmt_iterator *, VEC(tree,heap) **);
-extern bool vect_transform_strided_load (gimple, VEC(tree,heap) *, int,
+extern void vect_transform_strided_load (gimple, VEC(tree,heap) *, int,
gimple_stmt_iterator *);
+extern void vect_record_strided_load_vectors (gimple, VEC(tree,heap) *);
extern int vect_get_place_in_interleaving_chain (gimple, gimple);
extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *,
extern bool vect_schedule_slp (loop_vec_info, bb_vec_info);
extern void vect_update_slp_costs_according_to_vf (loop_vec_info);
extern bool vect_analyze_slp (loop_vec_info, bb_vec_info);
-extern void vect_make_slp_decision (loop_vec_info);
+extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_get_slp_defs (tree, tree, slp_tree, VEC (tree,heap) **,
VEC (tree,heap) **, int);
/* Pattern recognition functions.
Additional pattern recognition functions can (and will) be added
in the future. */
-typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *);
-#define NUM_PATTERNS 4
+typedef gimple (* vect_recog_func_ptr) (VEC (gimple, heap) **, tree *, tree *);
+#define NUM_PATTERNS 5
void vect_pattern_recog (loop_vec_info);
/* In tree-vectorizer.c. */
OSDN Git Service
