2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #ifndef GCC_TREE_VECTORIZER_H
23 #define GCC_TREE_VECTORIZER_H
25 #include "tree-data-ref.h"
27 typedef source_location LOC;
28 #define UNKNOWN_LOC UNKNOWN_LOCATION
29 #define EXPR_LOC(e) EXPR_LOCATION(e)
30 #define LOC_FILE(l) LOCATION_FILE (l)
31 #define LOC_LINE(l) LOCATION_LINE (l)
33 /* Used for naming of new temporaries. */
40 /* Defines type of operation. */
47 /* Define type of available alignment support. */
48 enum dr_alignment_support {
49 dr_unaligned_unsupported,
50 dr_unaligned_supported,
52 dr_explicit_realign_optimized,
56 /* Define type of def-use cross-iteration cycle. */
58 vect_uninitialized_def = 0,
59 vect_constant_def = 1,
64 vect_double_reduction_def,
69 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
70 || ((D) == vect_double_reduction_def) \
71 || ((D) == vect_nested_cycle))
73 /************************************************************************
75 ************************************************************************/
77 /* A computation tree of an SLP instance. Each node corresponds to a group of
78 stmts to be packed in a SIMD stmt. */
79 typedef struct _slp_tree {
80 /* Only binary and unary operations are supported. LEFT child corresponds to
81 the first operand and RIGHT child to the second if the operation is
83 struct _slp_tree *left;
84 struct _slp_tree *right;
85 /* A group of scalar stmts to be vectorized together. */
86 VEC (gimple, heap) *stmts;
87 /* Vectorized stmt/s. */
88 VEC (gimple, heap) *vec_stmts;
89 /* Number of vector stmts that are created to replace the group of scalar
90 stmts. It is calculated during the transformation phase as the number of
91 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
92 divided by vector size. */
93 unsigned int vec_stmts_size;
94 /* Vectorization costs associated with SLP node. */
97 int outside_of_loop; /* Statements generated outside loop. */
98 int inside_of_loop; /* Statements generated inside loop. */
103 DEF_VEC_ALLOC_P(slp_tree, heap);
105 /* SLP instance is a sequence of stmts in a loop that can be packed into
107 typedef struct _slp_instance {
108 /* The root of SLP tree. */
111 /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
112 unsigned int group_size;
114 /* The unrolling factor required to vectorized this SLP instance. */
115 unsigned int unrolling_factor;
117 /* Vectorization costs associated with SLP instance. */
120 int outside_of_loop; /* Statements generated outside loop. */
121 int inside_of_loop; /* Statements generated inside loop. */
124 /* Loads permutation relatively to the stores, NULL if there is no
126 VEC (int, heap) *load_permutation;
128 /* The group of nodes that contain loads of this SLP instance. */
129 VEC (slp_tree, heap) *loads;
131 /* The first scalar load of the instance. The created vector loads will be
132 inserted before this statement. */
136 DEF_VEC_P(slp_instance);
137 DEF_VEC_ALLOC_P(slp_instance, heap);
139 /* Access Functions. */
140 #define SLP_INSTANCE_TREE(S) (S)->root
141 #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
142 #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
143 #define SLP_INSTANCE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
144 #define SLP_INSTANCE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
145 #define SLP_INSTANCE_LOAD_PERMUTATION(S) (S)->load_permutation
146 #define SLP_INSTANCE_LOADS(S) (S)->loads
147 #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load
149 #define SLP_TREE_LEFT(S) (S)->left
150 #define SLP_TREE_RIGHT(S) (S)->right
151 #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
152 #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
153 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
154 #define SLP_TREE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
155 #define SLP_TREE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
158 typedef struct _vect_peel_info
161 struct data_reference *dr;
165 typedef struct _vect_peel_extended_info
167 struct _vect_peel_info peel_info;
168 unsigned int inside_cost;
169 unsigned int outside_cost;
170 } *vect_peel_extended_info;
172 /*-----------------------------------------------------------------*/
173 /* Info on vectorized loops. */
174 /*-----------------------------------------------------------------*/
175 typedef struct _loop_vec_info {
177 /* The loop to which this info struct refers to. */
180 /* The loop basic blocks. */
183 /* Number of iterations. */
185 tree num_iters_unchanged;
187 /* Minimum number of iterations below which vectorization is expected to
188 not be profitable (as estimated by the cost model).
189 -1 indicates that vectorization will not be profitable.
190 FORNOW: This field is an int. Will be a tree in the future, to represent
191 values unknown at compile time. */
192 int min_profitable_iters;
194 /* Is the loop vectorizable? */
197 /* Unrolling factor */
198 int vectorization_factor;
200 /* The loop location in the source. */
201 LOC loop_line_number;
203 /* Unknown DRs according to which loop was peeled. */
204 struct data_reference *unaligned_dr;
206 /* peeling_for_alignment indicates whether peeling for alignment will take
207 place, and what the peeling factor should be:
208 peeling_for_alignment = X means:
209 If X=0: Peeling for alignment will not be applied.
210 If X>0: Peel first X iterations.
211 If X=-1: Generate a runtime test to calculate the number of iterations
212 to be peeled, using the dataref recorded in the field
214 int peeling_for_alignment;
216 /* The mask used to check the alignment of pointers or arrays. */
219 /* The loop nest in which the data dependences are computed. */
220 VEC (loop_p, heap) *loop_nest;
222 /* All data references in the loop. */
223 VEC (data_reference_p, heap) *datarefs;
225 /* All data dependences in the loop. */
226 VEC (ddr_p, heap) *ddrs;
228 /* Data Dependence Relations defining address ranges that are candidates
229 for a run-time aliasing check. */
230 VEC (ddr_p, heap) *may_alias_ddrs;
232 /* Statements in the loop that have data references that are candidates for a
233 runtime (loop versioning) misalignment check. */
234 VEC(gimple,heap) *may_misalign_stmts;
236 /* All interleaving chains of stores in the loop, represented by the first
237 stmt in the chain. */
238 VEC(gimple, heap) *strided_stores;
240 /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES
242 VEC(slp_instance, heap) *slp_instances;
244 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
245 applied to the loop, i.e., no unrolling is needed, this is 1. */
246 unsigned slp_unrolling_factor;
248 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
249 VEC (gimple, heap) *reductions;
251 /* All reduction chains in the loop, represented by the first
252 stmt in the chain. */
253 VEC (gimple, heap) *reduction_chains;
255 /* Hash table used to choose the best peeling option. */
260 /* Access Functions. */
261 #define LOOP_VINFO_LOOP(L) (L)->loop
262 #define LOOP_VINFO_BBS(L) (L)->bbs
263 #define LOOP_VINFO_NITERS(L) (L)->num_iters
264 /* Since LOOP_VINFO_NITERS can change after prologue peeling
265 retain total unchanged scalar loop iterations for cost model. */
266 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
267 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
268 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
269 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
270 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
271 #define LOOP_VINFO_LOOP_NEST(L) (L)->loop_nest
272 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs
273 #define LOOP_VINFO_DDRS(L) (L)->ddrs
274 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
275 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
276 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
277 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
278 #define LOOP_VINFO_LOC(L) (L)->loop_line_number
279 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
280 #define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
281 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
282 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
283 #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
284 #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
285 #define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab
287 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
288 VEC_length (gimple, (L)->may_misalign_stmts) > 0
289 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
290 VEC_length (ddr_p, (L)->may_alias_ddrs) > 0
292 #define NITERS_KNOWN_P(n) \
293 (host_integerp ((n),0) \
294 && TREE_INT_CST_LOW ((n)) > 0)
296 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
297 NITERS_KNOWN_P((L)->num_iters)
299 static inline loop_vec_info
300 loop_vec_info_for_loop (struct loop *loop)
302 return (loop_vec_info) loop->aux;
306 nested_in_vect_loop_p (struct loop *loop, gimple stmt)
309 && (loop->inner == (gimple_bb (stmt))->loop_father));
312 typedef struct _bb_vec_info {
315 /* All interleaving chains of stores in the basic block, represented by the
316 first stmt in the chain. */
317 VEC(gimple, heap) *strided_stores;
319 /* All SLP instances in the basic block. This is a subset of the set of
320 STRIDED_STORES of the basic block. */
321 VEC(slp_instance, heap) *slp_instances;
323 /* All data references in the basic block. */
324 VEC (data_reference_p, heap) *datarefs;
326 /* All data dependences in the basic block. */
327 VEC (ddr_p, heap) *ddrs;
330 #define BB_VINFO_BB(B) (B)->bb
331 #define BB_VINFO_STRIDED_STORES(B) (B)->strided_stores
332 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
333 #define BB_VINFO_DATAREFS(B) (B)->datarefs
334 #define BB_VINFO_DDRS(B) (B)->ddrs
336 static inline bb_vec_info
337 vec_info_for_bb (basic_block bb)
339 return (bb_vec_info) bb->aux;
342 /*-----------------------------------------------------------------*/
343 /* Info on vectorized defs. */
344 /*-----------------------------------------------------------------*/
345 enum stmt_vec_info_type {
346 undef_vec_info_type = 0,
352 assignment_vec_info_type,
353 condition_vec_info_type,
356 type_promotion_vec_info_type,
357 type_demotion_vec_info_type,
358 type_conversion_vec_info_type,
359 loop_exit_ctrl_vec_info_type
362 /* Indicates whether/how a variable is used in the scope of loop/basic
365 vect_unused_in_scope = 0,
366 /* The def is in the inner loop, and the use is in the outer loop, and the
367 use is a reduction stmt. */
368 vect_used_in_outer_by_reduction,
369 /* The def is in the inner loop, and the use is in the outer loop (and is
370 not part of reduction). */
373 /* defs that feed computations that end up (only) in a reduction. These
374 defs may be used by non-reduction stmts, but eventually, any
375 computations/values that are affected by these defs are used to compute
376 a reduction (i.e. don't get stored to memory, for example). We use this
377 to identify computations that we can change the order in which they are
379 vect_used_by_reduction,
384 /* The type of vectorization that can be applied to the stmt: regular loop-based
385 vectorization; pure SLP - the stmt is a part of SLP instances and does not
386 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
387 a part of SLP instance and also must be loop-based vectorized, since it has
388 uses outside SLP sequences.
390 In the loop context the meanings of pure and hybrid SLP are slightly
391 different. By saying that pure SLP is applied to the loop, we mean that we
392 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
393 vectorized without doing any conceptual unrolling, cause we don't pack
394 together stmts from different iterations, only within a single iteration.
395 Loop hybrid SLP means that we exploit both intra-iteration and
396 inter-iteration parallelism (e.g., number of elements in the vector is 4
397 and the slp-group-size is 2, in which case we don't have enough parallelism
398 within an iteration, so we obtain the rest of the parallelism from subsequent
399 iterations by unrolling the loop by 2). */
407 typedef struct data_reference *dr_p;
409 DEF_VEC_ALLOC_P(dr_p,heap);
411 typedef struct _stmt_vec_info {
413 enum stmt_vec_info_type type;
415 /* Indicates whether this stmts is part of a computation whose result is
416 used outside the loop. */
419 /* Stmt is part of some pattern (computation idiom) */
422 /* For loads only, if there is a store with the same location, this field is
426 /* The stmt to which this info struct refers to. */
429 /* The loop_vec_info with respect to which STMT is vectorized. */
430 loop_vec_info loop_vinfo;
432 /* The vector type to be used for the LHS of this statement. */
435 /* The vectorized version of the stmt. */
436 gimple vectorized_stmt;
439 /** The following is relevant only for stmts that contain a non-scalar
440 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
441 at most one such data-ref. **/
443 /* Information about the data-ref (access function, etc),
444 relative to the inner-most containing loop. */
445 struct data_reference *data_ref_info;
447 /* Information about the data-ref relative to this loop
448 nest (the loop that is being considered for vectorization). */
449 tree dr_base_address;
455 /* Used for various bookkeeping purposes, generally holding a pointer to
456 some other stmt S that is in some way "related" to this stmt.
457 Current use of this field is:
458 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
459 true): S is the "pattern stmt" that represents (and replaces) the
460 sequence of stmts that constitutes the pattern. Similarly, the
461 related_stmt of the "pattern stmt" points back to this stmt (which is
462 the last stmt in the original sequence of stmts that constitutes the
466 /* List of datarefs that are known to have the same alignment as the dataref
468 VEC(dr_p,heap) *same_align_refs;
470 /* Classify the def of this stmt. */
471 enum vect_def_type def_type;
473 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
474 enum slp_vect_type slp_type;
476 /* Interleaving and reduction chains info. */
477 /* First element in the group. */
478 gimple first_element;
479 /* Pointer to the next element in the group. */
481 /* For data-refs, in case that two or more stmts share data-ref, this is the
482 pointer to the previously detected stmt with the same dr. */
484 /* The size of the group. */
486 /* For stores, number of stores from this group seen. We vectorize the last
488 unsigned int store_count;
489 /* For loads only, the gap from the previous load. For consecutive loads, GAP
493 /* Not all stmts in the loop need to be vectorized. e.g, the increment
494 of the loop induction variable and computation of array indexes. relevant
495 indicates whether the stmt needs to be vectorized. */
496 enum vect_relevant relevant;
498 /* Vectorization costs associated with statement. */
501 int outside_of_loop; /* Statements generated outside loop. */
502 int inside_of_loop; /* Statements generated inside loop. */
505 /* The bb_vec_info with respect to which STMT is vectorized. */
506 bb_vec_info bb_vinfo;
508 /* Is this statement vectorizable or should it be skipped in (partial)
513 /* Access Functions. */
514 #define STMT_VINFO_TYPE(S) (S)->type
515 #define STMT_VINFO_STMT(S) (S)->stmt
516 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo
517 #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo
518 #define STMT_VINFO_RELEVANT(S) (S)->relevant
519 #define STMT_VINFO_LIVE_P(S) (S)->live
520 #define STMT_VINFO_VECTYPE(S) (S)->vectype
521 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
522 #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
523 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info
525 #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address
526 #define STMT_VINFO_DR_INIT(S) (S)->dr_init
527 #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset
528 #define STMT_VINFO_DR_STEP(S) (S)->dr_step
529 #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to
531 #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
532 #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
533 #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
534 #define STMT_VINFO_DEF_TYPE(S) (S)->def_type
535 #define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element
536 #define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element
537 #define STMT_VINFO_GROUP_SIZE(S) (S)->size
538 #define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count
539 #define STMT_VINFO_GROUP_GAP(S) (S)->gap
540 #define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
541 #define STMT_VINFO_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
542 #define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info)
544 #define GROUP_FIRST_ELEMENT(S) (S)->first_element
545 #define GROUP_NEXT_ELEMENT(S) (S)->next_element
546 #define GROUP_SIZE(S) (S)->size
547 #define GROUP_STORE_COUNT(S) (S)->store_count
548 #define GROUP_GAP(S) (S)->gap
549 #define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
550 #define GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
552 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
553 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
554 #define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
556 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
557 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
558 #define STMT_SLP_TYPE(S) (S)->slp_type
560 #define VECT_MAX_COST 1000
562 /* The maximum number of intermediate steps required in multi-step type
564 #define MAX_INTERM_CVT_STEPS 3
566 /* The maximum vectorization factor supported by any target (V32QI). */
567 #define MAX_VECTORIZATION_FACTOR 32
569 /* Avoid GTY(()) on stmt_vec_info. */
570 typedef void *vec_void_p;
571 DEF_VEC_P (vec_void_p);
572 DEF_VEC_ALLOC_P (vec_void_p, heap);
574 extern VEC(vec_void_p,heap) *stmt_vec_info_vec;
576 void init_stmt_vec_info_vec (void);
577 void free_stmt_vec_info_vec (void);
579 /* Return a stmt_vec_info corresponding to STMT. */
581 static inline stmt_vec_info
582 vinfo_for_stmt (gimple stmt)
584 unsigned int uid = gimple_uid (stmt);
588 return (stmt_vec_info) VEC_index (vec_void_p, stmt_vec_info_vec, uid - 1);
591 /* Set vectorizer information INFO for STMT. */
594 set_vinfo_for_stmt (gimple stmt, stmt_vec_info info)
596 unsigned int uid = gimple_uid (stmt);
599 gcc_checking_assert (info);
600 uid = VEC_length (vec_void_p, stmt_vec_info_vec) + 1;
601 gimple_set_uid (stmt, uid);
602 VEC_safe_push (vec_void_p, heap, stmt_vec_info_vec, (vec_void_p) info);
605 VEC_replace (vec_void_p, stmt_vec_info_vec, uid - 1, (vec_void_p) info);
608 /* Return the earlier statement between STMT1 and STMT2. */
611 get_earlier_stmt (gimple stmt1, gimple stmt2)
613 unsigned int uid1, uid2;
621 uid1 = gimple_uid (stmt1);
622 uid2 = gimple_uid (stmt2);
624 if (uid1 == 0 || uid2 == 0)
627 gcc_checking_assert (uid1 <= VEC_length (vec_void_p, stmt_vec_info_vec)
628 && uid2 <= VEC_length (vec_void_p, stmt_vec_info_vec));
636 /* Return the later statement between STMT1 and STMT2. */
639 get_later_stmt (gimple stmt1, gimple stmt2)
641 unsigned int uid1, uid2;
649 uid1 = gimple_uid (stmt1);
650 uid2 = gimple_uid (stmt2);
652 if (uid1 == 0 || uid2 == 0)
655 gcc_assert (uid1 <= VEC_length (vec_void_p, stmt_vec_info_vec));
656 gcc_assert (uid2 <= VEC_length (vec_void_p, stmt_vec_info_vec));
664 /* Return TRUE if a statement represented by STMT_INFO is a part of a
668 is_pattern_stmt_p (stmt_vec_info stmt_info)
671 stmt_vec_info related_stmt_info;
673 related_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
675 && (related_stmt_info = vinfo_for_stmt (related_stmt))
676 && STMT_VINFO_IN_PATTERN_P (related_stmt_info))
682 /* Return true if BB is a loop header. */
685 is_loop_header_bb_p (basic_block bb)
687 if (bb == (bb->loop_father)->header)
689 gcc_checking_assert (EDGE_COUNT (bb->preds) == 1);
693 /* Set inside loop vectorization cost. */
696 stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
700 SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost;
702 STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost;
705 /* Set inside loop vectorization cost. */
708 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
712 SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost;
714 STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost;
717 /* Return pow2 (X). */
724 for (i = 0; i < x; i++)
730 /*-----------------------------------------------------------------*/
731 /* Info on data references alignment. */
732 /*-----------------------------------------------------------------*/
734 /* Reflects actual alignment of first access in the vectorized loop,
735 taking into account peeling/versioning if applied. */
736 #define DR_MISALIGNMENT(DR) ((int) (size_t) (DR)->aux)
737 #define SET_DR_MISALIGNMENT(DR, VAL) ((DR)->aux = (void *) (size_t) (VAL))
739 /* Return TRUE if the data access is aligned, and FALSE otherwise. */
742 aligned_access_p (struct data_reference *data_ref_info)
744 return (DR_MISALIGNMENT (data_ref_info) == 0);
747 /* Return TRUE if the alignment of the data access is known, and FALSE
751 known_alignment_for_access_p (struct data_reference *data_ref_info)
753 return (DR_MISALIGNMENT (data_ref_info) != -1);
756 /* vect_dump will be set to stderr or dump_file if exist. */
757 extern FILE *vect_dump;
758 extern LOC vect_loop_location;
760 /*-----------------------------------------------------------------*/
761 /* Function prototypes. */
762 /*-----------------------------------------------------------------*/
764 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
765 in tree-vect-loop-manip.c. */
766 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
767 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
768 extern void vect_loop_versioning (loop_vec_info, bool, tree *, gimple_seq *);
769 extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
771 extern void vect_do_peeling_for_alignment (loop_vec_info);
772 extern LOC find_loop_location (struct loop *);
773 extern bool vect_can_advance_ivs_p (loop_vec_info);
775 /* In tree-vect-stmts.c. */
776 extern unsigned int current_vector_size;
777 extern tree get_vectype_for_scalar_type (tree);
778 extern tree get_same_sized_vectype (tree, tree);
779 extern bool vect_is_simple_use (tree, loop_vec_info, bb_vec_info, gimple *,
780 tree *, enum vect_def_type *);
781 extern bool vect_is_simple_use_1 (tree, loop_vec_info, bb_vec_info, gimple *,
782 tree *, enum vect_def_type *, tree *);
783 extern bool supportable_widening_operation (enum tree_code, gimple, tree, tree,
784 tree *, tree *, enum tree_code *,
785 enum tree_code *, int *,
786 VEC (tree, heap) **);
787 extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
789 int *, VEC (tree, heap) **);
790 extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info,
792 extern void free_stmt_vec_info (gimple stmt);
793 extern tree vectorizable_function (gimple, tree, tree);
794 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
796 extern void vect_model_store_cost (stmt_vec_info, int, bool,
797 enum vect_def_type, slp_tree);
798 extern void vect_model_load_cost (stmt_vec_info, int, bool, slp_tree);
799 extern void vect_finish_stmt_generation (gimple, gimple,
800 gimple_stmt_iterator *);
801 extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
802 extern int cost_for_stmt (gimple);
803 extern tree vect_get_vec_def_for_operand (tree, gimple, tree *);
804 extern tree vect_init_vector (gimple, tree, tree,
805 gimple_stmt_iterator *);
806 extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree);
807 extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *,
808 bool *, slp_tree, slp_instance);
809 extern void vect_remove_stores (gimple);
810 extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
811 extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
813 extern void vect_get_load_cost (struct data_reference *, int, bool,
814 unsigned int *, unsigned int *);
815 extern void vect_get_store_cost (struct data_reference *, int, unsigned int *);
817 /* In tree-vect-data-refs.c. */
818 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
819 extern enum dr_alignment_support vect_supportable_dr_alignment
820 (struct data_reference *, bool);
821 extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *,
823 extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info,
825 extern bool vect_enhance_data_refs_alignment (loop_vec_info);
826 extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info);
827 extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info);
828 extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info);
829 extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
830 extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *);
831 extern tree vect_create_data_ref_ptr (gimple, tree, struct loop *, tree,
832 tree *, gimple_stmt_iterator *,
833 gimple *, bool, bool *);
834 extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);
835 extern tree vect_create_destination_var (tree, tree);
836 extern bool vect_strided_store_supported (tree, unsigned HOST_WIDE_INT);
837 extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT);
838 extern bool vect_strided_load_supported (tree, unsigned HOST_WIDE_INT);
839 extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT);
840 extern void vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
841 gimple_stmt_iterator *, VEC(tree,heap) **);
842 extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *,
843 enum dr_alignment_support, tree,
845 extern void vect_transform_strided_load (gimple, VEC(tree,heap) *, int,
846 gimple_stmt_iterator *);
847 extern void vect_record_strided_load_vectors (gimple, VEC(tree,heap) *);
848 extern int vect_get_place_in_interleaving_chain (gimple, gimple);
849 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
850 extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *,
851 tree, struct loop *);
853 /* In tree-vect-loop.c. */
854 /* FORNOW: Used in tree-parloops.c. */
855 extern void destroy_loop_vec_info (loop_vec_info, bool);
856 extern gimple vect_force_simple_reduction (loop_vec_info, gimple, bool, bool *);
857 /* Drive for loop analysis stage. */
858 extern loop_vec_info vect_analyze_loop (struct loop *);
859 /* Drive for loop transformation stage. */
860 extern void vect_transform_loop (loop_vec_info);
861 extern loop_vec_info vect_analyze_loop_form (struct loop *);
862 extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *,
864 extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *,
866 extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *);
867 extern int vect_estimate_min_profitable_iters (loop_vec_info);
868 extern tree get_initial_def_for_reduction (gimple, tree, tree *);
869 extern int vect_min_worthwhile_factor (enum tree_code);
870 extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, int);
871 extern int vect_get_single_scalar_iteraion_cost (loop_vec_info);
873 /* In tree-vect-slp.c. */
874 extern void vect_free_slp_instance (slp_instance);
875 extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *,
876 gimple_stmt_iterator *, int,
878 extern bool vect_schedule_slp (loop_vec_info, bb_vec_info);
879 extern void vect_update_slp_costs_according_to_vf (loop_vec_info);
880 extern bool vect_analyze_slp (loop_vec_info, bb_vec_info);
881 extern bool vect_make_slp_decision (loop_vec_info);
882 extern void vect_detect_hybrid_slp (loop_vec_info);
883 extern void vect_get_slp_defs (tree, tree, slp_tree, VEC (tree,heap) **,
884 VEC (tree,heap) **, int);
885 extern LOC find_bb_location (basic_block);
886 extern bb_vec_info vect_slp_analyze_bb (basic_block);
887 extern void vect_slp_transform_bb (basic_block);
889 /* In tree-vect-patterns.c. */
890 /* Pattern recognition functions.
891 Additional pattern recognition functions can (and will) be added
893 typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *);
894 #define NUM_PATTERNS 4
895 void vect_pattern_recog (loop_vec_info);
897 /* In tree-vectorizer.c. */
898 unsigned vectorize_loops (void);
899 /* Vectorization debug information */
900 extern bool vect_print_dump_info (enum vect_verbosity_levels);
902 #endif /* GCC_TREE_VECTORIZER_H */