1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "diagnostic.h"
32 #include "tree-flow.h"
33 #include "tree-dump.h"
35 #include "cfglayout.h"
40 #include "tree-vectorizer.h"
41 #include "langhooks.h"
44 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
46 /* Function vect_mark_relevant.
48 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
51 vect_mark_relevant (VEC(gimple,heap) **worklist, gimple stmt,
52 enum vect_relevant relevant, bool live_p)
54 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
55 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
56 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
58 if (vect_print_dump_info (REPORT_DETAILS))
59 fprintf (vect_dump, "mark relevant %d, live %d.", relevant, live_p);
61 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
65 /* This is the last stmt in a sequence that was detected as a
66 pattern that can potentially be vectorized. Don't mark the stmt
67 as relevant/live because it's not going to be vectorized.
68 Instead mark the pattern-stmt that replaces it. */
70 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
72 if (vect_print_dump_info (REPORT_DETAILS))
73 fprintf (vect_dump, "last stmt in pattern. don't mark relevant/live.");
74 stmt_info = vinfo_for_stmt (pattern_stmt);
75 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
76 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
77 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
81 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
82 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
83 STMT_VINFO_RELEVANT (stmt_info) = relevant;
85 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
86 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
88 if (vect_print_dump_info (REPORT_DETAILS))
89 fprintf (vect_dump, "already marked relevant/live.");
93 VEC_safe_push (gimple, heap, *worklist, stmt);
97 /* Function vect_stmt_relevant_p.
99 Return true if STMT in loop that is represented by LOOP_VINFO is
100 "relevant for vectorization".
102 A stmt is considered "relevant for vectorization" if:
103 - it has uses outside the loop.
104 - it has vdefs (it alters memory).
105 - control stmts in the loop (except for the exit condition).
107 CHECKME: what other side effects would the vectorizer allow? */
110 vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo,
111 enum vect_relevant *relevant, bool *live_p)
113 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
115 imm_use_iterator imm_iter;
119 *relevant = vect_unused_in_scope;
122 /* cond stmt other than loop exit cond. */
123 if (is_ctrl_stmt (stmt)
124 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
125 != loop_exit_ctrl_vec_info_type)
126 *relevant = vect_used_in_scope;
128 /* changing memory. */
129 if (gimple_code (stmt) != GIMPLE_PHI)
130 if (gimple_vdef (stmt))
132 if (vect_print_dump_info (REPORT_DETAILS))
133 fprintf (vect_dump, "vec_stmt_relevant_p: stmt has vdefs.");
134 *relevant = vect_used_in_scope;
137 /* uses outside the loop. */
138 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
140 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
142 basic_block bb = gimple_bb (USE_STMT (use_p));
143 if (!flow_bb_inside_loop_p (loop, bb))
145 if (vect_print_dump_info (REPORT_DETAILS))
146 fprintf (vect_dump, "vec_stmt_relevant_p: used out of loop.");
148 if (is_gimple_debug (USE_STMT (use_p)))
151 /* We expect all such uses to be in the loop exit phis
152 (because of loop closed form) */
153 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
154 gcc_assert (bb == single_exit (loop)->dest);
161 return (*live_p || *relevant);
165 /* Function exist_non_indexing_operands_for_use_p
167 USE is one of the uses attached to STMT. Check if USE is
168 used in STMT for anything other than indexing an array. */
171 exist_non_indexing_operands_for_use_p (tree use, gimple stmt)
174 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
176 /* USE corresponds to some operand in STMT. If there is no data
177 reference in STMT, then any operand that corresponds to USE
178 is not indexing an array. */
179 if (!STMT_VINFO_DATA_REF (stmt_info))
182 /* STMT has a data_ref. FORNOW this means that its of one of
186 (This should have been verified in analyze_data_refs).
188 'var' in the second case corresponds to a def, not a use,
189 so USE cannot correspond to any operands that are not used
192 Therefore, all we need to check is if STMT falls into the
193 first case, and whether var corresponds to USE. */
195 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
198 if (!gimple_assign_copy_p (stmt))
200 operand = gimple_assign_rhs1 (stmt);
202 if (TREE_CODE (operand) != SSA_NAME)
213 Function process_use.
216 - a USE in STMT in a loop represented by LOOP_VINFO
217 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
218 that defined USE. This is done by calling mark_relevant and passing it
219 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
222 Generally, LIVE_P and RELEVANT are used to define the liveness and
223 relevance info of the DEF_STMT of this USE:
224 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
225 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
227 - case 1: If USE is used only for address computations (e.g. array indexing),
228 which does not need to be directly vectorized, then the liveness/relevance
229 of the respective DEF_STMT is left unchanged.
230 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
231 skip DEF_STMT cause it had already been processed.
232 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
233 be modified accordingly.
235 Return true if everything is as expected. Return false otherwise. */
238 process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
239 enum vect_relevant relevant, VEC(gimple,heap) **worklist)
241 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
242 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
243 stmt_vec_info dstmt_vinfo;
244 basic_block bb, def_bb;
247 enum vect_def_type dt;
249 /* case 1: we are only interested in uses that need to be vectorized. Uses
250 that are used for address computation are not considered relevant. */
251 if (!exist_non_indexing_operands_for_use_p (use, stmt))
254 if (!vect_is_simple_use (use, loop_vinfo, NULL, &def_stmt, &def, &dt))
256 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
257 fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
261 if (!def_stmt || gimple_nop_p (def_stmt))
264 def_bb = gimple_bb (def_stmt);
265 if (!flow_bb_inside_loop_p (loop, def_bb))
267 if (vect_print_dump_info (REPORT_DETAILS))
268 fprintf (vect_dump, "def_stmt is out of loop.");
272 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
273 DEF_STMT must have already been processed, because this should be the
274 only way that STMT, which is a reduction-phi, was put in the worklist,
275 as there should be no other uses for DEF_STMT in the loop. So we just
276 check that everything is as expected, and we are done. */
277 dstmt_vinfo = vinfo_for_stmt (def_stmt);
278 bb = gimple_bb (stmt);
279 if (gimple_code (stmt) == GIMPLE_PHI
280 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
281 && gimple_code (def_stmt) != GIMPLE_PHI
282 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
283 && bb->loop_father == def_bb->loop_father)
285 if (vect_print_dump_info (REPORT_DETAILS))
286 fprintf (vect_dump, "reduc-stmt defining reduc-phi in the same nest.");
287 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
288 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
289 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
290 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
291 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
295 /* case 3a: outer-loop stmt defining an inner-loop stmt:
296 outer-loop-header-bb:
302 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
304 if (vect_print_dump_info (REPORT_DETAILS))
305 fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
309 case vect_unused_in_scope:
310 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
311 vect_used_in_scope : vect_unused_in_scope;
314 case vect_used_in_outer_by_reduction:
315 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
316 relevant = vect_used_by_reduction;
319 case vect_used_in_outer:
320 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
321 relevant = vect_used_in_scope;
324 case vect_used_in_scope:
332 /* case 3b: inner-loop stmt defining an outer-loop stmt:
333 outer-loop-header-bb:
337 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
339 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
341 if (vect_print_dump_info (REPORT_DETAILS))
342 fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
346 case vect_unused_in_scope:
347 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
348 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
349 vect_used_in_outer_by_reduction : vect_unused_in_scope;
352 case vect_used_by_reduction:
353 relevant = vect_used_in_outer_by_reduction;
356 case vect_used_in_scope:
357 relevant = vect_used_in_outer;
365 vect_mark_relevant (worklist, def_stmt, relevant, live_p);
370 /* Function vect_mark_stmts_to_be_vectorized.
372 Not all stmts in the loop need to be vectorized. For example:
381 Stmt 1 and 3 do not need to be vectorized, because loop control and
382 addressing of vectorized data-refs are handled differently.
384 This pass detects such stmts. */
387 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
389 VEC(gimple,heap) *worklist;
390 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
391 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
392 unsigned int nbbs = loop->num_nodes;
393 gimple_stmt_iterator si;
396 stmt_vec_info stmt_vinfo;
400 enum vect_relevant relevant, tmp_relevant;
401 enum vect_def_type def_type;
403 if (vect_print_dump_info (REPORT_DETAILS))
404 fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
406 worklist = VEC_alloc (gimple, heap, 64);
408 /* 1. Init worklist. */
409 for (i = 0; i < nbbs; i++)
412 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
415 if (vect_print_dump_info (REPORT_DETAILS))
417 fprintf (vect_dump, "init: phi relevant? ");
418 print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
421 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
422 vect_mark_relevant (&worklist, phi, relevant, live_p);
424 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
426 stmt = gsi_stmt (si);
427 if (vect_print_dump_info (REPORT_DETAILS))
429 fprintf (vect_dump, "init: stmt relevant? ");
430 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
433 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
434 vect_mark_relevant (&worklist, stmt, relevant, live_p);
438 /* 2. Process_worklist */
439 while (VEC_length (gimple, worklist) > 0)
444 stmt = VEC_pop (gimple, worklist);
445 if (vect_print_dump_info (REPORT_DETAILS))
447 fprintf (vect_dump, "worklist: examine stmt: ");
448 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
451 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
452 (DEF_STMT) as relevant/irrelevant and live/dead according to the
453 liveness and relevance properties of STMT. */
454 stmt_vinfo = vinfo_for_stmt (stmt);
455 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
456 live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
458 /* Generally, the liveness and relevance properties of STMT are
459 propagated as is to the DEF_STMTs of its USEs:
460 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
461 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
463 One exception is when STMT has been identified as defining a reduction
464 variable; in this case we set the liveness/relevance as follows:
466 relevant = vect_used_by_reduction
467 This is because we distinguish between two kinds of relevant stmts -
468 those that are used by a reduction computation, and those that are
469 (also) used by a regular computation. This allows us later on to
470 identify stmts that are used solely by a reduction, and therefore the
471 order of the results that they produce does not have to be kept. */
473 def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
474 tmp_relevant = relevant;
477 case vect_reduction_def:
478 switch (tmp_relevant)
480 case vect_unused_in_scope:
481 relevant = vect_used_by_reduction;
484 case vect_used_by_reduction:
485 if (gimple_code (stmt) == GIMPLE_PHI)
490 if (vect_print_dump_info (REPORT_DETAILS))
491 fprintf (vect_dump, "unsupported use of reduction.");
493 VEC_free (gimple, heap, worklist);
500 case vect_nested_cycle:
501 if (tmp_relevant != vect_unused_in_scope
502 && tmp_relevant != vect_used_in_outer_by_reduction
503 && tmp_relevant != vect_used_in_outer)
505 if (vect_print_dump_info (REPORT_DETAILS))
506 fprintf (vect_dump, "unsupported use of nested cycle.");
508 VEC_free (gimple, heap, worklist);
515 case vect_double_reduction_def:
516 if (tmp_relevant != vect_unused_in_scope
517 && tmp_relevant != vect_used_by_reduction)
519 if (vect_print_dump_info (REPORT_DETAILS))
520 fprintf (vect_dump, "unsupported use of double reduction.");
522 VEC_free (gimple, heap, worklist);
533 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
535 tree op = USE_FROM_PTR (use_p);
536 if (!process_use (stmt, op, loop_vinfo, live_p, relevant, &worklist))
538 VEC_free (gimple, heap, worklist);
542 } /* while worklist */
544 VEC_free (gimple, heap, worklist);
550 cost_for_stmt (gimple stmt)
552 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
554 switch (STMT_VINFO_TYPE (stmt_info))
556 case load_vec_info_type:
557 return TARG_SCALAR_LOAD_COST;
558 case store_vec_info_type:
559 return TARG_SCALAR_STORE_COST;
560 case op_vec_info_type:
561 case condition_vec_info_type:
562 case assignment_vec_info_type:
563 case reduc_vec_info_type:
564 case induc_vec_info_type:
565 case type_promotion_vec_info_type:
566 case type_demotion_vec_info_type:
567 case type_conversion_vec_info_type:
568 case call_vec_info_type:
569 return TARG_SCALAR_STMT_COST;
570 case undef_vec_info_type:
576 /* Function vect_model_simple_cost.
578 Models cost for simple operations, i.e. those that only emit ncopies of a
579 single op. Right now, this does not account for multiple insns that could
580 be generated for the single vector op. We will handle that shortly. */
583 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
584 enum vect_def_type *dt, slp_tree slp_node)
587 int inside_cost = 0, outside_cost = 0;
589 /* The SLP costs were already calculated during SLP tree build. */
590 if (PURE_SLP_STMT (stmt_info))
593 inside_cost = ncopies * TARG_VEC_STMT_COST;
595 /* FORNOW: Assuming maximum 2 args per stmts. */
596 for (i = 0; i < 2; i++)
598 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
599 outside_cost += TARG_SCALAR_TO_VEC_COST;
602 if (vect_print_dump_info (REPORT_COST))
603 fprintf (vect_dump, "vect_model_simple_cost: inside_cost = %d, "
604 "outside_cost = %d .", inside_cost, outside_cost);
606 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
607 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
608 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
612 /* Function vect_cost_strided_group_size
614 For strided load or store, return the group_size only if it is the first
615 load or store of a group, else return 1. This ensures that group size is
616 only returned once per group. */
619 vect_cost_strided_group_size (stmt_vec_info stmt_info)
621 gimple first_stmt = DR_GROUP_FIRST_DR (stmt_info);
623 if (first_stmt == STMT_VINFO_STMT (stmt_info))
624 return DR_GROUP_SIZE (stmt_info);
630 /* Function vect_model_store_cost
632 Models cost for stores. In the case of strided accesses, one access
633 has the overhead of the strided access attributed to it. */
636 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
637 enum vect_def_type dt, slp_tree slp_node)
640 int inside_cost = 0, outside_cost = 0;
642 /* The SLP costs were already calculated during SLP tree build. */
643 if (PURE_SLP_STMT (stmt_info))
646 if (dt == vect_constant_def || dt == vect_external_def)
647 outside_cost = TARG_SCALAR_TO_VEC_COST;
649 /* Strided access? */
650 if (DR_GROUP_FIRST_DR (stmt_info) && !slp_node)
651 group_size = vect_cost_strided_group_size (stmt_info);
652 /* Not a strided access. */
656 /* Is this an access in a group of stores, which provide strided access?
657 If so, add in the cost of the permutes. */
660 /* Uses a high and low interleave operation for each needed permute. */
661 inside_cost = ncopies * exact_log2(group_size) * group_size
662 * TARG_VEC_STMT_COST;
664 if (vect_print_dump_info (REPORT_COST))
665 fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .",
670 /* Costs of the stores. */
671 inside_cost += ncopies * TARG_VEC_STORE_COST;
673 if (vect_print_dump_info (REPORT_COST))
674 fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, "
675 "outside_cost = %d .", inside_cost, outside_cost);
677 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
678 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
679 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
683 /* Function vect_model_load_cost
685 Models cost for loads. In the case of strided accesses, the last access
686 has the overhead of the strided access attributed to it. Since unaligned
687 accesses are supported for loads, we also account for the costs of the
688 access scheme chosen. */
691 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node)
695 int alignment_support_cheme;
697 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
698 int inside_cost = 0, outside_cost = 0;
700 /* The SLP costs were already calculated during SLP tree build. */
701 if (PURE_SLP_STMT (stmt_info))
704 /* Strided accesses? */
705 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
706 if (first_stmt && !slp_node)
708 group_size = vect_cost_strided_group_size (stmt_info);
709 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
711 /* Not a strided access. */
718 alignment_support_cheme = vect_supportable_dr_alignment (first_dr);
720 /* Is this an access in a group of loads providing strided access?
721 If so, add in the cost of the permutes. */
724 /* Uses an even and odd extract operations for each needed permute. */
725 inside_cost = ncopies * exact_log2(group_size) * group_size
726 * TARG_VEC_STMT_COST;
728 if (vect_print_dump_info (REPORT_COST))
729 fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .",
734 /* The loads themselves. */
735 switch (alignment_support_cheme)
739 inside_cost += ncopies * TARG_VEC_LOAD_COST;
741 if (vect_print_dump_info (REPORT_COST))
742 fprintf (vect_dump, "vect_model_load_cost: aligned.");
746 case dr_unaligned_supported:
748 /* Here, we assign an additional cost for the unaligned load. */
749 inside_cost += ncopies * TARG_VEC_UNALIGNED_LOAD_COST;
751 if (vect_print_dump_info (REPORT_COST))
752 fprintf (vect_dump, "vect_model_load_cost: unaligned supported by "
757 case dr_explicit_realign:
759 inside_cost += ncopies * (2*TARG_VEC_LOAD_COST + TARG_VEC_STMT_COST);
761 /* FIXME: If the misalignment remains fixed across the iterations of
762 the containing loop, the following cost should be added to the
764 if (targetm.vectorize.builtin_mask_for_load)
765 inside_cost += TARG_VEC_STMT_COST;
769 case dr_explicit_realign_optimized:
771 if (vect_print_dump_info (REPORT_COST))
772 fprintf (vect_dump, "vect_model_load_cost: unaligned software "
775 /* Unaligned software pipeline has a load of an address, an initial
776 load, and possibly a mask operation to "prime" the loop. However,
777 if this is an access in a group of loads, which provide strided
778 access, then the above cost should only be considered for one
779 access in the group. Inside the loop, there is a load op
780 and a realignment op. */
782 if ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node)
784 outside_cost = 2*TARG_VEC_STMT_COST;
785 if (targetm.vectorize.builtin_mask_for_load)
786 outside_cost += TARG_VEC_STMT_COST;
789 inside_cost += ncopies * (TARG_VEC_LOAD_COST + TARG_VEC_STMT_COST);
798 if (vect_print_dump_info (REPORT_COST))
799 fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, "
800 "outside_cost = %d .", inside_cost, outside_cost);
802 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
803 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
804 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
808 /* Function vect_init_vector.
810 Insert a new stmt (INIT_STMT) that initializes a new vector variable with
811 the vector elements of VECTOR_VAR. Place the initialization at BSI if it
812 is not NULL. Otherwise, place the initialization at the loop preheader.
813 Return the DEF of INIT_STMT.
814 It will be used in the vectorization of STMT. */
817 vect_init_vector (gimple stmt, tree vector_var, tree vector_type,
818 gimple_stmt_iterator *gsi)
820 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
828 new_var = vect_get_new_vect_var (vector_type, vect_simple_var, "cst_");
829 add_referenced_var (new_var);
830 init_stmt = gimple_build_assign (new_var, vector_var);
831 new_temp = make_ssa_name (new_var, init_stmt);
832 gimple_assign_set_lhs (init_stmt, new_temp);
835 vect_finish_stmt_generation (stmt, init_stmt, gsi);
838 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
842 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
844 if (nested_in_vect_loop_p (loop, stmt))
847 pe = loop_preheader_edge (loop);
848 new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
849 gcc_assert (!new_bb);
853 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
855 gimple_stmt_iterator gsi_bb_start;
857 gcc_assert (bb_vinfo);
858 bb = BB_VINFO_BB (bb_vinfo);
859 gsi_bb_start = gsi_after_labels (bb);
860 gsi_insert_before (&gsi_bb_start, init_stmt, GSI_SAME_STMT);
864 if (vect_print_dump_info (REPORT_DETAILS))
866 fprintf (vect_dump, "created new init_stmt: ");
867 print_gimple_stmt (vect_dump, init_stmt, 0, TDF_SLIM);
870 vec_oprnd = gimple_assign_lhs (init_stmt);
875 /* Function vect_get_vec_def_for_operand.
877 OP is an operand in STMT. This function returns a (vector) def that will be
878 used in the vectorized stmt for STMT.
880 In the case that OP is an SSA_NAME which is defined in the loop, then
881 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
883 In case OP is an invariant or constant, a new stmt that creates a vector def
884 needs to be introduced. */
887 vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def)
892 stmt_vec_info def_stmt_info = NULL;
893 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
894 tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
895 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
896 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
902 enum vect_def_type dt;
906 if (vect_print_dump_info (REPORT_DETAILS))
908 fprintf (vect_dump, "vect_get_vec_def_for_operand: ");
909 print_generic_expr (vect_dump, op, TDF_SLIM);
912 is_simple_use = vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def,
914 gcc_assert (is_simple_use);
915 if (vect_print_dump_info (REPORT_DETAILS))
919 fprintf (vect_dump, "def = ");
920 print_generic_expr (vect_dump, def, TDF_SLIM);
924 fprintf (vect_dump, " def_stmt = ");
925 print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
931 /* Case 1: operand is a constant. */
932 case vect_constant_def:
934 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
935 gcc_assert (vector_type);
940 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
941 if (vect_print_dump_info (REPORT_DETAILS))
942 fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
944 for (i = nunits - 1; i >= 0; --i)
946 t = tree_cons (NULL_TREE, op, t);
948 vec_cst = build_vector (vector_type, t);
949 return vect_init_vector (stmt, vec_cst, vector_type, NULL);
952 /* Case 2: operand is defined outside the loop - loop invariant. */
953 case vect_external_def:
955 vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
956 gcc_assert (vector_type);
957 nunits = TYPE_VECTOR_SUBPARTS (vector_type);
962 /* Create 'vec_inv = {inv,inv,..,inv}' */
963 if (vect_print_dump_info (REPORT_DETAILS))
964 fprintf (vect_dump, "Create vector_inv.");
966 for (i = nunits - 1; i >= 0; --i)
968 t = tree_cons (NULL_TREE, def, t);
971 /* FIXME: use build_constructor directly. */
972 vec_inv = build_constructor_from_list (vector_type, t);
973 return vect_init_vector (stmt, vec_inv, vector_type, NULL);
976 /* Case 3: operand is defined inside the loop. */
977 case vect_internal_def:
980 *scalar_def = NULL/* FIXME tuples: def_stmt*/;
982 /* Get the def from the vectorized stmt. */
983 def_stmt_info = vinfo_for_stmt (def_stmt);
984 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
985 gcc_assert (vec_stmt);
986 if (gimple_code (vec_stmt) == GIMPLE_PHI)
987 vec_oprnd = PHI_RESULT (vec_stmt);
988 else if (is_gimple_call (vec_stmt))
989 vec_oprnd = gimple_call_lhs (vec_stmt);
991 vec_oprnd = gimple_assign_lhs (vec_stmt);
995 /* Case 4: operand is defined by a loop header phi - reduction */
996 case vect_reduction_def:
997 case vect_double_reduction_def:
998 case vect_nested_cycle:
1002 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1003 loop = (gimple_bb (def_stmt))->loop_father;
1005 /* Get the def before the loop */
1006 op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));
1007 return get_initial_def_for_reduction (stmt, op, scalar_def);
1010 /* Case 5: operand is defined by loop-header phi - induction. */
1011 case vect_induction_def:
1013 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1015 /* Get the def from the vectorized stmt. */
1016 def_stmt_info = vinfo_for_stmt (def_stmt);
1017 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1018 gcc_assert (vec_stmt && gimple_code (vec_stmt) == GIMPLE_PHI);
1019 vec_oprnd = PHI_RESULT (vec_stmt);
1029 /* Function vect_get_vec_def_for_stmt_copy
1031 Return a vector-def for an operand. This function is used when the
1032 vectorized stmt to be created (by the caller to this function) is a "copy"
1033 created in case the vectorized result cannot fit in one vector, and several
1034 copies of the vector-stmt are required. In this case the vector-def is
1035 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1036 of the stmt that defines VEC_OPRND.
1037 DT is the type of the vector def VEC_OPRND.
1040 In case the vectorization factor (VF) is bigger than the number
1041 of elements that can fit in a vectype (nunits), we have to generate
1042 more than one vector stmt to vectorize the scalar stmt. This situation
1043 arises when there are multiple data-types operated upon in the loop; the
1044 smallest data-type determines the VF, and as a result, when vectorizing
1045 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1046 vector stmt (each computing a vector of 'nunits' results, and together
1047 computing 'VF' results in each iteration). This function is called when
1048 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1049 which VF=16 and nunits=4, so the number of copies required is 4):
1051 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1053 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1054 VS1.1: vx.1 = memref1 VS1.2
1055 VS1.2: vx.2 = memref2 VS1.3
1056 VS1.3: vx.3 = memref3
1058 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1059 VSnew.1: vz1 = vx.1 + ... VSnew.2
1060 VSnew.2: vz2 = vx.2 + ... VSnew.3
1061 VSnew.3: vz3 = vx.3 + ...
1063 The vectorization of S1 is explained in vectorizable_load.
1064 The vectorization of S2:
1065 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1066 the function 'vect_get_vec_def_for_operand' is called to
1067 get the relevant vector-def for each operand of S2. For operand x it
1068 returns the vector-def 'vx.0'.
1070 To create the remaining copies of the vector-stmt (VSnew.j), this
1071 function is called to get the relevant vector-def for each operand. It is
1072 obtained from the respective VS1.j stmt, which is recorded in the
1073 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1075 For example, to obtain the vector-def 'vx.1' in order to create the
1076 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1077 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1078 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1079 and return its def ('vx.1').
1080 Overall, to create the above sequence this function will be called 3 times:
1081 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1082 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1083 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1086 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1088 gimple vec_stmt_for_operand;
1089 stmt_vec_info def_stmt_info;
1091 /* Do nothing; can reuse same def. */
1092 if (dt == vect_external_def || dt == vect_constant_def )
1095 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1096 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1097 gcc_assert (def_stmt_info);
1098 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1099 gcc_assert (vec_stmt_for_operand);
1100 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1101 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1102 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1104 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1109 /* Get vectorized definitions for the operands to create a copy of an original
1110 stmt. See vect_get_vec_def_for_stmt_copy() for details. */
1113 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1114 VEC(tree,heap) **vec_oprnds0,
1115 VEC(tree,heap) **vec_oprnds1)
1117 tree vec_oprnd = VEC_pop (tree, *vec_oprnds0);
1119 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1120 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1122 if (vec_oprnds1 && *vec_oprnds1)
1124 vec_oprnd = VEC_pop (tree, *vec_oprnds1);
1125 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1126 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1131 /* Get vectorized definitions for OP0 and OP1, or SLP_NODE if it is not NULL. */
1134 vect_get_vec_defs (tree op0, tree op1, gimple stmt,
1135 VEC(tree,heap) **vec_oprnds0, VEC(tree,heap) **vec_oprnds1,
1139 vect_get_slp_defs (slp_node, vec_oprnds0, vec_oprnds1);
1144 *vec_oprnds0 = VEC_alloc (tree, heap, 1);
1145 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt, NULL);
1146 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1150 *vec_oprnds1 = VEC_alloc (tree, heap, 1);
1151 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt, NULL);
1152 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1158 /* Function vect_finish_stmt_generation.
1160 Insert a new stmt. */
1163 vect_finish_stmt_generation (gimple stmt, gimple vec_stmt,
1164 gimple_stmt_iterator *gsi)
1166 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1167 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1168 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1170 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1172 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1174 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo,
1177 if (vect_print_dump_info (REPORT_DETAILS))
1179 fprintf (vect_dump, "add new stmt: ");
1180 print_gimple_stmt (vect_dump, vec_stmt, 0, TDF_SLIM);
1183 gimple_set_location (vec_stmt, gimple_location (gsi_stmt (*gsi)));
1186 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1187 a function declaration if the target has a vectorized version
1188 of the function, or NULL_TREE if the function cannot be vectorized. */
1191 vectorizable_function (gimple call, tree vectype_out, tree vectype_in)
1193 tree fndecl = gimple_call_fndecl (call);
1194 enum built_in_function code;
1196 /* We only handle functions that do not read or clobber memory -- i.e.
1197 const or novops ones. */
1198 if (!(gimple_call_flags (call) & (ECF_CONST | ECF_NOVOPS)))
1202 || TREE_CODE (fndecl) != FUNCTION_DECL
1203 || !DECL_BUILT_IN (fndecl))
1206 code = DECL_FUNCTION_CODE (fndecl);
1207 return targetm.vectorize.builtin_vectorized_function (code, vectype_out,
1211 /* Function vectorizable_call.
1213 Check if STMT performs a function call that can be vectorized.
1214 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1215 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1216 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1219 vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt)
1224 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1225 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
1226 tree vectype_out, vectype_in;
1229 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1230 tree fndecl, new_temp, def, rhs_type, lhs_type;
1232 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1233 gimple new_stmt = NULL;
1235 VEC(tree, heap) *vargs = NULL;
1236 enum { NARROW, NONE, WIDEN } modifier;
1239 /* FORNOW: unsupported in basic block SLP. */
1240 gcc_assert (loop_vinfo);
1242 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1245 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1248 /* FORNOW: SLP not supported. */
1249 if (STMT_SLP_TYPE (stmt_info))
1252 /* Is STMT a vectorizable call? */
1253 if (!is_gimple_call (stmt))
1256 if (TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
1259 /* Process function arguments. */
1260 rhs_type = NULL_TREE;
1261 nargs = gimple_call_num_args (stmt);
1263 /* Bail out if the function has more than two arguments, we
1264 do not have interesting builtin functions to vectorize with
1265 more than two arguments. No arguments is also not good. */
1266 if (nargs == 0 || nargs > 2)
1269 for (i = 0; i < nargs; i++)
1271 op = gimple_call_arg (stmt, i);
1273 /* We can only handle calls with arguments of the same type. */
1275 && rhs_type != TREE_TYPE (op))
1277 if (vect_print_dump_info (REPORT_DETAILS))
1278 fprintf (vect_dump, "argument types differ.");
1281 rhs_type = TREE_TYPE (op);
1283 if (!vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt[i]))
1285 if (vect_print_dump_info (REPORT_DETAILS))
1286 fprintf (vect_dump, "use not simple.");
1291 vectype_in = get_vectype_for_scalar_type (rhs_type);
1294 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1296 lhs_type = TREE_TYPE (gimple_call_lhs (stmt));
1297 vectype_out = get_vectype_for_scalar_type (lhs_type);
1300 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1303 if (nunits_in == nunits_out / 2)
1305 else if (nunits_out == nunits_in)
1307 else if (nunits_out == nunits_in / 2)
1312 /* For now, we only vectorize functions if a target specific builtin
1313 is available. TODO -- in some cases, it might be profitable to
1314 insert the calls for pieces of the vector, in order to be able
1315 to vectorize other operations in the loop. */
1316 fndecl = vectorizable_function (stmt, vectype_out, vectype_in);
1317 if (fndecl == NULL_TREE)
1319 if (vect_print_dump_info (REPORT_DETAILS))
1320 fprintf (vect_dump, "function is not vectorizable.");
1325 gcc_assert (!gimple_vuse (stmt));
1327 if (modifier == NARROW)
1328 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1330 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1332 /* Sanity check: make sure that at least one copy of the vectorized stmt
1333 needs to be generated. */
1334 gcc_assert (ncopies >= 1);
1336 if (!vec_stmt) /* transformation not required. */
1338 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
1339 if (vect_print_dump_info (REPORT_DETAILS))
1340 fprintf (vect_dump, "=== vectorizable_call ===");
1341 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
1347 if (vect_print_dump_info (REPORT_DETAILS))
1348 fprintf (vect_dump, "transform operation.");
1351 scalar_dest = gimple_call_lhs (stmt);
1352 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1354 prev_stmt_info = NULL;
1358 for (j = 0; j < ncopies; ++j)
1360 /* Build argument list for the vectorized call. */
1362 vargs = VEC_alloc (tree, heap, nargs);
1364 VEC_truncate (tree, vargs, 0);
1366 for (i = 0; i < nargs; i++)
1368 op = gimple_call_arg (stmt, i);
1371 = vect_get_vec_def_for_operand (op, stmt, NULL);
1374 vec_oprnd0 = gimple_call_arg (new_stmt, i);
1376 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1379 VEC_quick_push (tree, vargs, vec_oprnd0);
1382 new_stmt = gimple_build_call_vec (fndecl, vargs);
1383 new_temp = make_ssa_name (vec_dest, new_stmt);
1384 gimple_call_set_lhs (new_stmt, new_temp);
1386 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1387 mark_symbols_for_renaming (new_stmt);
1390 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1392 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1394 prev_stmt_info = vinfo_for_stmt (new_stmt);
1400 for (j = 0; j < ncopies; ++j)
1402 /* Build argument list for the vectorized call. */
1404 vargs = VEC_alloc (tree, heap, nargs * 2);
1406 VEC_truncate (tree, vargs, 0);
1408 for (i = 0; i < nargs; i++)
1410 op = gimple_call_arg (stmt, i);
1414 = vect_get_vec_def_for_operand (op, stmt, NULL);
1416 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1420 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i);
1422 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
1424 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1427 VEC_quick_push (tree, vargs, vec_oprnd0);
1428 VEC_quick_push (tree, vargs, vec_oprnd1);
1431 new_stmt = gimple_build_call_vec (fndecl, vargs);
1432 new_temp = make_ssa_name (vec_dest, new_stmt);
1433 gimple_call_set_lhs (new_stmt, new_temp);
1435 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1436 mark_symbols_for_renaming (new_stmt);
1439 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1441 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1443 prev_stmt_info = vinfo_for_stmt (new_stmt);
1446 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1451 /* No current target implements this case. */
1455 VEC_free (tree, heap, vargs);
1457 /* Update the exception handling table with the vector stmt if necessary. */
1458 if (maybe_clean_or_replace_eh_stmt (stmt, *vec_stmt))
1459 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1461 /* The call in STMT might prevent it from being removed in dce.
1462 We however cannot remove it here, due to the way the ssa name
1463 it defines is mapped to the new definition. So just replace
1464 rhs of the statement with something harmless. */
1466 type = TREE_TYPE (scalar_dest);
1467 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
1468 fold_convert (type, integer_zero_node));
1469 set_vinfo_for_stmt (new_stmt, stmt_info);
1470 set_vinfo_for_stmt (stmt, NULL);
1471 STMT_VINFO_STMT (stmt_info) = new_stmt;
1472 gsi_replace (gsi, new_stmt, false);
1473 SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
1479 /* Function vect_gen_widened_results_half
1481 Create a vector stmt whose code, type, number of arguments, and result
1482 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
1483 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
1484 In the case that CODE is a CALL_EXPR, this means that a call to DECL
1485 needs to be created (DECL is a function-decl of a target-builtin).
1486 STMT is the original scalar stmt that we are vectorizing. */
1489 vect_gen_widened_results_half (enum tree_code code,
1491 tree vec_oprnd0, tree vec_oprnd1, int op_type,
1492 tree vec_dest, gimple_stmt_iterator *gsi,
1498 /* Generate half of the widened result: */
1499 if (code == CALL_EXPR)
1501 /* Target specific support */
1502 if (op_type == binary_op)
1503 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
1505 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
1506 new_temp = make_ssa_name (vec_dest, new_stmt);
1507 gimple_call_set_lhs (new_stmt, new_temp);
1511 /* Generic support */
1512 gcc_assert (op_type == TREE_CODE_LENGTH (code));
1513 if (op_type != binary_op)
1515 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vec_oprnd0,
1517 new_temp = make_ssa_name (vec_dest, new_stmt);
1518 gimple_assign_set_lhs (new_stmt, new_temp);
1520 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1526 /* Check if STMT performs a conversion operation, that can be vectorized.
1527 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1528 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1529 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1532 vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
1533 gimple *vec_stmt, slp_tree slp_node)
1538 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1539 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1540 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1541 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
1542 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
1546 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1547 gimple new_stmt = NULL;
1548 stmt_vec_info prev_stmt_info;
1551 tree vectype_out, vectype_in;
1554 tree rhs_type, lhs_type;
1556 enum { NARROW, NONE, WIDEN } modifier;
1558 VEC(tree,heap) *vec_oprnds0 = NULL;
1561 VEC(tree,heap) *dummy = NULL;
1564 /* Is STMT a vectorizable conversion? */
1566 /* FORNOW: unsupported in basic block SLP. */
1567 gcc_assert (loop_vinfo);
1569 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1572 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1575 if (!is_gimple_assign (stmt))
1578 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
1581 code = gimple_assign_rhs_code (stmt);
1582 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
1585 /* Check types of lhs and rhs. */
1586 op0 = gimple_assign_rhs1 (stmt);
1587 rhs_type = TREE_TYPE (op0);
1588 vectype_in = get_vectype_for_scalar_type (rhs_type);
1591 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1593 scalar_dest = gimple_assign_lhs (stmt);
1594 lhs_type = TREE_TYPE (scalar_dest);
1595 vectype_out = get_vectype_for_scalar_type (lhs_type);
1598 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1601 if (nunits_in == nunits_out / 2)
1603 else if (nunits_out == nunits_in)
1605 else if (nunits_out == nunits_in / 2)
1610 if (modifier == NONE)
1611 gcc_assert (STMT_VINFO_VECTYPE (stmt_info) == vectype_out);
1613 /* Bail out if the types are both integral or non-integral. */
1614 if ((INTEGRAL_TYPE_P (rhs_type) && INTEGRAL_TYPE_P (lhs_type))
1615 || (!INTEGRAL_TYPE_P (rhs_type) && !INTEGRAL_TYPE_P (lhs_type)))
1618 integral_type = INTEGRAL_TYPE_P (rhs_type) ? vectype_in : vectype_out;
1620 if (modifier == NARROW)
1621 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1623 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1625 /* FORNOW: SLP with multiple types is not supported. The SLP analysis verifies
1626 this, so we can safely override NCOPIES with 1 here. */
1630 /* Sanity check: make sure that at least one copy of the vectorized stmt
1631 needs to be generated. */
1632 gcc_assert (ncopies >= 1);
1634 /* Check the operands of the operation. */
1635 if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
1637 if (vect_print_dump_info (REPORT_DETAILS))
1638 fprintf (vect_dump, "use not simple.");
1642 /* Supportable by target? */
1643 if ((modifier == NONE
1644 && !targetm.vectorize.builtin_conversion (code, integral_type))
1645 || (modifier == WIDEN
1646 && !supportable_widening_operation (code, stmt, vectype_in,
1649 &dummy_int, &dummy))
1650 || (modifier == NARROW
1651 && !supportable_narrowing_operation (code, stmt, vectype_in,
1652 &code1, &dummy_int, &dummy)))
1654 if (vect_print_dump_info (REPORT_DETAILS))
1655 fprintf (vect_dump, "conversion not supported by target.");
1659 if (modifier != NONE)
1661 STMT_VINFO_VECTYPE (stmt_info) = vectype_in;
1662 /* FORNOW: SLP not supported. */
1663 if (STMT_SLP_TYPE (stmt_info))
1667 if (!vec_stmt) /* transformation not required. */
1669 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
1674 if (vect_print_dump_info (REPORT_DETAILS))
1675 fprintf (vect_dump, "transform conversion.");
1678 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1680 if (modifier == NONE && !slp_node)
1681 vec_oprnds0 = VEC_alloc (tree, heap, 1);
1683 prev_stmt_info = NULL;
1687 for (j = 0; j < ncopies; j++)
1690 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node);
1692 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
1695 targetm.vectorize.builtin_conversion (code, integral_type);
1696 for (i = 0; VEC_iterate (tree, vec_oprnds0, i, vop0); i++)
1698 /* Arguments are ready. create the new vector stmt. */
1699 new_stmt = gimple_build_call (builtin_decl, 1, vop0);
1700 new_temp = make_ssa_name (vec_dest, new_stmt);
1701 gimple_call_set_lhs (new_stmt, new_temp);
1702 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1704 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
1708 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1710 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1711 prev_stmt_info = vinfo_for_stmt (new_stmt);
1716 /* In case the vectorization factor (VF) is bigger than the number
1717 of elements that we can fit in a vectype (nunits), we have to
1718 generate more than one vector stmt - i.e - we need to "unroll"
1719 the vector stmt by a factor VF/nunits. */
1720 for (j = 0; j < ncopies; j++)
1723 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1725 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1727 STMT_VINFO_VECTYPE (stmt_info) = vectype_in;
1729 /* Generate first half of the widened result: */
1731 = vect_gen_widened_results_half (code1, decl1,
1732 vec_oprnd0, vec_oprnd1,
1733 unary_op, vec_dest, gsi, stmt);
1735 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1737 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1738 prev_stmt_info = vinfo_for_stmt (new_stmt);
1740 /* Generate second half of the widened result: */
1742 = vect_gen_widened_results_half (code2, decl2,
1743 vec_oprnd0, vec_oprnd1,
1744 unary_op, vec_dest, gsi, stmt);
1745 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1746 prev_stmt_info = vinfo_for_stmt (new_stmt);
1751 /* In case the vectorization factor (VF) is bigger than the number
1752 of elements that we can fit in a vectype (nunits), we have to
1753 generate more than one vector stmt - i.e - we need to "unroll"
1754 the vector stmt by a factor VF/nunits. */
1755 for (j = 0; j < ncopies; j++)
1760 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1761 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1765 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd1);
1766 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1769 /* Arguments are ready. Create the new vector stmt. */
1770 expr = build2 (code1, vectype_out, vec_oprnd0, vec_oprnd1);
1771 new_stmt = gimple_build_assign_with_ops (code1, vec_dest, vec_oprnd0,
1773 new_temp = make_ssa_name (vec_dest, new_stmt);
1774 gimple_assign_set_lhs (new_stmt, new_temp);
1775 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1778 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1780 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1782 prev_stmt_info = vinfo_for_stmt (new_stmt);
1785 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1789 VEC_free (tree, heap, vec_oprnds0);
1793 /* Function vectorizable_assignment.
1795 Check if STMT performs an assignment (copy) that can be vectorized.
1796 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1797 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1798 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1801 vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi,
1802 gimple *vec_stmt, slp_tree slp_node)
1807 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1808 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1809 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1813 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1814 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
1817 VEC(tree,heap) *vec_oprnds = NULL;
1819 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1821 /* Multiple types in SLP are handled by creating the appropriate number of
1822 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1827 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
1829 gcc_assert (ncopies >= 1);
1831 return false; /* FORNOW */
1833 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
1836 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1839 /* Is vectorizable assignment? */
1840 if (!is_gimple_assign (stmt))
1843 scalar_dest = gimple_assign_lhs (stmt);
1844 if (TREE_CODE (scalar_dest) != SSA_NAME)
1847 if (gimple_assign_single_p (stmt)
1848 || gimple_assign_rhs_code (stmt) == PAREN_EXPR)
1849 op = gimple_assign_rhs1 (stmt);
1853 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[0]))
1855 if (vect_print_dump_info (REPORT_DETAILS))
1856 fprintf (vect_dump, "use not simple.");
1860 if (!vec_stmt) /* transformation not required. */
1862 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
1863 if (vect_print_dump_info (REPORT_DETAILS))
1864 fprintf (vect_dump, "=== vectorizable_assignment ===");
1865 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
1870 if (vect_print_dump_info (REPORT_DETAILS))
1871 fprintf (vect_dump, "transform assignment.");
1874 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1877 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node);
1879 /* Arguments are ready. create the new vector stmt. */
1880 for (i = 0; VEC_iterate (tree, vec_oprnds, i, vop); i++)
1882 *vec_stmt = gimple_build_assign (vec_dest, vop);
1883 new_temp = make_ssa_name (vec_dest, *vec_stmt);
1884 gimple_assign_set_lhs (*vec_stmt, new_temp);
1885 vect_finish_stmt_generation (stmt, *vec_stmt, gsi);
1886 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt;
1889 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), *vec_stmt);
1892 VEC_free (tree, heap, vec_oprnds);
1896 /* Function vectorizable_operation.
1898 Check if STMT performs a binary or unary operation that can be vectorized.
1899 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1900 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1901 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1904 vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
1905 gimple *vec_stmt, slp_tree slp_node)
1909 tree op0, op1 = NULL;
1910 tree vec_oprnd1 = NULL_TREE;
1911 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1912 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1913 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1914 enum tree_code code;
1915 enum machine_mode vec_mode;
1920 enum machine_mode optab_op2_mode;
1923 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1924 gimple new_stmt = NULL;
1925 stmt_vec_info prev_stmt_info;
1926 int nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
1931 VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
1934 bool shift_p = false;
1935 bool scalar_shift_arg = false;
1936 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1940 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1942 /* FORNOW: multiple types are not supported in basic block SLP. */
1945 /* Multiple types in SLP are handled by creating the appropriate number of
1946 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1951 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1953 gcc_assert (ncopies >= 1);
1955 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
1958 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1961 /* Is STMT a vectorizable binary/unary operation? */
1962 if (!is_gimple_assign (stmt))
1965 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
1968 scalar_dest = gimple_assign_lhs (stmt);
1969 vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
1972 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1973 if (nunits_out != nunits_in)
1976 code = gimple_assign_rhs_code (stmt);
1978 /* For pointer addition, we should use the normal plus for
1979 the vector addition. */
1980 if (code == POINTER_PLUS_EXPR)
1983 /* Support only unary or binary operations. */
1984 op_type = TREE_CODE_LENGTH (code);
1985 if (op_type != unary_op && op_type != binary_op)
1987 if (vect_print_dump_info (REPORT_DETAILS))
1988 fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type);
1992 op0 = gimple_assign_rhs1 (stmt);
1993 if (!vect_is_simple_use (op0, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[0]))
1995 if (vect_print_dump_info (REPORT_DETAILS))
1996 fprintf (vect_dump, "use not simple.");
2000 if (op_type == binary_op)
2002 op1 = gimple_assign_rhs2 (stmt);
2003 if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def,
2006 if (vect_print_dump_info (REPORT_DETAILS))
2007 fprintf (vect_dump, "use not simple.");
2012 /* If this is a shift/rotate, determine whether the shift amount is a vector,
2013 or scalar. If the shift/rotate amount is a vector, use the vector/vector
2015 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
2016 || code == RROTATE_EXPR)
2020 /* vector shifted by vector */
2021 if (dt[1] == vect_internal_def)
2023 optab = optab_for_tree_code (code, vectype, optab_vector);
2024 if (vect_print_dump_info (REPORT_DETAILS))
2025 fprintf (vect_dump, "vector/vector shift/rotate found.");
2028 /* See if the machine has a vector shifted by scalar insn and if not
2029 then see if it has a vector shifted by vector insn */
2030 else if (dt[1] == vect_constant_def || dt[1] == vect_external_def)
2032 optab = optab_for_tree_code (code, vectype, optab_scalar);
2034 && (optab_handler (optab, TYPE_MODE (vectype))->insn_code
2035 != CODE_FOR_nothing))
2037 scalar_shift_arg = true;
2038 if (vect_print_dump_info (REPORT_DETAILS))
2039 fprintf (vect_dump, "vector/scalar shift/rotate found.");
2043 optab = optab_for_tree_code (code, vectype, optab_vector);
2045 && (optab_handler (optab, TYPE_MODE (vectype))->insn_code
2046 != CODE_FOR_nothing))
2048 if (vect_print_dump_info (REPORT_DETAILS))
2049 fprintf (vect_dump, "vector/vector shift/rotate found.");
2051 /* Unlike the other binary operators, shifts/rotates have
2052 the rhs being int, instead of the same type as the lhs,
2053 so make sure the scalar is the right type if we are
2054 dealing with vectors of short/char. */
2055 if (dt[1] == vect_constant_def)
2056 op1 = fold_convert (TREE_TYPE (vectype), op1);
2063 if (vect_print_dump_info (REPORT_DETAILS))
2064 fprintf (vect_dump, "operand mode requires invariant argument.");
2069 optab = optab_for_tree_code (code, vectype, optab_default);
2071 /* Supportable by target? */
2074 if (vect_print_dump_info (REPORT_DETAILS))
2075 fprintf (vect_dump, "no optab.");
2078 vec_mode = TYPE_MODE (vectype);
2079 icode = (int) optab_handler (optab, vec_mode)->insn_code;
2080 if (icode == CODE_FOR_nothing)
2082 if (vect_print_dump_info (REPORT_DETAILS))
2083 fprintf (vect_dump, "op not supported by target.");
2084 /* Check only during analysis. */
2085 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
2086 || (vf < vect_min_worthwhile_factor (code)
2089 if (vect_print_dump_info (REPORT_DETAILS))
2090 fprintf (vect_dump, "proceeding using word mode.");
2093 /* Worthwhile without SIMD support? Check only during analysis. */
2094 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
2095 && vf < vect_min_worthwhile_factor (code)
2098 if (vect_print_dump_info (REPORT_DETAILS))
2099 fprintf (vect_dump, "not worthwhile without SIMD support.");
2103 if (!vec_stmt) /* transformation not required. */
2105 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
2106 if (vect_print_dump_info (REPORT_DETAILS))
2107 fprintf (vect_dump, "=== vectorizable_operation ===");
2108 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2114 if (vect_print_dump_info (REPORT_DETAILS))
2115 fprintf (vect_dump, "transform binary/unary operation.");
2118 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2120 /* Allocate VECs for vector operands. In case of SLP, vector operands are
2121 created in the previous stages of the recursion, so no allocation is
2122 needed, except for the case of shift with scalar shift argument. In that
2123 case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
2124 be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
2125 In case of loop-based vectorization we allocate VECs of size 1. We
2126 allocate VEC_OPRNDS1 only in case of binary operation. */
2129 vec_oprnds0 = VEC_alloc (tree, heap, 1);
2130 if (op_type == binary_op)
2131 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2133 else if (scalar_shift_arg)
2134 vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
2136 /* In case the vectorization factor (VF) is bigger than the number
2137 of elements that we can fit in a vectype (nunits), we have to generate
2138 more than one vector stmt - i.e - we need to "unroll" the
2139 vector stmt by a factor VF/nunits. In doing so, we record a pointer
2140 from one copy of the vector stmt to the next, in the field
2141 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
2142 stages to find the correct vector defs to be used when vectorizing
2143 stmts that use the defs of the current stmt. The example below illustrates
2144 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
2145 4 vectorized stmts):
2147 before vectorization:
2148 RELATED_STMT VEC_STMT
2152 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
2154 RELATED_STMT VEC_STMT
2155 VS1_0: vx0 = memref0 VS1_1 -
2156 VS1_1: vx1 = memref1 VS1_2 -
2157 VS1_2: vx2 = memref2 VS1_3 -
2158 VS1_3: vx3 = memref3 - -
2159 S1: x = load - VS1_0
2162 step2: vectorize stmt S2 (done here):
2163 To vectorize stmt S2 we first need to find the relevant vector
2164 def for the first operand 'x'. This is, as usual, obtained from
2165 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
2166 that defines 'x' (S1). This way we find the stmt VS1_0, and the
2167 relevant vector def 'vx0'. Having found 'vx0' we can generate
2168 the vector stmt VS2_0, and as usual, record it in the
2169 STMT_VINFO_VEC_STMT of stmt S2.
2170 When creating the second copy (VS2_1), we obtain the relevant vector
2171 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
2172 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
2173 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
2174 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
2175 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
2176 chain of stmts and pointers:
2177 RELATED_STMT VEC_STMT
2178 VS1_0: vx0 = memref0 VS1_1 -
2179 VS1_1: vx1 = memref1 VS1_2 -
2180 VS1_2: vx2 = memref2 VS1_3 -
2181 VS1_3: vx3 = memref3 - -
2182 S1: x = load - VS1_0
2183 VS2_0: vz0 = vx0 + v1 VS2_1 -
2184 VS2_1: vz1 = vx1 + v1 VS2_2 -
2185 VS2_2: vz2 = vx2 + v1 VS2_3 -
2186 VS2_3: vz3 = vx3 + v1 - -
2187 S2: z = x + 1 - VS2_0 */
2189 prev_stmt_info = NULL;
2190 for (j = 0; j < ncopies; j++)
2195 if (op_type == binary_op && scalar_shift_arg)
2197 /* Vector shl and shr insn patterns can be defined with scalar
2198 operand 2 (shift operand). In this case, use constant or loop
2199 invariant op1 directly, without extending it to vector mode
2201 optab_op2_mode = insn_data[icode].operand[2].mode;
2202 if (!VECTOR_MODE_P (optab_op2_mode))
2204 if (vect_print_dump_info (REPORT_DETAILS))
2205 fprintf (vect_dump, "operand 1 using scalar mode.");
2207 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2210 /* Store vec_oprnd1 for every vector stmt to be created
2211 for SLP_NODE. We check during the analysis that all the
2212 shift arguments are the same.
2213 TODO: Allow different constants for different vector
2214 stmts generated for an SLP instance. */
2215 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
2216 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2221 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
2222 (a special case for certain kind of vector shifts); otherwise,
2223 operand 1 should be of a vector type (the usual case). */
2224 if (op_type == binary_op && !vec_oprnd1)
2225 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
2228 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
2232 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
2234 /* Arguments are ready. Create the new vector stmt. */
2235 for (i = 0; VEC_iterate (tree, vec_oprnds0, i, vop0); i++)
2237 vop1 = ((op_type == binary_op)
2238 ? VEC_index (tree, vec_oprnds1, i) : NULL);
2239 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2240 new_temp = make_ssa_name (vec_dest, new_stmt);
2241 gimple_assign_set_lhs (new_stmt, new_temp);
2242 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2244 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2251 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2253 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2254 prev_stmt_info = vinfo_for_stmt (new_stmt);
2257 VEC_free (tree, heap, vec_oprnds0);
2259 VEC_free (tree, heap, vec_oprnds1);
2265 /* Get vectorized definitions for loop-based vectorization. For the first
2266 operand we call vect_get_vec_def_for_operand() (with OPRND containing
2267 scalar operand), and for the rest we get a copy with
2268 vect_get_vec_def_for_stmt_copy() using the previous vector definition
2269 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
2270 The vectors are collected into VEC_OPRNDS. */
2273 vect_get_loop_based_defs (tree *oprnd, gimple stmt, enum vect_def_type dt,
2274 VEC (tree, heap) **vec_oprnds, int multi_step_cvt)
2278 /* Get first vector operand. */
2279 /* All the vector operands except the very first one (that is scalar oprnd)
2281 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
2282 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt, NULL);
2284 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
2286 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2288 /* Get second vector operand. */
2289 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
2290 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2294 /* For conversion in multiple steps, continue to get operands
2297 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
2301 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
2302 For multi-step conversions store the resulting vectors and call the function
2306 vect_create_vectorized_demotion_stmts (VEC (tree, heap) **vec_oprnds,
2307 int multi_step_cvt, gimple stmt,
2308 VEC (tree, heap) *vec_dsts,
2309 gimple_stmt_iterator *gsi,
2310 slp_tree slp_node, enum tree_code code,
2311 stmt_vec_info *prev_stmt_info)
2314 tree vop0, vop1, new_tmp, vec_dest;
2316 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2318 vec_dest = VEC_pop (tree, vec_dsts);
2320 for (i = 0; i < VEC_length (tree, *vec_oprnds); i += 2)
2322 /* Create demotion operation. */
2323 vop0 = VEC_index (tree, *vec_oprnds, i);
2324 vop1 = VEC_index (tree, *vec_oprnds, i + 1);
2325 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2326 new_tmp = make_ssa_name (vec_dest, new_stmt);
2327 gimple_assign_set_lhs (new_stmt, new_tmp);
2328 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2331 /* Store the resulting vector for next recursive call. */
2332 VEC_replace (tree, *vec_oprnds, i/2, new_tmp);
2335 /* This is the last step of the conversion sequence. Store the
2336 vectors in SLP_NODE or in vector info of the scalar statement
2337 (or in STMT_VINFO_RELATED_STMT chain). */
2339 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2342 if (!*prev_stmt_info)
2343 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
2345 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
2347 *prev_stmt_info = vinfo_for_stmt (new_stmt);
2352 /* For multi-step demotion operations we first generate demotion operations
2353 from the source type to the intermediate types, and then combine the
2354 results (stored in VEC_OPRNDS) in demotion operation to the destination
2358 /* At each level of recursion we have have of the operands we had at the
2360 VEC_truncate (tree, *vec_oprnds, (i+1)/2);
2361 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
2362 stmt, vec_dsts, gsi, slp_node,
2363 code, prev_stmt_info);
2368 /* Function vectorizable_type_demotion
2370 Check if STMT performs a binary or unary operation that involves
2371 type demotion, and if it can be vectorized.
2372 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2373 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2374 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2377 vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi,
2378 gimple *vec_stmt, slp_tree slp_node)
2383 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2384 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2385 enum tree_code code, code1 = ERROR_MARK;
2388 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2389 stmt_vec_info prev_stmt_info;
2396 int multi_step_cvt = 0;
2397 VEC (tree, heap) *vec_oprnds0 = NULL;
2398 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
2399 tree last_oprnd, intermediate_type;
2401 /* FORNOW: not supported by basic block SLP vectorization. */
2402 gcc_assert (loop_vinfo);
2404 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2407 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2410 /* Is STMT a vectorizable type-demotion operation? */
2411 if (!is_gimple_assign (stmt))
2414 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2417 code = gimple_assign_rhs_code (stmt);
2418 if (!CONVERT_EXPR_CODE_P (code))
2421 op0 = gimple_assign_rhs1 (stmt);
2422 vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0));
2425 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2427 scalar_dest = gimple_assign_lhs (stmt);
2428 vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
2431 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2432 if (nunits_in >= nunits_out)
2435 /* Multiple types in SLP are handled by creating the appropriate number of
2436 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2441 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
2442 gcc_assert (ncopies >= 1);
2444 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
2445 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
2446 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
2447 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
2448 && CONVERT_EXPR_CODE_P (code))))
2451 /* Check the operands of the operation. */
2452 if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
2454 if (vect_print_dump_info (REPORT_DETAILS))
2455 fprintf (vect_dump, "use not simple.");
2459 /* Supportable by target? */
2460 if (!supportable_narrowing_operation (code, stmt, vectype_in, &code1,
2461 &multi_step_cvt, &interm_types))
2464 STMT_VINFO_VECTYPE (stmt_info) = vectype_in;
2466 if (!vec_stmt) /* transformation not required. */
2468 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
2469 if (vect_print_dump_info (REPORT_DETAILS))
2470 fprintf (vect_dump, "=== vectorizable_demotion ===");
2471 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2476 if (vect_print_dump_info (REPORT_DETAILS))
2477 fprintf (vect_dump, "transform type demotion operation. ncopies = %d.",
2480 /* In case of multi-step demotion, we first generate demotion operations to
2481 the intermediate types, and then from that types to the final one.
2482 We create vector destinations for the intermediate type (TYPES) received
2483 from supportable_narrowing_operation, and store them in the correct order
2484 for future use in vect_create_vectorized_demotion_stmts(). */
2486 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
2488 vec_dsts = VEC_alloc (tree, heap, 1);
2490 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2491 VEC_quick_push (tree, vec_dsts, vec_dest);
2495 for (i = VEC_length (tree, interm_types) - 1;
2496 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
2498 vec_dest = vect_create_destination_var (scalar_dest,
2500 VEC_quick_push (tree, vec_dsts, vec_dest);
2504 /* In case the vectorization factor (VF) is bigger than the number
2505 of elements that we can fit in a vectype (nunits), we have to generate
2506 more than one vector stmt - i.e - we need to "unroll" the
2507 vector stmt by a factor VF/nunits. */
2509 prev_stmt_info = NULL;
2510 for (j = 0; j < ncopies; j++)
2514 vect_get_slp_defs (slp_node, &vec_oprnds0, NULL);
2517 VEC_free (tree, heap, vec_oprnds0);
2518 vec_oprnds0 = VEC_alloc (tree, heap,
2519 (multi_step_cvt ? vect_pow2 (multi_step_cvt) * 2 : 2));
2520 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
2521 vect_pow2 (multi_step_cvt) - 1);
2524 /* Arguments are ready. Create the new vector stmts. */
2525 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
2526 vect_create_vectorized_demotion_stmts (&vec_oprnds0,
2527 multi_step_cvt, stmt, tmp_vec_dsts,
2528 gsi, slp_node, code1,
2532 VEC_free (tree, heap, vec_oprnds0);
2533 VEC_free (tree, heap, vec_dsts);
2534 VEC_free (tree, heap, tmp_vec_dsts);
2535 VEC_free (tree, heap, interm_types);
2537 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2542 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
2543 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
2544 the resulting vectors and call the function recursively. */
2547 vect_create_vectorized_promotion_stmts (VEC (tree, heap) **vec_oprnds0,
2548 VEC (tree, heap) **vec_oprnds1,
2549 int multi_step_cvt, gimple stmt,
2550 VEC (tree, heap) *vec_dsts,
2551 gimple_stmt_iterator *gsi,
2552 slp_tree slp_node, enum tree_code code1,
2553 enum tree_code code2, tree decl1,
2554 tree decl2, int op_type,
2555 stmt_vec_info *prev_stmt_info)
2558 tree vop0, vop1, new_tmp1, new_tmp2, vec_dest;
2559 gimple new_stmt1, new_stmt2;
2560 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2561 VEC (tree, heap) *vec_tmp;
2563 vec_dest = VEC_pop (tree, vec_dsts);
2564 vec_tmp = VEC_alloc (tree, heap, VEC_length (tree, *vec_oprnds0) * 2);
2566 for (i = 0; VEC_iterate (tree, *vec_oprnds0, i, vop0); i++)
2568 if (op_type == binary_op)
2569 vop1 = VEC_index (tree, *vec_oprnds1, i);
2573 /* Generate the two halves of promotion operation. */
2574 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
2575 op_type, vec_dest, gsi, stmt);
2576 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
2577 op_type, vec_dest, gsi, stmt);
2578 if (is_gimple_call (new_stmt1))
2580 new_tmp1 = gimple_call_lhs (new_stmt1);
2581 new_tmp2 = gimple_call_lhs (new_stmt2);
2585 new_tmp1 = gimple_assign_lhs (new_stmt1);
2586 new_tmp2 = gimple_assign_lhs (new_stmt2);
2591 /* Store the results for the recursive call. */
2592 VEC_quick_push (tree, vec_tmp, new_tmp1);
2593 VEC_quick_push (tree, vec_tmp, new_tmp2);
2597 /* Last step of promotion sequience - store the results. */
2600 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt1);
2601 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt2);
2605 if (!*prev_stmt_info)
2606 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt1;
2608 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt1;
2610 *prev_stmt_info = vinfo_for_stmt (new_stmt1);
2611 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt2;
2612 *prev_stmt_info = vinfo_for_stmt (new_stmt2);
2619 /* For multi-step promotion operation we first generate we call the
2620 function recurcively for every stage. We start from the input type,
2621 create promotion operations to the intermediate types, and then
2622 create promotions to the output type. */
2623 *vec_oprnds0 = VEC_copy (tree, heap, vec_tmp);
2624 VEC_free (tree, heap, vec_tmp);
2625 vect_create_vectorized_promotion_stmts (vec_oprnds0, vec_oprnds1,
2626 multi_step_cvt - 1, stmt,
2627 vec_dsts, gsi, slp_node, code1,
2628 code2, decl2, decl2, op_type,
2634 /* Function vectorizable_type_promotion
2636 Check if STMT performs a binary or unary operation that involves
2637 type promotion, and if it can be vectorized.
2638 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2639 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2640 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2643 vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi,
2644 gimple *vec_stmt, slp_tree slp_node)
2648 tree op0, op1 = NULL;
2649 tree vec_oprnd0=NULL, vec_oprnd1=NULL;
2650 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2651 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2652 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
2653 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
2657 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2658 stmt_vec_info prev_stmt_info;
2665 tree intermediate_type = NULL_TREE;
2666 int multi_step_cvt = 0;
2667 VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
2668 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
2670 /* FORNOW: not supported by basic block SLP vectorization. */
2671 gcc_assert (loop_vinfo);
2673 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2676 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2679 /* Is STMT a vectorizable type-promotion operation? */
2680 if (!is_gimple_assign (stmt))
2683 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2686 code = gimple_assign_rhs_code (stmt);
2687 if (!CONVERT_EXPR_CODE_P (code)
2688 && code != WIDEN_MULT_EXPR)
2691 op0 = gimple_assign_rhs1 (stmt);
2692 vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0));
2695 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2697 scalar_dest = gimple_assign_lhs (stmt);
2698 vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
2701 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2702 if (nunits_in <= nunits_out)
2705 /* Multiple types in SLP are handled by creating the appropriate number of
2706 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2711 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2713 gcc_assert (ncopies >= 1);
2715 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
2716 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
2717 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
2718 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
2719 && CONVERT_EXPR_CODE_P (code))))
2722 /* Check the operands of the operation. */
2723 if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0]))
2725 if (vect_print_dump_info (REPORT_DETAILS))
2726 fprintf (vect_dump, "use not simple.");
2730 op_type = TREE_CODE_LENGTH (code);
2731 if (op_type == binary_op)
2733 op1 = gimple_assign_rhs2 (stmt);
2734 if (!vect_is_simple_use (op1, loop_vinfo, NULL, &def_stmt, &def, &dt[1]))
2736 if (vect_print_dump_info (REPORT_DETAILS))
2737 fprintf (vect_dump, "use not simple.");
2742 /* Supportable by target? */
2743 if (!supportable_widening_operation (code, stmt, vectype_in,
2744 &decl1, &decl2, &code1, &code2,
2745 &multi_step_cvt, &interm_types))
2748 /* Binary widening operation can only be supported directly by the
2750 gcc_assert (!(multi_step_cvt && op_type == binary_op));
2752 STMT_VINFO_VECTYPE (stmt_info) = vectype_in;
2754 if (!vec_stmt) /* transformation not required. */
2756 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
2757 if (vect_print_dump_info (REPORT_DETAILS))
2758 fprintf (vect_dump, "=== vectorizable_promotion ===");
2759 vect_model_simple_cost (stmt_info, 2*ncopies, dt, NULL);
2765 if (vect_print_dump_info (REPORT_DETAILS))
2766 fprintf (vect_dump, "transform type promotion operation. ncopies = %d.",
2770 /* In case of multi-step promotion, we first generate promotion operations
2771 to the intermediate types, and then from that types to the final one.
2772 We store vector destination in VEC_DSTS in the correct order for
2773 recursive creation of promotion operations in
2774 vect_create_vectorized_promotion_stmts(). Vector destinations are created
2775 according to TYPES recieved from supportable_widening_operation(). */
2777 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
2779 vec_dsts = VEC_alloc (tree, heap, 1);
2781 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2782 VEC_quick_push (tree, vec_dsts, vec_dest);
2786 for (i = VEC_length (tree, interm_types) - 1;
2787 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
2789 vec_dest = vect_create_destination_var (scalar_dest,
2791 VEC_quick_push (tree, vec_dsts, vec_dest);
2797 vec_oprnds0 = VEC_alloc (tree, heap,
2798 (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
2799 if (op_type == binary_op)
2800 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2803 /* In case the vectorization factor (VF) is bigger than the number
2804 of elements that we can fit in a vectype (nunits), we have to generate
2805 more than one vector stmt - i.e - we need to "unroll" the
2806 vector stmt by a factor VF/nunits. */
2808 prev_stmt_info = NULL;
2809 for (j = 0; j < ncopies; j++)
2815 vect_get_slp_defs (slp_node, &vec_oprnds0, &vec_oprnds1);
2818 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
2819 VEC_quick_push (tree, vec_oprnds0, vec_oprnd0);
2820 if (op_type == binary_op)
2822 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt, NULL);
2823 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2829 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
2830 VEC_replace (tree, vec_oprnds0, 0, vec_oprnd0);
2831 if (op_type == binary_op)
2833 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd1);
2834 VEC_replace (tree, vec_oprnds1, 0, vec_oprnd1);
2838 /* Arguments are ready. Create the new vector stmts. */
2839 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
2840 vect_create_vectorized_promotion_stmts (&vec_oprnds0, &vec_oprnds1,
2841 multi_step_cvt, stmt,
2843 gsi, slp_node, code1, code2,
2844 decl1, decl2, op_type,
2848 VEC_free (tree, heap, vec_dsts);
2849 VEC_free (tree, heap, tmp_vec_dsts);
2850 VEC_free (tree, heap, interm_types);
2851 VEC_free (tree, heap, vec_oprnds0);
2852 VEC_free (tree, heap, vec_oprnds1);
2854 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2859 /* Function vectorizable_store.
2861 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
2863 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2864 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2865 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2868 vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
2874 tree vec_oprnd = NULL_TREE;
2875 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2876 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
2877 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2878 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2879 struct loop *loop = NULL;
2880 enum machine_mode vec_mode;
2882 enum dr_alignment_support alignment_support_scheme;
2885 enum vect_def_type dt;
2886 stmt_vec_info prev_stmt_info = NULL;
2887 tree dataref_ptr = NULL_TREE;
2888 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
2891 gimple next_stmt, first_stmt = NULL;
2892 bool strided_store = false;
2893 unsigned int group_size, i;
2894 VEC(tree,heap) *dr_chain = NULL, *oprnds = NULL, *result_chain = NULL;
2896 VEC(tree,heap) *vec_oprnds = NULL;
2897 bool slp = (slp_node != NULL);
2898 stmt_vec_info first_stmt_vinfo;
2899 unsigned int vec_num;
2900 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2903 loop = LOOP_VINFO_LOOP (loop_vinfo);
2905 /* Multiple types in SLP are handled by creating the appropriate number of
2906 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2911 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
2913 gcc_assert (ncopies >= 1);
2915 /* FORNOW. This restriction should be relaxed. */
2916 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
2918 if (vect_print_dump_info (REPORT_DETAILS))
2919 fprintf (vect_dump, "multiple types in nested loop.");
2923 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2926 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2929 /* Is vectorizable store? */
2931 if (!is_gimple_assign (stmt))
2934 scalar_dest = gimple_assign_lhs (stmt);
2935 if (TREE_CODE (scalar_dest) != ARRAY_REF
2936 && TREE_CODE (scalar_dest) != INDIRECT_REF
2937 && TREE_CODE (scalar_dest) != COMPONENT_REF
2938 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
2939 && TREE_CODE (scalar_dest) != REALPART_EXPR)
2942 gcc_assert (gimple_assign_single_p (stmt));
2943 op = gimple_assign_rhs1 (stmt);
2944 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt))
2946 if (vect_print_dump_info (REPORT_DETAILS))
2947 fprintf (vect_dump, "use not simple.");
2951 /* The scalar rhs type needs to be trivially convertible to the vector
2952 component type. This should always be the case. */
2953 if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (op)))
2955 if (vect_print_dump_info (REPORT_DETAILS))
2956 fprintf (vect_dump, "??? operands of different types");
2960 vec_mode = TYPE_MODE (vectype);
2961 /* FORNOW. In some cases can vectorize even if data-type not supported
2962 (e.g. - array initialization with 0). */
2963 if (optab_handler (mov_optab, (int)vec_mode)->insn_code == CODE_FOR_nothing)
2966 if (!STMT_VINFO_DATA_REF (stmt_info))
2969 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
2971 strided_store = true;
2972 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
2973 if (!vect_strided_store_supported (vectype)
2974 && !PURE_SLP_STMT (stmt_info) && !slp)
2977 if (first_stmt == stmt)
2979 /* STMT is the leader of the group. Check the operands of all the
2980 stmts of the group. */
2981 next_stmt = DR_GROUP_NEXT_DR (stmt_info);
2984 gcc_assert (gimple_assign_single_p (next_stmt));
2985 op = gimple_assign_rhs1 (next_stmt);
2986 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt,
2989 if (vect_print_dump_info (REPORT_DETAILS))
2990 fprintf (vect_dump, "use not simple.");
2993 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
2998 if (!vec_stmt) /* transformation not required. */
3000 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
3001 vect_model_store_cost (stmt_info, ncopies, dt, NULL);
3009 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3010 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3012 DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
3015 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
3017 /* We vectorize all the stmts of the interleaving group when we
3018 reach the last stmt in the group. */
3019 if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
3020 < DR_GROUP_SIZE (vinfo_for_stmt (first_stmt))
3028 strided_store = false;
3030 /* VEC_NUM is the number of vect stmts to be created for this group. */
3032 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3034 vec_num = group_size;
3040 group_size = vec_num = 1;
3041 first_stmt_vinfo = stmt_info;
3044 if (vect_print_dump_info (REPORT_DETAILS))
3045 fprintf (vect_dump, "transform store. ncopies = %d",ncopies);
3047 dr_chain = VEC_alloc (tree, heap, group_size);
3048 oprnds = VEC_alloc (tree, heap, group_size);
3050 alignment_support_scheme = vect_supportable_dr_alignment (first_dr);
3051 gcc_assert (alignment_support_scheme);
3053 /* In case the vectorization factor (VF) is bigger than the number
3054 of elements that we can fit in a vectype (nunits), we have to generate
3055 more than one vector stmt - i.e - we need to "unroll" the
3056 vector stmt by a factor VF/nunits. For more details see documentation in
3057 vect_get_vec_def_for_copy_stmt. */
3059 /* In case of interleaving (non-unit strided access):
3066 We create vectorized stores starting from base address (the access of the
3067 first stmt in the chain (S2 in the above example), when the last store stmt
3068 of the chain (S4) is reached:
3071 VS2: &base + vec_size*1 = vx0
3072 VS3: &base + vec_size*2 = vx1
3073 VS4: &base + vec_size*3 = vx3
3075 Then permutation statements are generated:
3077 VS5: vx5 = VEC_INTERLEAVE_HIGH_EXPR < vx0, vx3 >
3078 VS6: vx6 = VEC_INTERLEAVE_LOW_EXPR < vx0, vx3 >
3081 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3082 (the order of the data-refs in the output of vect_permute_store_chain
3083 corresponds to the order of scalar stmts in the interleaving chain - see
3084 the documentation of vect_permute_store_chain()).
3086 In case of both multiple types and interleaving, above vector stores and
3087 permutation stmts are created for every copy. The result vector stmts are
3088 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3089 STMT_VINFO_RELATED_STMT for the next copies.
3092 prev_stmt_info = NULL;
3093 for (j = 0; j < ncopies; j++)
3102 /* Get vectorized arguments for SLP_NODE. */
3103 vect_get_slp_defs (slp_node, &vec_oprnds, NULL);
3105 vec_oprnd = VEC_index (tree, vec_oprnds, 0);
3109 /* For interleaved stores we collect vectorized defs for all the
3110 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
3111 used as an input to vect_permute_store_chain(), and OPRNDS as
3112 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
3114 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3115 OPRNDS are of size 1. */
3116 next_stmt = first_stmt;
3117 for (i = 0; i < group_size; i++)
3119 /* Since gaps are not supported for interleaved stores,
3120 GROUP_SIZE is the exact number of stmts in the chain.
3121 Therefore, NEXT_STMT can't be NULL_TREE. In case that
3122 there is no interleaving, GROUP_SIZE is 1, and only one
3123 iteration of the loop will be executed. */
3124 gcc_assert (next_stmt
3125 && gimple_assign_single_p (next_stmt));
3126 op = gimple_assign_rhs1 (next_stmt);
3128 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
3130 VEC_quick_push(tree, dr_chain, vec_oprnd);
3131 VEC_quick_push(tree, oprnds, vec_oprnd);
3132 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3136 /* We should have catched mismatched types earlier. */
3137 gcc_assert (useless_type_conversion_p (vectype,
3138 TREE_TYPE (vec_oprnd)));
3139 dataref_ptr = vect_create_data_ref_ptr (first_stmt, NULL, NULL_TREE,
3140 &dummy, &ptr_incr, false,
3142 gcc_assert (bb_vinfo || !inv_p);
3146 /* For interleaved stores we created vectorized defs for all the
3147 defs stored in OPRNDS in the previous iteration (previous copy).
3148 DR_CHAIN is then used as an input to vect_permute_store_chain(),
3149 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
3151 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3152 OPRNDS are of size 1. */
3153 for (i = 0; i < group_size; i++)
3155 op = VEC_index (tree, oprnds, i);
3156 vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def,
3158 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
3159 VEC_replace(tree, dr_chain, i, vec_oprnd);
3160 VEC_replace(tree, oprnds, i, vec_oprnd);
3163 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
3168 result_chain = VEC_alloc (tree, heap, group_size);
3170 if (!vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
3175 next_stmt = first_stmt;
3176 for (i = 0; i < vec_num; i++)
3179 /* Bump the vector pointer. */
3180 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
3184 vec_oprnd = VEC_index (tree, vec_oprnds, i);
3185 else if (strided_store)
3186 /* For strided stores vectorized defs are interleaved in
3187 vect_permute_store_chain(). */
3188 vec_oprnd = VEC_index (tree, result_chain, i);
3190 if (aligned_access_p (first_dr))
3191 data_ref = build_fold_indirect_ref (dataref_ptr);
3194 int mis = DR_MISALIGNMENT (first_dr);
3195 tree tmis = (mis == -1 ? size_zero_node : size_int (mis));
3196 tmis = size_binop (MULT_EXPR, tmis, size_int (BITS_PER_UNIT));
3197 data_ref = build2 (MISALIGNED_INDIRECT_REF, vectype, dataref_ptr, tmis);
3200 /* If accesses through a pointer to vectype do not alias the original
3201 memory reference we have a problem. This should never happen. */
3202 gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref),
3203 get_alias_set (gimple_assign_lhs (stmt))));
3205 /* Arguments are ready. Create the new vector stmt. */
3206 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
3207 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3208 mark_symbols_for_renaming (new_stmt);
3214 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3216 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3218 prev_stmt_info = vinfo_for_stmt (new_stmt);
3219 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3225 VEC_free (tree, heap, dr_chain);
3226 VEC_free (tree, heap, oprnds);
3228 VEC_free (tree, heap, result_chain);
3233 /* vectorizable_load.
3235 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
3237 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3238 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3239 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3242 vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
3243 slp_tree slp_node, slp_instance slp_node_instance)
3246 tree vec_dest = NULL;
3247 tree data_ref = NULL;
3248 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3249 stmt_vec_info prev_stmt_info;
3250 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3251 struct loop *loop = NULL;
3252 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
3253 bool nested_in_vect_loop = false;
3254 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
3255 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3258 gimple new_stmt = NULL;
3260 enum dr_alignment_support alignment_support_scheme;
3261 tree dataref_ptr = NULL_TREE;
3263 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3265 int i, j, group_size;
3266 tree msq = NULL_TREE, lsq;
3267 tree offset = NULL_TREE;
3268 tree realignment_token = NULL_TREE;
3270 VEC(tree,heap) *dr_chain = NULL;
3271 bool strided_load = false;
3275 bool compute_in_loop = false;
3276 struct loop *at_loop;
3278 bool slp = (slp_node != NULL);
3279 bool slp_perm = false;
3280 enum tree_code code;
3281 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3286 loop = LOOP_VINFO_LOOP (loop_vinfo);
3287 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
3288 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
3291 /* FORNOW: multiple types are not supported in basic block SLP. */
3294 /* Multiple types in SLP are handled by creating the appropriate number of
3295 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3300 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3302 gcc_assert (ncopies >= 1);
3304 /* FORNOW. This restriction should be relaxed. */
3305 if (nested_in_vect_loop && ncopies > 1)
3307 if (vect_print_dump_info (REPORT_DETAILS))
3308 fprintf (vect_dump, "multiple types in nested loop.");
3312 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3315 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3318 /* Is vectorizable load? */
3319 if (!is_gimple_assign (stmt))
3322 scalar_dest = gimple_assign_lhs (stmt);
3323 if (TREE_CODE (scalar_dest) != SSA_NAME)
3326 code = gimple_assign_rhs_code (stmt);
3327 if (code != ARRAY_REF
3328 && code != INDIRECT_REF
3329 && code != COMPONENT_REF
3330 && code != IMAGPART_EXPR
3331 && code != REALPART_EXPR)
3334 if (!STMT_VINFO_DATA_REF (stmt_info))
3337 scalar_type = TREE_TYPE (DR_REF (dr));
3338 mode = (int) TYPE_MODE (vectype);
3340 /* FORNOW. In some cases can vectorize even if data-type not supported
3341 (e.g. - data copies). */
3342 if (optab_handler (mov_optab, mode)->insn_code == CODE_FOR_nothing)
3344 if (vect_print_dump_info (REPORT_DETAILS))
3345 fprintf (vect_dump, "Aligned load, but unsupported type.");
3349 /* The vector component type needs to be trivially convertible to the
3350 scalar lhs. This should always be the case. */
3351 if (!useless_type_conversion_p (TREE_TYPE (scalar_dest), TREE_TYPE (vectype)))
3353 if (vect_print_dump_info (REPORT_DETAILS))
3354 fprintf (vect_dump, "??? operands of different types");
3358 /* Check if the load is a part of an interleaving chain. */
3359 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
3361 strided_load = true;
3363 gcc_assert (! nested_in_vect_loop);
3365 /* Check if interleaving is supported. */
3366 if (!vect_strided_load_supported (vectype)
3367 && !PURE_SLP_STMT (stmt_info) && !slp)
3371 if (!vec_stmt) /* transformation not required. */
3373 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
3374 vect_model_load_cost (stmt_info, ncopies, NULL);
3378 if (vect_print_dump_info (REPORT_DETAILS))
3379 fprintf (vect_dump, "transform load.");
3385 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
3386 /* Check if the chain of loads is already vectorized. */
3387 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt)))
3389 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3392 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3393 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3395 /* VEC_NUM is the number of vect stmts to be created for this group. */
3398 strided_load = false;
3399 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3400 if (SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance))
3404 vec_num = group_size;
3406 dr_chain = VEC_alloc (tree, heap, vec_num);
3412 group_size = vec_num = 1;
3415 alignment_support_scheme = vect_supportable_dr_alignment (first_dr);
3416 gcc_assert (alignment_support_scheme);
3418 /* In case the vectorization factor (VF) is bigger than the number
3419 of elements that we can fit in a vectype (nunits), we have to generate
3420 more than one vector stmt - i.e - we need to "unroll" the
3421 vector stmt by a factor VF/nunits. In doing so, we record a pointer
3422 from one copy of the vector stmt to the next, in the field
3423 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
3424 stages to find the correct vector defs to be used when vectorizing
3425 stmts that use the defs of the current stmt. The example below illustrates
3426 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
3427 4 vectorized stmts):
3429 before vectorization:
3430 RELATED_STMT VEC_STMT
3434 step 1: vectorize stmt S1:
3435 We first create the vector stmt VS1_0, and, as usual, record a
3436 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
3437 Next, we create the vector stmt VS1_1, and record a pointer to
3438 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
3439 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
3441 RELATED_STMT VEC_STMT
3442 VS1_0: vx0 = memref0 VS1_1 -
3443 VS1_1: vx1 = memref1 VS1_2 -
3444 VS1_2: vx2 = memref2 VS1_3 -
3445 VS1_3: vx3 = memref3 - -
3446 S1: x = load - VS1_0
3449 See in documentation in vect_get_vec_def_for_stmt_copy for how the
3450 information we recorded in RELATED_STMT field is used to vectorize
3453 /* In case of interleaving (non-unit strided access):
3460 Vectorized loads are created in the order of memory accesses
3461 starting from the access of the first stmt of the chain:
3464 VS2: vx1 = &base + vec_size*1
3465 VS3: vx3 = &base + vec_size*2
3466 VS4: vx4 = &base + vec_size*3
3468 Then permutation statements are generated:
3470 VS5: vx5 = VEC_EXTRACT_EVEN_EXPR < vx0, vx1 >
3471 VS6: vx6 = VEC_EXTRACT_ODD_EXPR < vx0, vx1 >
3474 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3475 (the order of the data-refs in the output of vect_permute_load_chain
3476 corresponds to the order of scalar stmts in the interleaving chain - see
3477 the documentation of vect_permute_load_chain()).
3478 The generation of permutation stmts and recording them in
3479 STMT_VINFO_VEC_STMT is done in vect_transform_strided_load().
3481 In case of both multiple types and interleaving, the vector loads and
3482 permutation stmts above are created for every copy. The result vector stmts
3483 are put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3484 STMT_VINFO_RELATED_STMT for the next copies. */
3486 /* If the data reference is aligned (dr_aligned) or potentially unaligned
3487 on a target that supports unaligned accesses (dr_unaligned_supported)
3488 we generate the following code:
3492 p = p + indx * vectype_size;
3497 Otherwise, the data reference is potentially unaligned on a target that
3498 does not support unaligned accesses (dr_explicit_realign_optimized) -
3499 then generate the following code, in which the data in each iteration is
3500 obtained by two vector loads, one from the previous iteration, and one
3501 from the current iteration:
3503 msq_init = *(floor(p1))
3504 p2 = initial_addr + VS - 1;
3505 realignment_token = call target_builtin;
3508 p2 = p2 + indx * vectype_size
3510 vec_dest = realign_load (msq, lsq, realignment_token)
3515 /* If the misalignment remains the same throughout the execution of the
3516 loop, we can create the init_addr and permutation mask at the loop
3517 preheader. Otherwise, it needs to be created inside the loop.
3518 This can only occur when vectorizing memory accesses in the inner-loop
3519 nested within an outer-loop that is being vectorized. */
3521 if (loop && nested_in_vect_loop_p (loop, stmt)
3522 && (TREE_INT_CST_LOW (DR_STEP (dr))
3523 % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
3525 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
3526 compute_in_loop = true;
3529 if ((alignment_support_scheme == dr_explicit_realign_optimized
3530 || alignment_support_scheme == dr_explicit_realign)
3531 && !compute_in_loop)
3533 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
3534 alignment_support_scheme, NULL_TREE,
3536 if (alignment_support_scheme == dr_explicit_realign_optimized)
3538 phi = SSA_NAME_DEF_STMT (msq);
3539 offset = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
3545 prev_stmt_info = NULL;
3546 for (j = 0; j < ncopies; j++)
3548 /* 1. Create the vector pointer update chain. */
3550 dataref_ptr = vect_create_data_ref_ptr (first_stmt,
3552 &dummy, &ptr_incr, false,
3556 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
3558 for (i = 0; i < vec_num; i++)
3561 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
3564 /* 2. Create the vector-load in the loop. */
3565 switch (alignment_support_scheme)
3568 gcc_assert (aligned_access_p (first_dr));
3569 data_ref = build_fold_indirect_ref (dataref_ptr);
3571 case dr_unaligned_supported:
3573 int mis = DR_MISALIGNMENT (first_dr);
3574 tree tmis = (mis == -1 ? size_zero_node : size_int (mis));
3576 tmis = size_binop (MULT_EXPR, tmis, size_int(BITS_PER_UNIT));
3578 build2 (MISALIGNED_INDIRECT_REF, vectype, dataref_ptr, tmis);
3581 case dr_explicit_realign:
3584 tree vs_minus_1 = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
3586 if (compute_in_loop)
3587 msq = vect_setup_realignment (first_stmt, gsi,
3589 dr_explicit_realign,
3592 data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr);
3593 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3594 new_stmt = gimple_build_assign (vec_dest, data_ref);
3595 new_temp = make_ssa_name (vec_dest, new_stmt);
3596 gimple_assign_set_lhs (new_stmt, new_temp);
3597 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
3598 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
3599 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3602 bump = size_binop (MULT_EXPR, vs_minus_1,
3603 TYPE_SIZE_UNIT (scalar_type));
3604 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
3605 data_ref = build1 (ALIGN_INDIRECT_REF, vectype, ptr);
3608 case dr_explicit_realign_optimized:
3609 data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr);
3614 /* If accesses through a pointer to vectype do not alias the original
3615 memory reference we have a problem. This should never happen. */
3616 gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref),
3617 get_alias_set (gimple_assign_rhs1 (stmt))));
3618 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3619 new_stmt = gimple_build_assign (vec_dest, data_ref);
3620 new_temp = make_ssa_name (vec_dest, new_stmt);
3621 gimple_assign_set_lhs (new_stmt, new_temp);
3622 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3623 mark_symbols_for_renaming (new_stmt);
3625 /* 3. Handle explicit realignment if necessary/supported. Create in
3626 loop: vec_dest = realign_load (msq, lsq, realignment_token) */
3627 if (alignment_support_scheme == dr_explicit_realign_optimized
3628 || alignment_support_scheme == dr_explicit_realign)
3632 lsq = gimple_assign_lhs (new_stmt);
3633 if (!realignment_token)
3634 realignment_token = dataref_ptr;
3635 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3636 tmp = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq,
3638 new_stmt = gimple_build_assign (vec_dest, tmp);
3639 new_temp = make_ssa_name (vec_dest, new_stmt);
3640 gimple_assign_set_lhs (new_stmt, new_temp);
3641 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3643 if (alignment_support_scheme == dr_explicit_realign_optimized)
3646 if (i == vec_num - 1 && j == ncopies - 1)
3647 add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
3653 /* 4. Handle invariant-load. */
3654 if (inv_p && !bb_vinfo)
3656 gcc_assert (!strided_load);
3657 gcc_assert (nested_in_vect_loop_p (loop, stmt));
3662 tree vec_inv, bitpos, bitsize = TYPE_SIZE (scalar_type);
3664 /* CHECKME: bitpos depends on endianess? */
3665 bitpos = bitsize_zero_node;
3666 vec_inv = build3 (BIT_FIELD_REF, scalar_type, new_temp,
3669 vect_create_destination_var (scalar_dest, NULL_TREE);
3670 new_stmt = gimple_build_assign (vec_dest, vec_inv);
3671 new_temp = make_ssa_name (vec_dest, new_stmt);
3672 gimple_assign_set_lhs (new_stmt, new_temp);
3673 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3675 for (k = nunits - 1; k >= 0; --k)
3676 t = tree_cons (NULL_TREE, new_temp, t);
3677 /* FIXME: use build_constructor directly. */
3678 vec_inv = build_constructor_from_list (vectype, t);
3679 new_temp = vect_init_vector (stmt, vec_inv, vectype, gsi);
3680 new_stmt = SSA_NAME_DEF_STMT (new_temp);
3683 gcc_unreachable (); /* FORNOW. */
3686 /* Collect vector loads and later create their permutation in
3687 vect_transform_strided_load (). */
3688 if (strided_load || slp_perm)
3689 VEC_quick_push (tree, dr_chain, new_temp);
3691 /* Store vector loads in the corresponding SLP_NODE. */
3692 if (slp && !slp_perm)
3693 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
3696 if (slp && !slp_perm)
3701 if (!vect_transform_slp_perm_load (stmt, dr_chain, gsi, vf,
3702 slp_node_instance, false))
3704 VEC_free (tree, heap, dr_chain);
3712 if (!vect_transform_strided_load (stmt, dr_chain, group_size, gsi))
3715 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3716 VEC_free (tree, heap, dr_chain);
3717 dr_chain = VEC_alloc (tree, heap, group_size);
3722 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3724 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3725 prev_stmt_info = vinfo_for_stmt (new_stmt);
3731 VEC_free (tree, heap, dr_chain);
3736 /* Function vect_is_simple_cond.
3739 LOOP - the loop that is being vectorized.
3740 COND - Condition that is checked for simple use.
3742 Returns whether a COND can be vectorized. Checks whether
3743 condition operands are supportable using vec_is_simple_use. */
3746 vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
3750 enum vect_def_type dt;
3752 if (!COMPARISON_CLASS_P (cond))
3755 lhs = TREE_OPERAND (cond, 0);
3756 rhs = TREE_OPERAND (cond, 1);
3758 if (TREE_CODE (lhs) == SSA_NAME)
3760 gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
3761 if (!vect_is_simple_use (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
3765 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
3766 && TREE_CODE (lhs) != FIXED_CST)
3769 if (TREE_CODE (rhs) == SSA_NAME)
3771 gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
3772 if (!vect_is_simple_use (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
3776 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
3777 && TREE_CODE (rhs) != FIXED_CST)
3783 /* vectorizable_condition.
3785 Check if STMT is conditional modify expression that can be vectorized.
3786 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3787 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
3790 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
3791 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
3792 else caluse if it is 2).
3794 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3797 vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
3798 gimple *vec_stmt, tree reduc_def, int reduc_index)
3800 tree scalar_dest = NULL_TREE;
3801 tree vec_dest = NULL_TREE;
3802 tree op = NULL_TREE;
3803 tree cond_expr, then_clause, else_clause;
3804 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3805 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3806 tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause;
3807 tree vec_compare, vec_cond_expr;
3809 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3810 enum machine_mode vec_mode;
3812 enum vect_def_type dt;
3813 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3814 int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3815 enum tree_code code;
3817 /* FORNOW: unsupported in basic block SLP. */
3818 gcc_assert (loop_vinfo);
3820 gcc_assert (ncopies >= 1);
3822 return false; /* FORNOW */
3824 if (!STMT_VINFO_RELEVANT_P (stmt_info))
3827 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3828 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3832 /* FORNOW: SLP not supported. */
3833 if (STMT_SLP_TYPE (stmt_info))
3836 /* FORNOW: not yet supported. */
3837 if (STMT_VINFO_LIVE_P (stmt_info))
3839 if (vect_print_dump_info (REPORT_DETAILS))
3840 fprintf (vect_dump, "value used after loop.");
3844 /* Is vectorizable conditional operation? */
3845 if (!is_gimple_assign (stmt))
3848 code = gimple_assign_rhs_code (stmt);
3850 if (code != COND_EXPR)
3853 gcc_assert (gimple_assign_single_p (stmt));
3854 op = gimple_assign_rhs1 (stmt);
3855 cond_expr = TREE_OPERAND (op, 0);
3856 then_clause = TREE_OPERAND (op, 1);
3857 else_clause = TREE_OPERAND (op, 2);
3859 if (!vect_is_simple_cond (cond_expr, loop_vinfo))
3862 /* We do not handle two different vector types for the condition
3864 if (TREE_TYPE (TREE_OPERAND (cond_expr, 0)) != TREE_TYPE (vectype))
3867 if (TREE_CODE (then_clause) == SSA_NAME)
3869 gimple then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
3870 if (!vect_is_simple_use (then_clause, loop_vinfo, NULL,
3871 &then_def_stmt, &def, &dt))
3874 else if (TREE_CODE (then_clause) != INTEGER_CST
3875 && TREE_CODE (then_clause) != REAL_CST
3876 && TREE_CODE (then_clause) != FIXED_CST)
3879 if (TREE_CODE (else_clause) == SSA_NAME)
3881 gimple else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
3882 if (!vect_is_simple_use (else_clause, loop_vinfo, NULL,
3883 &else_def_stmt, &def, &dt))
3886 else if (TREE_CODE (else_clause) != INTEGER_CST
3887 && TREE_CODE (else_clause) != REAL_CST
3888 && TREE_CODE (else_clause) != FIXED_CST)
3892 vec_mode = TYPE_MODE (vectype);
3896 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
3897 return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
3903 scalar_dest = gimple_assign_lhs (stmt);
3904 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3906 /* Handle cond expr. */
3908 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt, NULL);
3910 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt, NULL);
3911 if (reduc_index == 1)
3912 vec_then_clause = reduc_def;
3914 vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL);
3915 if (reduc_index == 2)
3916 vec_else_clause = reduc_def;
3918 vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL);
3920 /* Arguments are ready. Create the new vector stmt. */
3921 vec_compare = build2 (TREE_CODE (cond_expr), vectype,
3922 vec_cond_lhs, vec_cond_rhs);
3923 vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
3924 vec_compare, vec_then_clause, vec_else_clause);
3926 *vec_stmt = gimple_build_assign (vec_dest, vec_cond_expr);
3927 new_temp = make_ssa_name (vec_dest, *vec_stmt);
3928 gimple_assign_set_lhs (*vec_stmt, new_temp);
3929 vect_finish_stmt_generation (stmt, *vec_stmt, gsi);
3935 /* Make sure the statement is vectorizable. */
3938 vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
3940 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3941 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3942 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
3944 HOST_WIDE_INT dummy;
3945 tree scalar_type, vectype;
3947 if (vect_print_dump_info (REPORT_DETAILS))
3949 fprintf (vect_dump, "==> examining statement: ");
3950 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
3953 if (gimple_has_volatile_ops (stmt))
3955 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
3956 fprintf (vect_dump, "not vectorized: stmt has volatile operands");
3961 /* Skip stmts that do not need to be vectorized. In loops this is expected
3963 - the COND_EXPR which is the loop exit condition
3964 - any LABEL_EXPRs in the loop
3965 - computations that are used only for array indexing or loop control.
3966 In basic blocks we only analyze statements that are a part of some SLP
3967 instance, therefore, all the statements are relevant. */
3969 if (!STMT_VINFO_RELEVANT_P (stmt_info)
3970 && !STMT_VINFO_LIVE_P (stmt_info))
3972 if (vect_print_dump_info (REPORT_DETAILS))
3973 fprintf (vect_dump, "irrelevant.");
3978 switch (STMT_VINFO_DEF_TYPE (stmt_info))
3980 case vect_internal_def:
3983 case vect_reduction_def:
3984 case vect_nested_cycle:
3985 gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
3986 || relevance == vect_used_in_outer_by_reduction
3987 || relevance == vect_unused_in_scope));
3990 case vect_induction_def:
3991 case vect_constant_def:
3992 case vect_external_def:
3993 case vect_unknown_def_type:
4000 gcc_assert (PURE_SLP_STMT (stmt_info));
4002 scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
4003 if (vect_print_dump_info (REPORT_DETAILS))
4005 fprintf (vect_dump, "get vectype for scalar type: ");
4006 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4009 vectype = get_vectype_for_scalar_type (scalar_type);
4012 if (vect_print_dump_info (REPORT_DETAILS))
4014 fprintf (vect_dump, "not SLPed: unsupported data-type ");
4015 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4020 if (vect_print_dump_info (REPORT_DETAILS))
4022 fprintf (vect_dump, "vectype: ");
4023 print_generic_expr (vect_dump, vectype, TDF_SLIM);
4026 STMT_VINFO_VECTYPE (stmt_info) = vectype;
4029 if (STMT_VINFO_RELEVANT_P (stmt_info))
4031 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
4032 gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
4033 *need_to_vectorize = true;
4038 && (STMT_VINFO_RELEVANT_P (stmt_info)
4039 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
4040 ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
4041 || vectorizable_type_demotion (stmt, NULL, NULL, NULL)
4042 || vectorizable_conversion (stmt, NULL, NULL, NULL)
4043 || vectorizable_operation (stmt, NULL, NULL, NULL)
4044 || vectorizable_assignment (stmt, NULL, NULL, NULL)
4045 || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
4046 || vectorizable_call (stmt, NULL, NULL)
4047 || vectorizable_store (stmt, NULL, NULL, NULL)
4048 || vectorizable_reduction (stmt, NULL, NULL)
4049 || vectorizable_condition (stmt, NULL, NULL, NULL, 0));
4053 ok = (vectorizable_operation (stmt, NULL, NULL, node)
4054 || vectorizable_assignment (stmt, NULL, NULL, node)
4055 || vectorizable_load (stmt, NULL, NULL, node, NULL)
4056 || vectorizable_store (stmt, NULL, NULL, node));
4061 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4063 fprintf (vect_dump, "not vectorized: relevant stmt not ");
4064 fprintf (vect_dump, "supported: ");
4065 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4074 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
4075 need extra handling, except for vectorizable reductions. */
4076 if (STMT_VINFO_LIVE_P (stmt_info)
4077 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4078 ok = vectorizable_live_operation (stmt, NULL, NULL);
4082 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4084 fprintf (vect_dump, "not vectorized: live stmt not ");
4085 fprintf (vect_dump, "supported: ");
4086 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4092 if (!PURE_SLP_STMT (stmt_info))
4094 /* Groups of strided accesses whose size is not a power of 2 are not
4095 vectorizable yet using loop-vectorization. Therefore, if this stmt
4096 feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
4097 loop-based vectorized), the loop cannot be vectorized. */
4098 if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
4099 && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
4100 DR_GROUP_FIRST_DR (stmt_info)))) == -1)
4102 if (vect_print_dump_info (REPORT_DETAILS))
4104 fprintf (vect_dump, "not vectorized: the size of group "
4105 "of strided accesses is not a power of 2");
4106 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4117 /* Function vect_transform_stmt.
4119 Create a vectorized stmt to replace STMT, and insert it at BSI. */
4122 vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi,
4123 bool *strided_store, slp_tree slp_node,
4124 slp_instance slp_node_instance)
4126 bool is_store = false;
4127 gimple vec_stmt = NULL;
4128 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4129 gimple orig_stmt_in_pattern;
4132 switch (STMT_VINFO_TYPE (stmt_info))
4134 case type_demotion_vec_info_type:
4135 done = vectorizable_type_demotion (stmt, gsi, &vec_stmt, slp_node);
4139 case type_promotion_vec_info_type:
4140 done = vectorizable_type_promotion (stmt, gsi, &vec_stmt, slp_node);
4144 case type_conversion_vec_info_type:
4145 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
4149 case induc_vec_info_type:
4150 gcc_assert (!slp_node);
4151 done = vectorizable_induction (stmt, gsi, &vec_stmt);
4155 case op_vec_info_type:
4156 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
4160 case assignment_vec_info_type:
4161 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
4165 case load_vec_info_type:
4166 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
4171 case store_vec_info_type:
4172 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
4174 if (STMT_VINFO_STRIDED_ACCESS (stmt_info) && !slp_node)
4176 /* In case of interleaving, the whole chain is vectorized when the
4177 last store in the chain is reached. Store stmts before the last
4178 one are skipped, and there vec_stmt_info shouldn't be freed
4180 *strided_store = true;
4181 if (STMT_VINFO_VEC_STMT (stmt_info))
4188 case condition_vec_info_type:
4189 gcc_assert (!slp_node);
4190 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0);
4194 case call_vec_info_type:
4195 gcc_assert (!slp_node);
4196 done = vectorizable_call (stmt, gsi, &vec_stmt);
4199 case reduc_vec_info_type:
4200 gcc_assert (!slp_node);
4201 done = vectorizable_reduction (stmt, gsi, &vec_stmt);
4206 if (!STMT_VINFO_LIVE_P (stmt_info))
4208 if (vect_print_dump_info (REPORT_DETAILS))
4209 fprintf (vect_dump, "stmt not supported.");
4214 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
4215 is being vectorized, but outside the immediately enclosing loop. */
4217 && STMT_VINFO_LOOP_VINFO (stmt_info)
4218 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
4219 STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
4220 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
4221 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
4222 || STMT_VINFO_RELEVANT (stmt_info) ==
4223 vect_used_in_outer_by_reduction))
4225 struct loop *innerloop = LOOP_VINFO_LOOP (
4226 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
4227 imm_use_iterator imm_iter;
4228 use_operand_p use_p;
4232 if (vect_print_dump_info (REPORT_DETAILS))
4233 fprintf (vect_dump, "Record the vdef for outer-loop vectorization.");
4235 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
4236 (to be used when vectorizing outer-loop stmts that use the DEF of
4238 if (gimple_code (stmt) == GIMPLE_PHI)
4239 scalar_dest = PHI_RESULT (stmt);
4241 scalar_dest = gimple_assign_lhs (stmt);
4243 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
4245 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
4247 exit_phi = USE_STMT (use_p);
4248 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
4253 /* Handle stmts whose DEF is used outside the loop-nest that is
4254 being vectorized. */
4255 if (STMT_VINFO_LIVE_P (stmt_info)
4256 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4258 done = vectorizable_live_operation (stmt, gsi, &vec_stmt);
4264 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
4265 orig_stmt_in_pattern = STMT_VINFO_RELATED_STMT (stmt_info);
4266 if (orig_stmt_in_pattern)
4268 stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt_in_pattern);
4269 /* STMT was inserted by the vectorizer to replace a computation idiom.
4270 ORIG_STMT_IN_PATTERN is a stmt in the original sequence that
4271 computed this idiom. We need to record a pointer to VEC_STMT in
4272 the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the
4273 documentation of vect_pattern_recog. */
4274 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
4276 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
4277 STMT_VINFO_VEC_STMT (stmt_vinfo) = vec_stmt;
4286 /* Remove a group of stores (for SLP or interleaving), free their
4290 vect_remove_stores (gimple first_stmt)
4292 gimple next = first_stmt;
4294 gimple_stmt_iterator next_si;
4298 /* Free the attached stmt_vec_info and remove the stmt. */
4299 next_si = gsi_for_stmt (next);
4300 gsi_remove (&next_si, true);
4301 tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
4302 free_stmt_vec_info (next);
4308 /* Function new_stmt_vec_info.
4310 Create and initialize a new stmt_vec_info struct for STMT. */
4313 new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
4314 bb_vec_info bb_vinfo)
4317 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
4319 STMT_VINFO_TYPE (res) = undef_vec_info_type;
4320 STMT_VINFO_STMT (res) = stmt;
4321 STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
4322 STMT_VINFO_BB_VINFO (res) = bb_vinfo;
4323 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
4324 STMT_VINFO_LIVE_P (res) = false;
4325 STMT_VINFO_VECTYPE (res) = NULL;
4326 STMT_VINFO_VEC_STMT (res) = NULL;
4327 STMT_VINFO_IN_PATTERN_P (res) = false;
4328 STMT_VINFO_RELATED_STMT (res) = NULL;
4329 STMT_VINFO_DATA_REF (res) = NULL;
4331 STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
4332 STMT_VINFO_DR_OFFSET (res) = NULL;
4333 STMT_VINFO_DR_INIT (res) = NULL;
4334 STMT_VINFO_DR_STEP (res) = NULL;
4335 STMT_VINFO_DR_ALIGNED_TO (res) = NULL;
4337 if (gimple_code (stmt) == GIMPLE_PHI
4338 && is_loop_header_bb_p (gimple_bb (stmt)))
4339 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
4341 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
4343 STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5);
4344 STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0;
4345 STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0;
4346 STMT_SLP_TYPE (res) = loop_vect;
4347 DR_GROUP_FIRST_DR (res) = NULL;
4348 DR_GROUP_NEXT_DR (res) = NULL;
4349 DR_GROUP_SIZE (res) = 0;
4350 DR_GROUP_STORE_COUNT (res) = 0;
4351 DR_GROUP_GAP (res) = 0;
4352 DR_GROUP_SAME_DR_STMT (res) = NULL;
4353 DR_GROUP_READ_WRITE_DEPENDENCE (res) = false;
4359 /* Create a hash table for stmt_vec_info. */
4362 init_stmt_vec_info_vec (void)
4364 gcc_assert (!stmt_vec_info_vec);
4365 stmt_vec_info_vec = VEC_alloc (vec_void_p, heap, 50);
4369 /* Free hash table for stmt_vec_info. */
4372 free_stmt_vec_info_vec (void)
4374 gcc_assert (stmt_vec_info_vec);
4375 VEC_free (vec_void_p, heap, stmt_vec_info_vec);
4379 /* Free stmt vectorization related info. */
4382 free_stmt_vec_info (gimple stmt)
4384 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4389 VEC_free (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmt_info));
4390 set_vinfo_for_stmt (stmt, NULL);
4395 /* Function get_vectype_for_scalar_type.
4397 Returns the vector type corresponding to SCALAR_TYPE as supported
4401 get_vectype_for_scalar_type (tree scalar_type)
4403 enum machine_mode inner_mode = TYPE_MODE (scalar_type);
4404 int nbytes = GET_MODE_SIZE (inner_mode);
4408 if (nbytes == 0 || nbytes >= UNITS_PER_SIMD_WORD (inner_mode))
4411 /* FORNOW: Only a single vector size per mode (UNITS_PER_SIMD_WORD)
4413 nunits = UNITS_PER_SIMD_WORD (inner_mode) / nbytes;
4415 vectype = build_vector_type (scalar_type, nunits);
4416 if (vect_print_dump_info (REPORT_DETAILS))
4418 fprintf (vect_dump, "get vectype with %d units of type ", nunits);
4419 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4425 if (vect_print_dump_info (REPORT_DETAILS))
4427 fprintf (vect_dump, "vectype: ");
4428 print_generic_expr (vect_dump, vectype, TDF_SLIM);
4431 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
4432 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
4434 if (vect_print_dump_info (REPORT_DETAILS))
4435 fprintf (vect_dump, "mode not supported by target.");
4442 /* Function vect_is_simple_use.
4445 LOOP_VINFO - the vect info of the loop that is being vectorized.
4446 BB_VINFO - the vect info of the basic block that is being vectorized.
4447 OPERAND - operand of a stmt in the loop or bb.
4448 DEF - the defining stmt in case OPERAND is an SSA_NAME.
4450 Returns whether a stmt with OPERAND can be vectorized.
4451 For loops, supportable operands are constants, loop invariants, and operands
4452 that are defined by the current iteration of the loop. Unsupportable
4453 operands are those that are defined by a previous iteration of the loop (as
4454 is the case in reduction/induction computations).
4455 For basic blocks, supportable operands are constants and bb invariants.
4456 For now, operands defined outside the basic block are not supported. */
4459 vect_is_simple_use (tree operand, loop_vec_info loop_vinfo,
4460 bb_vec_info bb_vinfo, gimple *def_stmt,
4461 tree *def, enum vect_def_type *dt)
4464 stmt_vec_info stmt_vinfo;
4465 struct loop *loop = NULL;
4468 loop = LOOP_VINFO_LOOP (loop_vinfo);
4473 if (vect_print_dump_info (REPORT_DETAILS))
4475 fprintf (vect_dump, "vect_is_simple_use: operand ");
4476 print_generic_expr (vect_dump, operand, TDF_SLIM);
4479 if (TREE_CODE (operand) == INTEGER_CST || TREE_CODE (operand) == REAL_CST)
4481 *dt = vect_constant_def;
4485 if (is_gimple_min_invariant (operand))
4488 *dt = vect_external_def;
4492 if (TREE_CODE (operand) == PAREN_EXPR)
4494 if (vect_print_dump_info (REPORT_DETAILS))
4495 fprintf (vect_dump, "non-associatable copy.");
4496 operand = TREE_OPERAND (operand, 0);
4499 if (TREE_CODE (operand) != SSA_NAME)
4501 if (vect_print_dump_info (REPORT_DETAILS))
4502 fprintf (vect_dump, "not ssa-name.");
4506 *def_stmt = SSA_NAME_DEF_STMT (operand);
4507 if (*def_stmt == NULL)
4509 if (vect_print_dump_info (REPORT_DETAILS))
4510 fprintf (vect_dump, "no def_stmt.");
4514 if (vect_print_dump_info (REPORT_DETAILS))
4516 fprintf (vect_dump, "def_stmt: ");
4517 print_gimple_stmt (vect_dump, *def_stmt, 0, TDF_SLIM);
4520 /* Empty stmt is expected only in case of a function argument.
4521 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
4522 if (gimple_nop_p (*def_stmt))
4525 *dt = vect_external_def;
4529 bb = gimple_bb (*def_stmt);
4531 if ((loop && !flow_bb_inside_loop_p (loop, bb))
4532 || (!loop && bb != BB_VINFO_BB (bb_vinfo))
4533 || (!loop && gimple_code (*def_stmt) == GIMPLE_PHI))
4534 *dt = vect_external_def;
4537 stmt_vinfo = vinfo_for_stmt (*def_stmt);
4538 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
4541 if (*dt == vect_unknown_def_type)
4543 if (vect_print_dump_info (REPORT_DETAILS))
4544 fprintf (vect_dump, "Unsupported pattern.");
4548 if (vect_print_dump_info (REPORT_DETAILS))
4549 fprintf (vect_dump, "type of def: %d.",*dt);
4551 switch (gimple_code (*def_stmt))
4554 *def = gimple_phi_result (*def_stmt);
4558 *def = gimple_assign_lhs (*def_stmt);
4562 *def = gimple_call_lhs (*def_stmt);
4567 if (vect_print_dump_info (REPORT_DETAILS))
4568 fprintf (vect_dump, "unsupported defining stmt: ");
4576 /* Function supportable_widening_operation
4578 Check whether an operation represented by the code CODE is a
4579 widening operation that is supported by the target platform in
4580 vector form (i.e., when operating on arguments of type VECTYPE).
4582 Widening operations we currently support are NOP (CONVERT), FLOAT
4583 and WIDEN_MULT. This function checks if these operations are supported
4584 by the target platform either directly (via vector tree-codes), or via
4588 - CODE1 and CODE2 are codes of vector operations to be used when
4589 vectorizing the operation, if available.
4590 - DECL1 and DECL2 are decls of target builtin functions to be used
4591 when vectorizing the operation, if available. In this case,
4592 CODE1 and CODE2 are CALL_EXPR.
4593 - MULTI_STEP_CVT determines the number of required intermediate steps in
4594 case of multi-step conversion (like char->short->int - in that case
4595 MULTI_STEP_CVT will be 1).
4596 - INTERM_TYPES contains the intermediate type required to perform the
4597 widening operation (short in the above example). */
4600 supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype,
4601 tree *decl1, tree *decl2,
4602 enum tree_code *code1, enum tree_code *code2,
4603 int *multi_step_cvt,
4604 VEC (tree, heap) **interm_types)
4606 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4607 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
4608 struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
4610 enum machine_mode vec_mode;
4611 enum insn_code icode1, icode2;
4612 optab optab1, optab2;
4613 tree type = gimple_expr_type (stmt);
4614 tree wide_vectype = get_vectype_for_scalar_type (type);
4615 enum tree_code c1, c2;
4617 /* The result of a vectorized widening operation usually requires two vectors
4618 (because the widened results do not fit int one vector). The generated
4619 vector results would normally be expected to be generated in the same
4620 order as in the original scalar computation, i.e. if 8 results are
4621 generated in each vector iteration, they are to be organized as follows:
4622 vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
4624 However, in the special case that the result of the widening operation is
4625 used in a reduction computation only, the order doesn't matter (because
4626 when vectorizing a reduction we change the order of the computation).
4627 Some targets can take advantage of this and generate more efficient code.
4628 For example, targets like Altivec, that support widen_mult using a sequence
4629 of {mult_even,mult_odd} generate the following vectors:
4630 vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8].
4632 When vectorizing outer-loops, we execute the inner-loop sequentially
4633 (each vectorized inner-loop iteration contributes to VF outer-loop
4634 iterations in parallel). We therefore don't allow to change the order
4635 of the computation in the inner-loop during outer-loop vectorization. */
4637 if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
4638 && !nested_in_vect_loop_p (vect_loop, stmt))
4644 && code == WIDEN_MULT_EXPR
4645 && targetm.vectorize.builtin_mul_widen_even
4646 && targetm.vectorize.builtin_mul_widen_even (vectype)
4647 && targetm.vectorize.builtin_mul_widen_odd
4648 && targetm.vectorize.builtin_mul_widen_odd (vectype))
4650 if (vect_print_dump_info (REPORT_DETAILS))
4651 fprintf (vect_dump, "Unordered widening operation detected.");
4653 *code1 = *code2 = CALL_EXPR;
4654 *decl1 = targetm.vectorize.builtin_mul_widen_even (vectype);
4655 *decl2 = targetm.vectorize.builtin_mul_widen_odd (vectype);
4661 case WIDEN_MULT_EXPR:
4662 if (BYTES_BIG_ENDIAN)
4664 c1 = VEC_WIDEN_MULT_HI_EXPR;
4665 c2 = VEC_WIDEN_MULT_LO_EXPR;
4669 c2 = VEC_WIDEN_MULT_HI_EXPR;
4670 c1 = VEC_WIDEN_MULT_LO_EXPR;
4675 if (BYTES_BIG_ENDIAN)
4677 c1 = VEC_UNPACK_HI_EXPR;
4678 c2 = VEC_UNPACK_LO_EXPR;
4682 c2 = VEC_UNPACK_HI_EXPR;
4683 c1 = VEC_UNPACK_LO_EXPR;
4688 if (BYTES_BIG_ENDIAN)
4690 c1 = VEC_UNPACK_FLOAT_HI_EXPR;
4691 c2 = VEC_UNPACK_FLOAT_LO_EXPR;
4695 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
4696 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
4700 case FIX_TRUNC_EXPR:
4701 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
4702 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
4703 computing the operation. */
4710 if (code == FIX_TRUNC_EXPR)
4712 /* The signedness is determined from output operand. */
4713 optab1 = optab_for_tree_code (c1, type, optab_default);
4714 optab2 = optab_for_tree_code (c2, type, optab_default);
4718 optab1 = optab_for_tree_code (c1, vectype, optab_default);
4719 optab2 = optab_for_tree_code (c2, vectype, optab_default);
4722 if (!optab1 || !optab2)
4725 vec_mode = TYPE_MODE (vectype);
4726 if ((icode1 = optab_handler (optab1, vec_mode)->insn_code) == CODE_FOR_nothing
4727 || (icode2 = optab_handler (optab2, vec_mode)->insn_code)
4728 == CODE_FOR_nothing)
4731 /* Check if it's a multi-step conversion that can be done using intermediate
4733 if (insn_data[icode1].operand[0].mode != TYPE_MODE (wide_vectype)
4734 || insn_data[icode2].operand[0].mode != TYPE_MODE (wide_vectype))
4737 tree prev_type = vectype, intermediate_type;
4738 enum machine_mode intermediate_mode, prev_mode = vec_mode;
4739 optab optab3, optab4;
4741 if (!CONVERT_EXPR_CODE_P (code))
4747 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
4748 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
4749 to get to NARROW_VECTYPE, and fail if we do not. */
4750 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
4751 for (i = 0; i < 3; i++)
4753 intermediate_mode = insn_data[icode1].operand[0].mode;
4754 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
4755 TYPE_UNSIGNED (prev_type));
4756 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
4757 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
4759 if (!optab3 || !optab4
4760 || (icode1 = optab1->handlers[(int) prev_mode].insn_code)
4762 || insn_data[icode1].operand[0].mode != intermediate_mode
4763 || (icode2 = optab2->handlers[(int) prev_mode].insn_code)
4765 || insn_data[icode2].operand[0].mode != intermediate_mode
4766 || (icode1 = optab3->handlers[(int) intermediate_mode].insn_code)
4768 || (icode2 = optab4->handlers[(int) intermediate_mode].insn_code)
4769 == CODE_FOR_nothing)
4772 VEC_quick_push (tree, *interm_types, intermediate_type);
4773 (*multi_step_cvt)++;
4775 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
4776 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
4779 prev_type = intermediate_type;
4780 prev_mode = intermediate_mode;
4792 /* Function supportable_narrowing_operation
4794 Check whether an operation represented by the code CODE is a
4795 narrowing operation that is supported by the target platform in
4796 vector form (i.e., when operating on arguments of type VECTYPE).
4798 Narrowing operations we currently support are NOP (CONVERT) and
4799 FIX_TRUNC. This function checks if these operations are supported by
4800 the target platform directly via vector tree-codes.
4803 - CODE1 is the code of a vector operation to be used when
4804 vectorizing the operation, if available.
4805 - MULTI_STEP_CVT determines the number of required intermediate steps in
4806 case of multi-step conversion (like int->short->char - in that case
4807 MULTI_STEP_CVT will be 1).
4808 - INTERM_TYPES contains the intermediate type required to perform the
4809 narrowing operation (short in the above example). */
4812 supportable_narrowing_operation (enum tree_code code,
4813 const_gimple stmt, tree vectype,
4814 enum tree_code *code1, int *multi_step_cvt,
4815 VEC (tree, heap) **interm_types)
4817 enum machine_mode vec_mode;
4818 enum insn_code icode1;
4819 optab optab1, interm_optab;
4820 tree type = gimple_expr_type (stmt);
4821 tree narrow_vectype = get_vectype_for_scalar_type (type);
4823 tree intermediate_type, prev_type;
4829 c1 = VEC_PACK_TRUNC_EXPR;
4832 case FIX_TRUNC_EXPR:
4833 c1 = VEC_PACK_FIX_TRUNC_EXPR;
4837 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
4838 tree code and optabs used for computing the operation. */
4845 if (code == FIX_TRUNC_EXPR)
4846 /* The signedness is determined from output operand. */
4847 optab1 = optab_for_tree_code (c1, type, optab_default);
4849 optab1 = optab_for_tree_code (c1, vectype, optab_default);
4854 vec_mode = TYPE_MODE (vectype);
4855 if ((icode1 = optab_handler (optab1, vec_mode)->insn_code)
4856 == CODE_FOR_nothing)
4859 /* Check if it's a multi-step conversion that can be done using intermediate
4861 if (insn_data[icode1].operand[0].mode != TYPE_MODE (narrow_vectype))
4863 enum machine_mode intermediate_mode, prev_mode = vec_mode;
4866 prev_type = vectype;
4867 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
4868 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
4869 to get to NARROW_VECTYPE, and fail if we do not. */
4870 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
4871 for (i = 0; i < 3; i++)
4873 intermediate_mode = insn_data[icode1].operand[0].mode;
4874 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
4875 TYPE_UNSIGNED (prev_type));
4876 interm_optab = optab_for_tree_code (c1, intermediate_type,
4879 || (icode1 = optab1->handlers[(int) prev_mode].insn_code)
4881 || insn_data[icode1].operand[0].mode != intermediate_mode
4883 = interm_optab->handlers[(int) intermediate_mode].insn_code)
4884 == CODE_FOR_nothing)
4887 VEC_quick_push (tree, *interm_types, intermediate_type);
4888 (*multi_step_cvt)++;
4890 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
4893 prev_type = intermediate_type;
4894 prev_mode = intermediate_mode;