1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "tree-pretty-print.h"
32 #include "gimple-pretty-print.h"
33 #include "tree-flow.h"
34 #include "tree-dump.h"
36 #include "cfglayout.h"
40 #include "diagnostic-core.h"
42 #include "tree-vectorizer.h"
43 #include "langhooks.h"
46 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
48 /* Function vect_mark_relevant.
50 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
53 vect_mark_relevant (VEC(gimple,heap) **worklist, gimple stmt,
54 enum vect_relevant relevant, bool live_p)
56 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
57 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
58 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
60 if (vect_print_dump_info (REPORT_DETAILS))
61 fprintf (vect_dump, "mark relevant %d, live %d.", relevant, live_p);
63 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
67 /* This is the last stmt in a sequence that was detected as a
68 pattern that can potentially be vectorized. Don't mark the stmt
69 as relevant/live because it's not going to be vectorized.
70 Instead mark the pattern-stmt that replaces it. */
72 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
74 if (vect_print_dump_info (REPORT_DETAILS))
75 fprintf (vect_dump, "last stmt in pattern. don't mark relevant/live.");
76 stmt_info = vinfo_for_stmt (pattern_stmt);
77 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
78 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
79 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
83 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
84 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
85 STMT_VINFO_RELEVANT (stmt_info) = relevant;
87 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
88 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
90 if (vect_print_dump_info (REPORT_DETAILS))
91 fprintf (vect_dump, "already marked relevant/live.");
95 VEC_safe_push (gimple, heap, *worklist, stmt);
99 /* Function vect_stmt_relevant_p.
101 Return true if STMT in loop that is represented by LOOP_VINFO is
102 "relevant for vectorization".
104 A stmt is considered "relevant for vectorization" if:
105 - it has uses outside the loop.
106 - it has vdefs (it alters memory).
107 - control stmts in the loop (except for the exit condition).
109 CHECKME: what other side effects would the vectorizer allow? */
112 vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo,
113 enum vect_relevant *relevant, bool *live_p)
115 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
117 imm_use_iterator imm_iter;
121 *relevant = vect_unused_in_scope;
124 /* cond stmt other than loop exit cond. */
125 if (is_ctrl_stmt (stmt)
126 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
127 != loop_exit_ctrl_vec_info_type)
128 *relevant = vect_used_in_scope;
130 /* changing memory. */
131 if (gimple_code (stmt) != GIMPLE_PHI)
132 if (gimple_vdef (stmt))
134 if (vect_print_dump_info (REPORT_DETAILS))
135 fprintf (vect_dump, "vec_stmt_relevant_p: stmt has vdefs.");
136 *relevant = vect_used_in_scope;
139 /* uses outside the loop. */
140 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
142 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
144 basic_block bb = gimple_bb (USE_STMT (use_p));
145 if (!flow_bb_inside_loop_p (loop, bb))
147 if (vect_print_dump_info (REPORT_DETAILS))
148 fprintf (vect_dump, "vec_stmt_relevant_p: used out of loop.");
150 if (is_gimple_debug (USE_STMT (use_p)))
153 /* We expect all such uses to be in the loop exit phis
154 (because of loop closed form) */
155 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
156 gcc_assert (bb == single_exit (loop)->dest);
163 return (*live_p || *relevant);
167 /* Function exist_non_indexing_operands_for_use_p
169 USE is one of the uses attached to STMT. Check if USE is
170 used in STMT for anything other than indexing an array. */
173 exist_non_indexing_operands_for_use_p (tree use, gimple stmt)
176 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
178 /* USE corresponds to some operand in STMT. If there is no data
179 reference in STMT, then any operand that corresponds to USE
180 is not indexing an array. */
181 if (!STMT_VINFO_DATA_REF (stmt_info))
184 /* STMT has a data_ref. FORNOW this means that its of one of
188 (This should have been verified in analyze_data_refs).
190 'var' in the second case corresponds to a def, not a use,
191 so USE cannot correspond to any operands that are not used
194 Therefore, all we need to check is if STMT falls into the
195 first case, and whether var corresponds to USE. */
197 if (!gimple_assign_copy_p (stmt))
199 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
201 operand = gimple_assign_rhs1 (stmt);
202 if (TREE_CODE (operand) != SSA_NAME)
213 Function process_use.
216 - a USE in STMT in a loop represented by LOOP_VINFO
217 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
218 that defined USE. This is done by calling mark_relevant and passing it
219 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
222 Generally, LIVE_P and RELEVANT are used to define the liveness and
223 relevance info of the DEF_STMT of this USE:
224 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
225 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
227 - case 1: If USE is used only for address computations (e.g. array indexing),
228 which does not need to be directly vectorized, then the liveness/relevance
229 of the respective DEF_STMT is left unchanged.
230 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
231 skip DEF_STMT cause it had already been processed.
232 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
233 be modified accordingly.
235 Return true if everything is as expected. Return false otherwise. */
238 process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
239 enum vect_relevant relevant, VEC(gimple,heap) **worklist)
241 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
242 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
243 stmt_vec_info dstmt_vinfo;
244 basic_block bb, def_bb;
247 enum vect_def_type dt;
249 /* case 1: we are only interested in uses that need to be vectorized. Uses
250 that are used for address computation are not considered relevant. */
251 if (!exist_non_indexing_operands_for_use_p (use, stmt))
254 if (!vect_is_simple_use (use, loop_vinfo, NULL, &def_stmt, &def, &dt))
256 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
257 fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
261 if (!def_stmt || gimple_nop_p (def_stmt))
264 def_bb = gimple_bb (def_stmt);
265 if (!flow_bb_inside_loop_p (loop, def_bb))
267 if (vect_print_dump_info (REPORT_DETAILS))
268 fprintf (vect_dump, "def_stmt is out of loop.");
272 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
273 DEF_STMT must have already been processed, because this should be the
274 only way that STMT, which is a reduction-phi, was put in the worklist,
275 as there should be no other uses for DEF_STMT in the loop. So we just
276 check that everything is as expected, and we are done. */
277 dstmt_vinfo = vinfo_for_stmt (def_stmt);
278 bb = gimple_bb (stmt);
279 if (gimple_code (stmt) == GIMPLE_PHI
280 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
281 && gimple_code (def_stmt) != GIMPLE_PHI
282 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
283 && bb->loop_father == def_bb->loop_father)
285 if (vect_print_dump_info (REPORT_DETAILS))
286 fprintf (vect_dump, "reduc-stmt defining reduc-phi in the same nest.");
287 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
288 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
289 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
290 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
291 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
295 /* case 3a: outer-loop stmt defining an inner-loop stmt:
296 outer-loop-header-bb:
302 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
304 if (vect_print_dump_info (REPORT_DETAILS))
305 fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
309 case vect_unused_in_scope:
310 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
311 vect_used_in_scope : vect_unused_in_scope;
314 case vect_used_in_outer_by_reduction:
315 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
316 relevant = vect_used_by_reduction;
319 case vect_used_in_outer:
320 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
321 relevant = vect_used_in_scope;
324 case vect_used_in_scope:
332 /* case 3b: inner-loop stmt defining an outer-loop stmt:
333 outer-loop-header-bb:
337 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
339 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
341 if (vect_print_dump_info (REPORT_DETAILS))
342 fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
346 case vect_unused_in_scope:
347 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
348 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
349 vect_used_in_outer_by_reduction : vect_unused_in_scope;
352 case vect_used_by_reduction:
353 relevant = vect_used_in_outer_by_reduction;
356 case vect_used_in_scope:
357 relevant = vect_used_in_outer;
365 vect_mark_relevant (worklist, def_stmt, relevant, live_p);
370 /* Function vect_mark_stmts_to_be_vectorized.
372 Not all stmts in the loop need to be vectorized. For example:
381 Stmt 1 and 3 do not need to be vectorized, because loop control and
382 addressing of vectorized data-refs are handled differently.
384 This pass detects such stmts. */
387 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
389 VEC(gimple,heap) *worklist;
390 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
391 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
392 unsigned int nbbs = loop->num_nodes;
393 gimple_stmt_iterator si;
396 stmt_vec_info stmt_vinfo;
400 enum vect_relevant relevant, tmp_relevant;
401 enum vect_def_type def_type;
403 if (vect_print_dump_info (REPORT_DETAILS))
404 fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
406 worklist = VEC_alloc (gimple, heap, 64);
408 /* 1. Init worklist. */
409 for (i = 0; i < nbbs; i++)
412 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
415 if (vect_print_dump_info (REPORT_DETAILS))
417 fprintf (vect_dump, "init: phi relevant? ");
418 print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
421 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
422 vect_mark_relevant (&worklist, phi, relevant, live_p);
424 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
426 stmt = gsi_stmt (si);
427 if (vect_print_dump_info (REPORT_DETAILS))
429 fprintf (vect_dump, "init: stmt relevant? ");
430 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
433 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
434 vect_mark_relevant (&worklist, stmt, relevant, live_p);
438 /* 2. Process_worklist */
439 while (VEC_length (gimple, worklist) > 0)
444 stmt = VEC_pop (gimple, worklist);
445 if (vect_print_dump_info (REPORT_DETAILS))
447 fprintf (vect_dump, "worklist: examine stmt: ");
448 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
451 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
452 (DEF_STMT) as relevant/irrelevant and live/dead according to the
453 liveness and relevance properties of STMT. */
454 stmt_vinfo = vinfo_for_stmt (stmt);
455 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
456 live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
458 /* Generally, the liveness and relevance properties of STMT are
459 propagated as is to the DEF_STMTs of its USEs:
460 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
461 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
463 One exception is when STMT has been identified as defining a reduction
464 variable; in this case we set the liveness/relevance as follows:
466 relevant = vect_used_by_reduction
467 This is because we distinguish between two kinds of relevant stmts -
468 those that are used by a reduction computation, and those that are
469 (also) used by a regular computation. This allows us later on to
470 identify stmts that are used solely by a reduction, and therefore the
471 order of the results that they produce does not have to be kept. */
473 def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
474 tmp_relevant = relevant;
477 case vect_reduction_def:
478 switch (tmp_relevant)
480 case vect_unused_in_scope:
481 relevant = vect_used_by_reduction;
484 case vect_used_by_reduction:
485 if (gimple_code (stmt) == GIMPLE_PHI)
490 if (vect_print_dump_info (REPORT_DETAILS))
491 fprintf (vect_dump, "unsupported use of reduction.");
493 VEC_free (gimple, heap, worklist);
500 case vect_nested_cycle:
501 if (tmp_relevant != vect_unused_in_scope
502 && tmp_relevant != vect_used_in_outer_by_reduction
503 && tmp_relevant != vect_used_in_outer)
505 if (vect_print_dump_info (REPORT_DETAILS))
506 fprintf (vect_dump, "unsupported use of nested cycle.");
508 VEC_free (gimple, heap, worklist);
515 case vect_double_reduction_def:
516 if (tmp_relevant != vect_unused_in_scope
517 && tmp_relevant != vect_used_by_reduction)
519 if (vect_print_dump_info (REPORT_DETAILS))
520 fprintf (vect_dump, "unsupported use of double reduction.");
522 VEC_free (gimple, heap, worklist);
533 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
535 tree op = USE_FROM_PTR (use_p);
536 if (!process_use (stmt, op, loop_vinfo, live_p, relevant, &worklist))
538 VEC_free (gimple, heap, worklist);
542 } /* while worklist */
544 VEC_free (gimple, heap, worklist);
549 /* Get cost by calling cost target builtin. */
552 int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost)
554 tree dummy_type = NULL;
557 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
562 cost_for_stmt (gimple stmt)
564 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
566 switch (STMT_VINFO_TYPE (stmt_info))
568 case load_vec_info_type:
569 return vect_get_stmt_cost (scalar_load);
570 case store_vec_info_type:
571 return vect_get_stmt_cost (scalar_store);
572 case op_vec_info_type:
573 case condition_vec_info_type:
574 case assignment_vec_info_type:
575 case reduc_vec_info_type:
576 case induc_vec_info_type:
577 case type_promotion_vec_info_type:
578 case type_demotion_vec_info_type:
579 case type_conversion_vec_info_type:
580 case call_vec_info_type:
581 return vect_get_stmt_cost (scalar_stmt);
582 case undef_vec_info_type:
588 /* Function vect_model_simple_cost.
590 Models cost for simple operations, i.e. those that only emit ncopies of a
591 single op. Right now, this does not account for multiple insns that could
592 be generated for the single vector op. We will handle that shortly. */
595 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
596 enum vect_def_type *dt, slp_tree slp_node)
599 int inside_cost = 0, outside_cost = 0;
601 /* The SLP costs were already calculated during SLP tree build. */
602 if (PURE_SLP_STMT (stmt_info))
605 inside_cost = ncopies * vect_get_stmt_cost (vector_stmt);
607 /* FORNOW: Assuming maximum 2 args per stmts. */
608 for (i = 0; i < 2; i++)
610 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
611 outside_cost += vect_get_stmt_cost (vector_stmt);
614 if (vect_print_dump_info (REPORT_COST))
615 fprintf (vect_dump, "vect_model_simple_cost: inside_cost = %d, "
616 "outside_cost = %d .", inside_cost, outside_cost);
618 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
619 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
620 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
624 /* Function vect_cost_strided_group_size
626 For strided load or store, return the group_size only if it is the first
627 load or store of a group, else return 1. This ensures that group size is
628 only returned once per group. */
631 vect_cost_strided_group_size (stmt_vec_info stmt_info)
633 gimple first_stmt = DR_GROUP_FIRST_DR (stmt_info);
635 if (first_stmt == STMT_VINFO_STMT (stmt_info))
636 return DR_GROUP_SIZE (stmt_info);
642 /* Function vect_model_store_cost
644 Models cost for stores. In the case of strided accesses, one access
645 has the overhead of the strided access attributed to it. */
648 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
649 enum vect_def_type dt, slp_tree slp_node)
652 unsigned int inside_cost = 0, outside_cost = 0;
653 struct data_reference *first_dr;
656 /* The SLP costs were already calculated during SLP tree build. */
657 if (PURE_SLP_STMT (stmt_info))
660 if (dt == vect_constant_def || dt == vect_external_def)
661 outside_cost = vect_get_stmt_cost (scalar_to_vec);
663 /* Strided access? */
664 if (DR_GROUP_FIRST_DR (stmt_info))
668 first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
673 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
674 group_size = vect_cost_strided_group_size (stmt_info);
677 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
679 /* Not a strided access. */
683 first_dr = STMT_VINFO_DATA_REF (stmt_info);
686 /* Is this an access in a group of stores, which provide strided access?
687 If so, add in the cost of the permutes. */
690 /* Uses a high and low interleave operation for each needed permute. */
691 inside_cost = ncopies * exact_log2(group_size) * group_size
692 * vect_get_stmt_cost (vector_stmt);
694 if (vect_print_dump_info (REPORT_COST))
695 fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .",
700 /* Costs of the stores. */
701 vect_get_store_cost (first_dr, ncopies, &inside_cost);
703 if (vect_print_dump_info (REPORT_COST))
704 fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, "
705 "outside_cost = %d .", inside_cost, outside_cost);
707 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
708 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
709 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
713 /* Calculate cost of DR's memory access. */
715 vect_get_store_cost (struct data_reference *dr, int ncopies,
716 unsigned int *inside_cost)
718 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
720 switch (alignment_support_scheme)
724 *inside_cost += ncopies * vect_get_stmt_cost (vector_store);
726 if (vect_print_dump_info (REPORT_COST))
727 fprintf (vect_dump, "vect_model_store_cost: aligned.");
732 case dr_unaligned_supported:
734 gimple stmt = DR_STMT (dr);
735 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
736 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
738 /* Here, we assign an additional cost for the unaligned store. */
739 *inside_cost += ncopies
740 * targetm.vectorize.builtin_vectorization_cost (unaligned_store,
741 vectype, DR_MISALIGNMENT (dr));
743 if (vect_print_dump_info (REPORT_COST))
744 fprintf (vect_dump, "vect_model_store_cost: unaligned supported by "
756 /* Function vect_model_load_cost
758 Models cost for loads. In the case of strided accesses, the last access
759 has the overhead of the strided access attributed to it. Since unaligned
760 accesses are supported for loads, we also account for the costs of the
761 access scheme chosen. */
764 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node)
769 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
770 unsigned int inside_cost = 0, outside_cost = 0;
772 /* The SLP costs were already calculated during SLP tree build. */
773 if (PURE_SLP_STMT (stmt_info))
776 /* Strided accesses? */
777 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
778 if (first_stmt && !slp_node)
780 group_size = vect_cost_strided_group_size (stmt_info);
781 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
783 /* Not a strided access. */
790 /* Is this an access in a group of loads providing strided access?
791 If so, add in the cost of the permutes. */
794 /* Uses an even and odd extract operations for each needed permute. */
795 inside_cost = ncopies * exact_log2(group_size) * group_size
796 * vect_get_stmt_cost (vector_stmt);
798 if (vect_print_dump_info (REPORT_COST))
799 fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .",
803 /* The loads themselves. */
804 vect_get_load_cost (first_dr, ncopies,
805 ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node),
806 &inside_cost, &outside_cost);
808 if (vect_print_dump_info (REPORT_COST))
809 fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, "
810 "outside_cost = %d .", inside_cost, outside_cost);
812 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
813 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
814 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
818 /* Calculate cost of DR's memory access. */
820 vect_get_load_cost (struct data_reference *dr, int ncopies,
821 bool add_realign_cost, unsigned int *inside_cost,
822 unsigned int *outside_cost)
824 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
826 switch (alignment_support_scheme)
830 *inside_cost += ncopies * vect_get_stmt_cost (vector_load);
832 if (vect_print_dump_info (REPORT_COST))
833 fprintf (vect_dump, "vect_model_load_cost: aligned.");
837 case dr_unaligned_supported:
839 gimple stmt = DR_STMT (dr);
840 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
841 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
843 /* Here, we assign an additional cost for the unaligned load. */
844 *inside_cost += ncopies
845 * targetm.vectorize.builtin_vectorization_cost (unaligned_load,
846 vectype, DR_MISALIGNMENT (dr));
847 if (vect_print_dump_info (REPORT_COST))
848 fprintf (vect_dump, "vect_model_load_cost: unaligned supported by "
853 case dr_explicit_realign:
855 *inside_cost += ncopies * (2 * vect_get_stmt_cost (vector_load)
856 + vect_get_stmt_cost (vector_stmt));
858 /* FIXME: If the misalignment remains fixed across the iterations of
859 the containing loop, the following cost should be added to the
861 if (targetm.vectorize.builtin_mask_for_load)
862 *inside_cost += vect_get_stmt_cost (vector_stmt);
866 case dr_explicit_realign_optimized:
868 if (vect_print_dump_info (REPORT_COST))
869 fprintf (vect_dump, "vect_model_load_cost: unaligned software "
872 /* Unaligned software pipeline has a load of an address, an initial
873 load, and possibly a mask operation to "prime" the loop. However,
874 if this is an access in a group of loads, which provide strided
875 access, then the above cost should only be considered for one
876 access in the group. Inside the loop, there is a load op
877 and a realignment op. */
879 if (add_realign_cost)
881 *outside_cost = 2 * vect_get_stmt_cost (vector_stmt);
882 if (targetm.vectorize.builtin_mask_for_load)
883 *outside_cost += vect_get_stmt_cost (vector_stmt);
886 *inside_cost += ncopies * (vect_get_stmt_cost (vector_load)
887 + vect_get_stmt_cost (vector_stmt));
897 /* Function vect_init_vector.
899 Insert a new stmt (INIT_STMT) that initializes a new vector variable with
900 the vector elements of VECTOR_VAR. Place the initialization at BSI if it
901 is not NULL. Otherwise, place the initialization at the loop preheader.
902 Return the DEF of INIT_STMT.
903 It will be used in the vectorization of STMT. */
906 vect_init_vector (gimple stmt, tree vector_var, tree vector_type,
907 gimple_stmt_iterator *gsi)
909 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
917 new_var = vect_get_new_vect_var (vector_type, vect_simple_var, "cst_");
918 add_referenced_var (new_var);
919 init_stmt = gimple_build_assign (new_var, vector_var);
920 new_temp = make_ssa_name (new_var, init_stmt);
921 gimple_assign_set_lhs (init_stmt, new_temp);
924 vect_finish_stmt_generation (stmt, init_stmt, gsi);
927 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
931 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
933 if (nested_in_vect_loop_p (loop, stmt))
936 pe = loop_preheader_edge (loop);
937 new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
938 gcc_assert (!new_bb);
942 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
944 gimple_stmt_iterator gsi_bb_start;
946 gcc_assert (bb_vinfo);
947 bb = BB_VINFO_BB (bb_vinfo);
948 gsi_bb_start = gsi_after_labels (bb);
949 gsi_insert_before (&gsi_bb_start, init_stmt, GSI_SAME_STMT);
953 if (vect_print_dump_info (REPORT_DETAILS))
955 fprintf (vect_dump, "created new init_stmt: ");
956 print_gimple_stmt (vect_dump, init_stmt, 0, TDF_SLIM);
959 vec_oprnd = gimple_assign_lhs (init_stmt);
964 /* Function vect_get_vec_def_for_operand.
966 OP is an operand in STMT. This function returns a (vector) def that will be
967 used in the vectorized stmt for STMT.
969 In the case that OP is an SSA_NAME which is defined in the loop, then
970 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
972 In case OP is an invariant or constant, a new stmt that creates a vector def
973 needs to be introduced. */
976 vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def)
981 stmt_vec_info def_stmt_info = NULL;
982 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
983 tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
984 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
985 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
991 enum vect_def_type dt;
995 if (vect_print_dump_info (REPORT_DETAILS))
997 fprintf (vect_dump, "vect_get_vec_def_for_operand: ");
998 print_generic_expr (vect_dump, op, TDF_SLIM);
1001 is_simple_use = vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def,
1003 gcc_assert (is_simple_use);
1004 if (vect_print_dump_info (REPORT_DETAILS))
1008 fprintf (vect_dump, "def = ");
1009 print_generic_expr (vect_dump, def, TDF_SLIM);
1013 fprintf (vect_dump, " def_stmt = ");
1014 print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
1020 /* Case 1: operand is a constant. */
1021 case vect_constant_def:
1023 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
1024 gcc_assert (vector_type);
1029 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1030 if (vect_print_dump_info (REPORT_DETAILS))
1031 fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
1033 for (i = nunits - 1; i >= 0; --i)
1035 t = tree_cons (NULL_TREE, op, t);
1037 vec_cst = build_vector (vector_type, t);
1038 return vect_init_vector (stmt, vec_cst, vector_type, NULL);
1041 /* Case 2: operand is defined outside the loop - loop invariant. */
1042 case vect_external_def:
1044 vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
1045 gcc_assert (vector_type);
1046 nunits = TYPE_VECTOR_SUBPARTS (vector_type);
1051 /* Create 'vec_inv = {inv,inv,..,inv}' */
1052 if (vect_print_dump_info (REPORT_DETAILS))
1053 fprintf (vect_dump, "Create vector_inv.");
1055 for (i = nunits - 1; i >= 0; --i)
1057 t = tree_cons (NULL_TREE, def, t);
1060 /* FIXME: use build_constructor directly. */
1061 vec_inv = build_constructor_from_list (vector_type, t);
1062 return vect_init_vector (stmt, vec_inv, vector_type, NULL);
1065 /* Case 3: operand is defined inside the loop. */
1066 case vect_internal_def:
1069 *scalar_def = NULL/* FIXME tuples: def_stmt*/;
1071 /* Get the def from the vectorized stmt. */
1072 def_stmt_info = vinfo_for_stmt (def_stmt);
1073 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1074 gcc_assert (vec_stmt);
1075 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1076 vec_oprnd = PHI_RESULT (vec_stmt);
1077 else if (is_gimple_call (vec_stmt))
1078 vec_oprnd = gimple_call_lhs (vec_stmt);
1080 vec_oprnd = gimple_assign_lhs (vec_stmt);
1084 /* Case 4: operand is defined by a loop header phi - reduction */
1085 case vect_reduction_def:
1086 case vect_double_reduction_def:
1087 case vect_nested_cycle:
1091 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1092 loop = (gimple_bb (def_stmt))->loop_father;
1094 /* Get the def before the loop */
1095 op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));
1096 return get_initial_def_for_reduction (stmt, op, scalar_def);
1099 /* Case 5: operand is defined by loop-header phi - induction. */
1100 case vect_induction_def:
1102 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1104 /* Get the def from the vectorized stmt. */
1105 def_stmt_info = vinfo_for_stmt (def_stmt);
1106 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1107 gcc_assert (vec_stmt && gimple_code (vec_stmt) == GIMPLE_PHI);
1108 vec_oprnd = PHI_RESULT (vec_stmt);
1118 /* Function vect_get_vec_def_for_stmt_copy
1120 Return a vector-def for an operand. This function is used when the
1121 vectorized stmt to be created (by the caller to this function) is a "copy"
1122 created in case the vectorized result cannot fit in one vector, and several
1123 copies of the vector-stmt are required. In this case the vector-def is
1124 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1125 of the stmt that defines VEC_OPRND.
1126 DT is the type of the vector def VEC_OPRND.
1129 In case the vectorization factor (VF) is bigger than the number
1130 of elements that can fit in a vectype (nunits), we have to generate
1131 more than one vector stmt to vectorize the scalar stmt. This situation
1132 arises when there are multiple data-types operated upon in the loop; the
1133 smallest data-type determines the VF, and as a result, when vectorizing
1134 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1135 vector stmt (each computing a vector of 'nunits' results, and together
1136 computing 'VF' results in each iteration). This function is called when
1137 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1138 which VF=16 and nunits=4, so the number of copies required is 4):
1140 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1142 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1143 VS1.1: vx.1 = memref1 VS1.2
1144 VS1.2: vx.2 = memref2 VS1.3
1145 VS1.3: vx.3 = memref3
1147 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1148 VSnew.1: vz1 = vx.1 + ... VSnew.2
1149 VSnew.2: vz2 = vx.2 + ... VSnew.3
1150 VSnew.3: vz3 = vx.3 + ...
1152 The vectorization of S1 is explained in vectorizable_load.
1153 The vectorization of S2:
1154 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1155 the function 'vect_get_vec_def_for_operand' is called to
1156 get the relevant vector-def for each operand of S2. For operand x it
1157 returns the vector-def 'vx.0'.
1159 To create the remaining copies of the vector-stmt (VSnew.j), this
1160 function is called to get the relevant vector-def for each operand. It is
1161 obtained from the respective VS1.j stmt, which is recorded in the
1162 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1164 For example, to obtain the vector-def 'vx.1' in order to create the
1165 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1166 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1167 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1168 and return its def ('vx.1').
1169 Overall, to create the above sequence this function will be called 3 times:
1170 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1171 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1172 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1175 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1177 gimple vec_stmt_for_operand;
1178 stmt_vec_info def_stmt_info;
1180 /* Do nothing; can reuse same def. */
1181 if (dt == vect_external_def || dt == vect_constant_def )
1184 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1185 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1186 gcc_assert (def_stmt_info);
1187 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1188 gcc_assert (vec_stmt_for_operand);
1189 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1190 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1191 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1193 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1198 /* Get vectorized definitions for the operands to create a copy of an original
1199 stmt. See vect_get_vec_def_for_stmt_copy() for details. */
1202 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1203 VEC(tree,heap) **vec_oprnds0,
1204 VEC(tree,heap) **vec_oprnds1)
1206 tree vec_oprnd = VEC_pop (tree, *vec_oprnds0);
1208 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1209 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1211 if (vec_oprnds1 && *vec_oprnds1)
1213 vec_oprnd = VEC_pop (tree, *vec_oprnds1);
1214 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1215 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1220 /* Get vectorized definitions for OP0 and OP1, or SLP_NODE if it is not NULL. */
1223 vect_get_vec_defs (tree op0, tree op1, gimple stmt,
1224 VEC(tree,heap) **vec_oprnds0, VEC(tree,heap) **vec_oprnds1,
1228 vect_get_slp_defs (slp_node, vec_oprnds0, vec_oprnds1, -1);
1233 *vec_oprnds0 = VEC_alloc (tree, heap, 1);
1234 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt, NULL);
1235 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1239 *vec_oprnds1 = VEC_alloc (tree, heap, 1);
1240 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt, NULL);
1241 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1247 /* Function vect_finish_stmt_generation.
1249 Insert a new stmt. */
1252 vect_finish_stmt_generation (gimple stmt, gimple vec_stmt,
1253 gimple_stmt_iterator *gsi)
1255 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1256 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1257 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1259 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1261 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1263 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo,
1266 if (vect_print_dump_info (REPORT_DETAILS))
1268 fprintf (vect_dump, "add new stmt: ");
1269 print_gimple_stmt (vect_dump, vec_stmt, 0, TDF_SLIM);
1272 gimple_set_location (vec_stmt, gimple_location (gsi_stmt (*gsi)));
1275 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1276 a function declaration if the target has a vectorized version
1277 of the function, or NULL_TREE if the function cannot be vectorized. */
1280 vectorizable_function (gimple call, tree vectype_out, tree vectype_in)
1282 tree fndecl = gimple_call_fndecl (call);
1284 /* We only handle functions that do not read or clobber memory -- i.e.
1285 const or novops ones. */
1286 if (!(gimple_call_flags (call) & (ECF_CONST | ECF_NOVOPS)))
1290 || TREE_CODE (fndecl) != FUNCTION_DECL
1291 || !DECL_BUILT_IN (fndecl))
1294 return targetm.vectorize.builtin_vectorized_function (fndecl, vectype_out,
1298 /* Function vectorizable_call.
1300 Check if STMT performs a function call that can be vectorized.
1301 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1302 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1303 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1306 vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt)
1311 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1312 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
1313 tree vectype_out, vectype_in;
1316 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1317 tree fndecl, new_temp, def, rhs_type;
1319 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1320 gimple new_stmt = NULL;
1322 VEC(tree, heap) *vargs = NULL;
1323 enum { NARROW, NONE, WIDEN } modifier;
1326 /* FORNOW: unsupported in basic block SLP. */
1327 gcc_assert (loop_vinfo);
1329 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1332 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1335 /* FORNOW: SLP not supported. */
1336 if (STMT_SLP_TYPE (stmt_info))
1339 /* Is STMT a vectorizable call? */
1340 if (!is_gimple_call (stmt))
1343 if (TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
1346 if (stmt_could_throw_p (stmt))
1349 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
1351 /* Process function arguments. */
1352 rhs_type = NULL_TREE;
1353 vectype_in = NULL_TREE;
1354 nargs = gimple_call_num_args (stmt);
1356 /* Bail out if the function has more than two arguments, we
1357 do not have interesting builtin functions to vectorize with
1358 more than two arguments. No arguments is also not good. */
1359 if (nargs == 0 || nargs > 2)
1362 for (i = 0; i < nargs; i++)
1366 op = gimple_call_arg (stmt, i);
1368 /* We can only handle calls with arguments of the same type. */
1370 && !types_compatible_p (rhs_type, TREE_TYPE (op)))
1372 if (vect_print_dump_info (REPORT_DETAILS))
1373 fprintf (vect_dump, "argument types differ.");
1377 rhs_type = TREE_TYPE (op);
1379 if (!vect_is_simple_use_1 (op, loop_vinfo, NULL,
1380 &def_stmt, &def, &dt[i], &opvectype))
1382 if (vect_print_dump_info (REPORT_DETAILS))
1383 fprintf (vect_dump, "use not simple.");
1388 vectype_in = opvectype;
1390 && opvectype != vectype_in)
1392 if (vect_print_dump_info (REPORT_DETAILS))
1393 fprintf (vect_dump, "argument vector types differ.");
1397 /* If all arguments are external or constant defs use a vector type with
1398 the same size as the output vector type. */
1400 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
1402 gcc_assert (vectype_in);
1405 if (vect_print_dump_info (REPORT_DETAILS))
1407 fprintf (vect_dump, "no vectype for scalar type ");
1408 print_generic_expr (vect_dump, rhs_type, TDF_SLIM);
1415 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1416 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1417 if (nunits_in == nunits_out / 2)
1419 else if (nunits_out == nunits_in)
1421 else if (nunits_out == nunits_in / 2)
1426 /* For now, we only vectorize functions if a target specific builtin
1427 is available. TODO -- in some cases, it might be profitable to
1428 insert the calls for pieces of the vector, in order to be able
1429 to vectorize other operations in the loop. */
1430 fndecl = vectorizable_function (stmt, vectype_out, vectype_in);
1431 if (fndecl == NULL_TREE)
1433 if (vect_print_dump_info (REPORT_DETAILS))
1434 fprintf (vect_dump, "function is not vectorizable.");
1439 gcc_assert (!gimple_vuse (stmt));
1441 if (modifier == NARROW)
1442 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1444 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1446 /* Sanity check: make sure that at least one copy of the vectorized stmt
1447 needs to be generated. */
1448 gcc_assert (ncopies >= 1);
1450 if (!vec_stmt) /* transformation not required. */
1452 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
1453 if (vect_print_dump_info (REPORT_DETAILS))
1454 fprintf (vect_dump, "=== vectorizable_call ===");
1455 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
1461 if (vect_print_dump_info (REPORT_DETAILS))
1462 fprintf (vect_dump, "transform operation.");
1465 scalar_dest = gimple_call_lhs (stmt);
1466 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1468 prev_stmt_info = NULL;
1472 for (j = 0; j < ncopies; ++j)
1474 /* Build argument list for the vectorized call. */
1476 vargs = VEC_alloc (tree, heap, nargs);
1478 VEC_truncate (tree, vargs, 0);
1480 for (i = 0; i < nargs; i++)
1482 op = gimple_call_arg (stmt, i);
1485 = vect_get_vec_def_for_operand (op, stmt, NULL);
1488 vec_oprnd0 = gimple_call_arg (new_stmt, i);
1490 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1493 VEC_quick_push (tree, vargs, vec_oprnd0);
1496 new_stmt = gimple_build_call_vec (fndecl, vargs);
1497 new_temp = make_ssa_name (vec_dest, new_stmt);
1498 gimple_call_set_lhs (new_stmt, new_temp);
1500 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1501 mark_symbols_for_renaming (new_stmt);
1504 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1506 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1508 prev_stmt_info = vinfo_for_stmt (new_stmt);
1514 for (j = 0; j < ncopies; ++j)
1516 /* Build argument list for the vectorized call. */
1518 vargs = VEC_alloc (tree, heap, nargs * 2);
1520 VEC_truncate (tree, vargs, 0);
1522 for (i = 0; i < nargs; i++)
1524 op = gimple_call_arg (stmt, i);
1528 = vect_get_vec_def_for_operand (op, stmt, NULL);
1530 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1534 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i);
1536 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
1538 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1541 VEC_quick_push (tree, vargs, vec_oprnd0);
1542 VEC_quick_push (tree, vargs, vec_oprnd1);
1545 new_stmt = gimple_build_call_vec (fndecl, vargs);
1546 new_temp = make_ssa_name (vec_dest, new_stmt);
1547 gimple_call_set_lhs (new_stmt, new_temp);
1549 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1550 mark_symbols_for_renaming (new_stmt);
1553 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1555 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1557 prev_stmt_info = vinfo_for_stmt (new_stmt);
1560 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1565 /* No current target implements this case. */
1569 VEC_free (tree, heap, vargs);
1571 /* Update the exception handling table with the vector stmt if necessary. */
1572 if (maybe_clean_or_replace_eh_stmt (stmt, *vec_stmt))
1573 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1575 /* The call in STMT might prevent it from being removed in dce.
1576 We however cannot remove it here, due to the way the ssa name
1577 it defines is mapped to the new definition. So just replace
1578 rhs of the statement with something harmless. */
1580 type = TREE_TYPE (scalar_dest);
1581 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
1582 fold_convert (type, integer_zero_node));
1583 set_vinfo_for_stmt (new_stmt, stmt_info);
1584 set_vinfo_for_stmt (stmt, NULL);
1585 STMT_VINFO_STMT (stmt_info) = new_stmt;
1586 gsi_replace (gsi, new_stmt, false);
1587 SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
1593 /* Function vect_gen_widened_results_half
1595 Create a vector stmt whose code, type, number of arguments, and result
1596 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
1597 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
1598 In the case that CODE is a CALL_EXPR, this means that a call to DECL
1599 needs to be created (DECL is a function-decl of a target-builtin).
1600 STMT is the original scalar stmt that we are vectorizing. */
1603 vect_gen_widened_results_half (enum tree_code code,
1605 tree vec_oprnd0, tree vec_oprnd1, int op_type,
1606 tree vec_dest, gimple_stmt_iterator *gsi,
1612 /* Generate half of the widened result: */
1613 if (code == CALL_EXPR)
1615 /* Target specific support */
1616 if (op_type == binary_op)
1617 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
1619 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
1620 new_temp = make_ssa_name (vec_dest, new_stmt);
1621 gimple_call_set_lhs (new_stmt, new_temp);
1625 /* Generic support */
1626 gcc_assert (op_type == TREE_CODE_LENGTH (code));
1627 if (op_type != binary_op)
1629 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vec_oprnd0,
1631 new_temp = make_ssa_name (vec_dest, new_stmt);
1632 gimple_assign_set_lhs (new_stmt, new_temp);
1634 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1640 /* Check if STMT performs a conversion operation, that can be vectorized.
1641 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1642 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1643 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1646 vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
1647 gimple *vec_stmt, slp_tree slp_node)
1652 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1653 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1654 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1655 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
1656 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
1660 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1661 gimple new_stmt = NULL;
1662 stmt_vec_info prev_stmt_info;
1665 tree vectype_out, vectype_in;
1669 enum { NARROW, NONE, WIDEN } modifier;
1671 VEC(tree,heap) *vec_oprnds0 = NULL;
1673 VEC(tree,heap) *dummy = NULL;
1676 /* Is STMT a vectorizable conversion? */
1678 /* FORNOW: unsupported in basic block SLP. */
1679 gcc_assert (loop_vinfo);
1681 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1684 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1687 if (!is_gimple_assign (stmt))
1690 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
1693 code = gimple_assign_rhs_code (stmt);
1694 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
1697 /* Check types of lhs and rhs. */
1698 scalar_dest = gimple_assign_lhs (stmt);
1699 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
1701 op0 = gimple_assign_rhs1 (stmt);
1702 rhs_type = TREE_TYPE (op0);
1703 /* Check the operands of the operation. */
1704 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
1705 &def_stmt, &def, &dt[0], &vectype_in))
1707 if (vect_print_dump_info (REPORT_DETAILS))
1708 fprintf (vect_dump, "use not simple.");
1711 /* If op0 is an external or constant defs use a vector type of
1712 the same size as the output vector type. */
1714 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
1716 gcc_assert (vectype_in);
1719 if (vect_print_dump_info (REPORT_DETAILS))
1721 fprintf (vect_dump, "no vectype for scalar type ");
1722 print_generic_expr (vect_dump, rhs_type, TDF_SLIM);
1729 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1730 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1731 if (nunits_in == nunits_out / 2)
1733 else if (nunits_out == nunits_in)
1735 else if (nunits_out == nunits_in / 2)
1740 if (modifier == NARROW)
1741 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1743 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1745 /* FORNOW: SLP with multiple types is not supported. The SLP analysis verifies
1746 this, so we can safely override NCOPIES with 1 here. */
1750 /* Sanity check: make sure that at least one copy of the vectorized stmt
1751 needs to be generated. */
1752 gcc_assert (ncopies >= 1);
1754 /* Supportable by target? */
1755 if ((modifier == NONE
1756 && !targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
1757 || (modifier == WIDEN
1758 && !supportable_widening_operation (code, stmt,
1759 vectype_out, vectype_in,
1762 &dummy_int, &dummy))
1763 || (modifier == NARROW
1764 && !supportable_narrowing_operation (code, vectype_out, vectype_in,
1765 &code1, &dummy_int, &dummy)))
1767 if (vect_print_dump_info (REPORT_DETAILS))
1768 fprintf (vect_dump, "conversion not supported by target.");
1772 if (modifier != NONE)
1774 /* FORNOW: SLP not supported. */
1775 if (STMT_SLP_TYPE (stmt_info))
1779 if (!vec_stmt) /* transformation not required. */
1781 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
1786 if (vect_print_dump_info (REPORT_DETAILS))
1787 fprintf (vect_dump, "transform conversion.");
1790 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1792 if (modifier == NONE && !slp_node)
1793 vec_oprnds0 = VEC_alloc (tree, heap, 1);
1795 prev_stmt_info = NULL;
1799 for (j = 0; j < ncopies; j++)
1802 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node);
1804 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
1807 targetm.vectorize.builtin_conversion (code,
1808 vectype_out, vectype_in);
1809 FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
1811 /* Arguments are ready. create the new vector stmt. */
1812 new_stmt = gimple_build_call (builtin_decl, 1, vop0);
1813 new_temp = make_ssa_name (vec_dest, new_stmt);
1814 gimple_call_set_lhs (new_stmt, new_temp);
1815 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1817 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
1821 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1823 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1824 prev_stmt_info = vinfo_for_stmt (new_stmt);
1829 /* In case the vectorization factor (VF) is bigger than the number
1830 of elements that we can fit in a vectype (nunits), we have to
1831 generate more than one vector stmt - i.e - we need to "unroll"
1832 the vector stmt by a factor VF/nunits. */
1833 for (j = 0; j < ncopies; j++)
1836 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1838 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1840 /* Generate first half of the widened result: */
1842 = vect_gen_widened_results_half (code1, decl1,
1843 vec_oprnd0, vec_oprnd1,
1844 unary_op, vec_dest, gsi, stmt);
1846 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1848 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1849 prev_stmt_info = vinfo_for_stmt (new_stmt);
1851 /* Generate second half of the widened result: */
1853 = vect_gen_widened_results_half (code2, decl2,
1854 vec_oprnd0, vec_oprnd1,
1855 unary_op, vec_dest, gsi, stmt);
1856 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1857 prev_stmt_info = vinfo_for_stmt (new_stmt);
1862 /* In case the vectorization factor (VF) is bigger than the number
1863 of elements that we can fit in a vectype (nunits), we have to
1864 generate more than one vector stmt - i.e - we need to "unroll"
1865 the vector stmt by a factor VF/nunits. */
1866 for (j = 0; j < ncopies; j++)
1871 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1872 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1876 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd1);
1877 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1880 /* Arguments are ready. Create the new vector stmt. */
1881 new_stmt = gimple_build_assign_with_ops (code1, vec_dest, vec_oprnd0,
1883 new_temp = make_ssa_name (vec_dest, new_stmt);
1884 gimple_assign_set_lhs (new_stmt, new_temp);
1885 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1888 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1890 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1892 prev_stmt_info = vinfo_for_stmt (new_stmt);
1895 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1899 VEC_free (tree, heap, vec_oprnds0);
1903 /* Function vectorizable_assignment.
1905 Check if STMT performs an assignment (copy) that can be vectorized.
1906 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1907 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1908 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1911 vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi,
1912 gimple *vec_stmt, slp_tree slp_node)
1917 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1918 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1919 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1923 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1924 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
1927 VEC(tree,heap) *vec_oprnds = NULL;
1929 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1930 gimple new_stmt = NULL;
1931 stmt_vec_info prev_stmt_info = NULL;
1932 enum tree_code code;
1935 /* Multiple types in SLP are handled by creating the appropriate number of
1936 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1941 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
1943 gcc_assert (ncopies >= 1);
1945 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
1948 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1951 /* Is vectorizable assignment? */
1952 if (!is_gimple_assign (stmt))
1955 scalar_dest = gimple_assign_lhs (stmt);
1956 if (TREE_CODE (scalar_dest) != SSA_NAME)
1959 code = gimple_assign_rhs_code (stmt);
1960 if (gimple_assign_single_p (stmt)
1961 || code == PAREN_EXPR
1962 || CONVERT_EXPR_CODE_P (code))
1963 op = gimple_assign_rhs1 (stmt);
1967 if (!vect_is_simple_use_1 (op, loop_vinfo, bb_vinfo,
1968 &def_stmt, &def, &dt[0], &vectype_in))
1970 if (vect_print_dump_info (REPORT_DETAILS))
1971 fprintf (vect_dump, "use not simple.");
1975 /* We can handle NOP_EXPR conversions that do not change the number
1976 of elements or the vector size. */
1977 if (CONVERT_EXPR_CODE_P (code)
1979 || TYPE_VECTOR_SUBPARTS (vectype_in) != nunits
1980 || (GET_MODE_SIZE (TYPE_MODE (vectype))
1981 != GET_MODE_SIZE (TYPE_MODE (vectype_in)))))
1984 if (!vec_stmt) /* transformation not required. */
1986 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
1987 if (vect_print_dump_info (REPORT_DETAILS))
1988 fprintf (vect_dump, "=== vectorizable_assignment ===");
1989 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
1994 if (vect_print_dump_info (REPORT_DETAILS))
1995 fprintf (vect_dump, "transform assignment.");
1998 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2001 for (j = 0; j < ncopies; j++)
2005 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node);
2007 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
2009 /* Arguments are ready. create the new vector stmt. */
2010 FOR_EACH_VEC_ELT (tree, vec_oprnds, i, vop)
2012 if (CONVERT_EXPR_CODE_P (code))
2013 vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
2014 new_stmt = gimple_build_assign (vec_dest, vop);
2015 new_temp = make_ssa_name (vec_dest, new_stmt);
2016 gimple_assign_set_lhs (new_stmt, new_temp);
2017 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2019 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2026 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2028 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2030 prev_stmt_info = vinfo_for_stmt (new_stmt);
2033 VEC_free (tree, heap, vec_oprnds);
2037 /* Function vectorizable_operation.
2039 Check if STMT performs a binary or unary operation that can be vectorized.
2040 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2041 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2042 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2045 vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
2046 gimple *vec_stmt, slp_tree slp_node)
2050 tree op0, op1 = NULL;
2051 tree vec_oprnd1 = NULL_TREE;
2052 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2054 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2055 enum tree_code code;
2056 enum machine_mode vec_mode;
2061 enum machine_mode optab_op2_mode;
2064 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2065 gimple new_stmt = NULL;
2066 stmt_vec_info prev_stmt_info;
2072 VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
2075 bool scalar_shift_arg = false;
2076 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2079 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2082 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2085 /* Is STMT a vectorizable binary/unary operation? */
2086 if (!is_gimple_assign (stmt))
2089 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2092 code = gimple_assign_rhs_code (stmt);
2094 /* For pointer addition, we should use the normal plus for
2095 the vector addition. */
2096 if (code == POINTER_PLUS_EXPR)
2099 /* Support only unary or binary operations. */
2100 op_type = TREE_CODE_LENGTH (code);
2101 if (op_type != unary_op && op_type != binary_op)
2103 if (vect_print_dump_info (REPORT_DETAILS))
2104 fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type);
2108 scalar_dest = gimple_assign_lhs (stmt);
2109 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2111 op0 = gimple_assign_rhs1 (stmt);
2112 if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
2113 &def_stmt, &def, &dt[0], &vectype))
2115 if (vect_print_dump_info (REPORT_DETAILS))
2116 fprintf (vect_dump, "use not simple.");
2119 /* If op0 is an external or constant def use a vector type with
2120 the same size as the output vector type. */
2122 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2124 gcc_assert (vectype);
2127 if (vect_print_dump_info (REPORT_DETAILS))
2129 fprintf (vect_dump, "no vectype for scalar type ");
2130 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2136 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2137 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
2138 if (nunits_out != nunits_in)
2141 if (op_type == binary_op)
2143 op1 = gimple_assign_rhs2 (stmt);
2144 if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def,
2147 if (vect_print_dump_info (REPORT_DETAILS))
2148 fprintf (vect_dump, "use not simple.");
2154 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2158 /* Multiple types in SLP are handled by creating the appropriate number of
2159 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2164 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2166 gcc_assert (ncopies >= 1);
2168 /* If this is a shift/rotate, determine whether the shift amount is a vector,
2169 or scalar. If the shift/rotate amount is a vector, use the vector/vector
2171 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
2172 || code == RROTATE_EXPR)
2174 /* vector shifted by vector */
2175 if (dt[1] == vect_internal_def)
2177 optab = optab_for_tree_code (code, vectype, optab_vector);
2178 if (vect_print_dump_info (REPORT_DETAILS))
2179 fprintf (vect_dump, "vector/vector shift/rotate found.");
2182 /* See if the machine has a vector shifted by scalar insn and if not
2183 then see if it has a vector shifted by vector insn */
2184 else if (dt[1] == vect_constant_def || dt[1] == vect_external_def)
2186 optab = optab_for_tree_code (code, vectype, optab_scalar);
2188 && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
2190 scalar_shift_arg = true;
2191 if (vect_print_dump_info (REPORT_DETAILS))
2192 fprintf (vect_dump, "vector/scalar shift/rotate found.");
2196 optab = optab_for_tree_code (code, vectype, optab_vector);
2198 && (optab_handler (optab, TYPE_MODE (vectype))
2199 != CODE_FOR_nothing))
2201 if (vect_print_dump_info (REPORT_DETAILS))
2202 fprintf (vect_dump, "vector/vector shift/rotate found.");
2204 /* Unlike the other binary operators, shifts/rotates have
2205 the rhs being int, instead of the same type as the lhs,
2206 so make sure the scalar is the right type if we are
2207 dealing with vectors of short/char. */
2208 if (dt[1] == vect_constant_def)
2209 op1 = fold_convert (TREE_TYPE (vectype), op1);
2216 if (vect_print_dump_info (REPORT_DETAILS))
2217 fprintf (vect_dump, "operand mode requires invariant argument.");
2222 optab = optab_for_tree_code (code, vectype, optab_default);
2224 /* Supportable by target? */
2227 if (vect_print_dump_info (REPORT_DETAILS))
2228 fprintf (vect_dump, "no optab.");
2231 vec_mode = TYPE_MODE (vectype);
2232 icode = (int) optab_handler (optab, vec_mode);
2233 if (icode == CODE_FOR_nothing)
2235 if (vect_print_dump_info (REPORT_DETAILS))
2236 fprintf (vect_dump, "op not supported by target.");
2237 /* Check only during analysis. */
2238 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
2239 || (vf < vect_min_worthwhile_factor (code)
2242 if (vect_print_dump_info (REPORT_DETAILS))
2243 fprintf (vect_dump, "proceeding using word mode.");
2246 /* Worthwhile without SIMD support? Check only during analysis. */
2247 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
2248 && vf < vect_min_worthwhile_factor (code)
2251 if (vect_print_dump_info (REPORT_DETAILS))
2252 fprintf (vect_dump, "not worthwhile without SIMD support.");
2256 if (!vec_stmt) /* transformation not required. */
2258 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
2259 if (vect_print_dump_info (REPORT_DETAILS))
2260 fprintf (vect_dump, "=== vectorizable_operation ===");
2261 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2267 if (vect_print_dump_info (REPORT_DETAILS))
2268 fprintf (vect_dump, "transform binary/unary operation.");
2271 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2273 /* Allocate VECs for vector operands. In case of SLP, vector operands are
2274 created in the previous stages of the recursion, so no allocation is
2275 needed, except for the case of shift with scalar shift argument. In that
2276 case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
2277 be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
2278 In case of loop-based vectorization we allocate VECs of size 1. We
2279 allocate VEC_OPRNDS1 only in case of binary operation. */
2282 vec_oprnds0 = VEC_alloc (tree, heap, 1);
2283 if (op_type == binary_op)
2284 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2286 else if (scalar_shift_arg)
2287 vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
2289 /* In case the vectorization factor (VF) is bigger than the number
2290 of elements that we can fit in a vectype (nunits), we have to generate
2291 more than one vector stmt - i.e - we need to "unroll" the
2292 vector stmt by a factor VF/nunits. In doing so, we record a pointer
2293 from one copy of the vector stmt to the next, in the field
2294 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
2295 stages to find the correct vector defs to be used when vectorizing
2296 stmts that use the defs of the current stmt. The example below illustrates
2297 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
2298 4 vectorized stmts):
2300 before vectorization:
2301 RELATED_STMT VEC_STMT
2305 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
2307 RELATED_STMT VEC_STMT
2308 VS1_0: vx0 = memref0 VS1_1 -
2309 VS1_1: vx1 = memref1 VS1_2 -
2310 VS1_2: vx2 = memref2 VS1_3 -
2311 VS1_3: vx3 = memref3 - -
2312 S1: x = load - VS1_0
2315 step2: vectorize stmt S2 (done here):
2316 To vectorize stmt S2 we first need to find the relevant vector
2317 def for the first operand 'x'. This is, as usual, obtained from
2318 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
2319 that defines 'x' (S1). This way we find the stmt VS1_0, and the
2320 relevant vector def 'vx0'. Having found 'vx0' we can generate
2321 the vector stmt VS2_0, and as usual, record it in the
2322 STMT_VINFO_VEC_STMT of stmt S2.
2323 When creating the second copy (VS2_1), we obtain the relevant vector
2324 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
2325 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
2326 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
2327 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
2328 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
2329 chain of stmts and pointers:
2330 RELATED_STMT VEC_STMT
2331 VS1_0: vx0 = memref0 VS1_1 -
2332 VS1_1: vx1 = memref1 VS1_2 -
2333 VS1_2: vx2 = memref2 VS1_3 -
2334 VS1_3: vx3 = memref3 - -
2335 S1: x = load - VS1_0
2336 VS2_0: vz0 = vx0 + v1 VS2_1 -
2337 VS2_1: vz1 = vx1 + v1 VS2_2 -
2338 VS2_2: vz2 = vx2 + v1 VS2_3 -
2339 VS2_3: vz3 = vx3 + v1 - -
2340 S2: z = x + 1 - VS2_0 */
2342 prev_stmt_info = NULL;
2343 for (j = 0; j < ncopies; j++)
2348 if (op_type == binary_op && scalar_shift_arg)
2350 /* Vector shl and shr insn patterns can be defined with scalar
2351 operand 2 (shift operand). In this case, use constant or loop
2352 invariant op1 directly, without extending it to vector mode
2354 optab_op2_mode = insn_data[icode].operand[2].mode;
2355 if (!VECTOR_MODE_P (optab_op2_mode))
2357 if (vect_print_dump_info (REPORT_DETAILS))
2358 fprintf (vect_dump, "operand 1 using scalar mode.");
2360 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2363 /* Store vec_oprnd1 for every vector stmt to be created
2364 for SLP_NODE. We check during the analysis that all the
2365 shift arguments are the same.
2366 TODO: Allow different constants for different vector
2367 stmts generated for an SLP instance. */
2368 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
2369 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2374 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
2375 (a special case for certain kind of vector shifts); otherwise,
2376 operand 1 should be of a vector type (the usual case). */
2377 if (op_type == binary_op && !vec_oprnd1)
2378 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
2381 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
2385 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
2387 /* Arguments are ready. Create the new vector stmt. */
2388 FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
2390 vop1 = ((op_type == binary_op)
2391 ? VEC_index (tree, vec_oprnds1, i) : NULL);
2392 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2393 new_temp = make_ssa_name (vec_dest, new_stmt);
2394 gimple_assign_set_lhs (new_stmt, new_temp);
2395 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2397 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2404 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2406 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2407 prev_stmt_info = vinfo_for_stmt (new_stmt);
2410 VEC_free (tree, heap, vec_oprnds0);
2412 VEC_free (tree, heap, vec_oprnds1);
2418 /* Get vectorized definitions for loop-based vectorization. For the first
2419 operand we call vect_get_vec_def_for_operand() (with OPRND containing
2420 scalar operand), and for the rest we get a copy with
2421 vect_get_vec_def_for_stmt_copy() using the previous vector definition
2422 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
2423 The vectors are collected into VEC_OPRNDS. */
2426 vect_get_loop_based_defs (tree *oprnd, gimple stmt, enum vect_def_type dt,
2427 VEC (tree, heap) **vec_oprnds, int multi_step_cvt)
2431 /* Get first vector operand. */
2432 /* All the vector operands except the very first one (that is scalar oprnd)
2434 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
2435 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt, NULL);
2437 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
2439 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2441 /* Get second vector operand. */
2442 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
2443 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2447 /* For conversion in multiple steps, continue to get operands
2450 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
2454 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
2455 For multi-step conversions store the resulting vectors and call the function
2459 vect_create_vectorized_demotion_stmts (VEC (tree, heap) **vec_oprnds,
2460 int multi_step_cvt, gimple stmt,
2461 VEC (tree, heap) *vec_dsts,
2462 gimple_stmt_iterator *gsi,
2463 slp_tree slp_node, enum tree_code code,
2464 stmt_vec_info *prev_stmt_info)
2467 tree vop0, vop1, new_tmp, vec_dest;
2469 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2471 vec_dest = VEC_pop (tree, vec_dsts);
2473 for (i = 0; i < VEC_length (tree, *vec_oprnds); i += 2)
2475 /* Create demotion operation. */
2476 vop0 = VEC_index (tree, *vec_oprnds, i);
2477 vop1 = VEC_index (tree, *vec_oprnds, i + 1);
2478 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2479 new_tmp = make_ssa_name (vec_dest, new_stmt);
2480 gimple_assign_set_lhs (new_stmt, new_tmp);
2481 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2484 /* Store the resulting vector for next recursive call. */
2485 VEC_replace (tree, *vec_oprnds, i/2, new_tmp);
2488 /* This is the last step of the conversion sequence. Store the
2489 vectors in SLP_NODE or in vector info of the scalar statement
2490 (or in STMT_VINFO_RELATED_STMT chain). */
2492 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2495 if (!*prev_stmt_info)
2496 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
2498 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
2500 *prev_stmt_info = vinfo_for_stmt (new_stmt);
2505 /* For multi-step demotion operations we first generate demotion operations
2506 from the source type to the intermediate types, and then combine the
2507 results (stored in VEC_OPRNDS) in demotion operation to the destination
2511 /* At each level of recursion we have have of the operands we had at the
2513 VEC_truncate (tree, *vec_oprnds, (i+1)/2);
2514 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
2515 stmt, vec_dsts, gsi, slp_node,
2516 code, prev_stmt_info);
2521 /* Function vectorizable_type_demotion
2523 Check if STMT performs a binary or unary operation that involves
2524 type demotion, and if it can be vectorized.
2525 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2526 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2527 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2530 vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi,
2531 gimple *vec_stmt, slp_tree slp_node)
2536 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2537 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2538 enum tree_code code, code1 = ERROR_MARK;
2541 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2542 stmt_vec_info prev_stmt_info;
2549 int multi_step_cvt = 0;
2550 VEC (tree, heap) *vec_oprnds0 = NULL;
2551 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
2552 tree last_oprnd, intermediate_type;
2554 /* FORNOW: not supported by basic block SLP vectorization. */
2555 gcc_assert (loop_vinfo);
2557 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2560 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2563 /* Is STMT a vectorizable type-demotion operation? */
2564 if (!is_gimple_assign (stmt))
2567 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2570 code = gimple_assign_rhs_code (stmt);
2571 if (!CONVERT_EXPR_CODE_P (code))
2574 scalar_dest = gimple_assign_lhs (stmt);
2575 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2577 /* Check the operands of the operation. */
2578 op0 = gimple_assign_rhs1 (stmt);
2579 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
2580 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
2581 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
2582 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
2583 && CONVERT_EXPR_CODE_P (code))))
2585 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
2586 &def_stmt, &def, &dt[0], &vectype_in))
2588 if (vect_print_dump_info (REPORT_DETAILS))
2589 fprintf (vect_dump, "use not simple.");
2592 /* If op0 is an external def use a vector type with the
2593 same size as the output vector type if possible. */
2595 vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2597 gcc_assert (vectype_in);
2600 if (vect_print_dump_info (REPORT_DETAILS))
2602 fprintf (vect_dump, "no vectype for scalar type ");
2603 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2609 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2610 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2611 if (nunits_in >= nunits_out)
2614 /* Multiple types in SLP are handled by creating the appropriate number of
2615 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2620 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
2621 gcc_assert (ncopies >= 1);
2623 /* Supportable by target? */
2624 if (!supportable_narrowing_operation (code, vectype_out, vectype_in,
2625 &code1, &multi_step_cvt, &interm_types))
2628 if (!vec_stmt) /* transformation not required. */
2630 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
2631 if (vect_print_dump_info (REPORT_DETAILS))
2632 fprintf (vect_dump, "=== vectorizable_demotion ===");
2633 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2638 if (vect_print_dump_info (REPORT_DETAILS))
2639 fprintf (vect_dump, "transform type demotion operation. ncopies = %d.",
2642 /* In case of multi-step demotion, we first generate demotion operations to
2643 the intermediate types, and then from that types to the final one.
2644 We create vector destinations for the intermediate type (TYPES) received
2645 from supportable_narrowing_operation, and store them in the correct order
2646 for future use in vect_create_vectorized_demotion_stmts(). */
2648 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
2650 vec_dsts = VEC_alloc (tree, heap, 1);
2652 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2653 VEC_quick_push (tree, vec_dsts, vec_dest);
2657 for (i = VEC_length (tree, interm_types) - 1;
2658 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
2660 vec_dest = vect_create_destination_var (scalar_dest,
2662 VEC_quick_push (tree, vec_dsts, vec_dest);
2666 /* In case the vectorization factor (VF) is bigger than the number
2667 of elements that we can fit in a vectype (nunits), we have to generate
2668 more than one vector stmt - i.e - we need to "unroll" the
2669 vector stmt by a factor VF/nunits. */
2671 prev_stmt_info = NULL;
2672 for (j = 0; j < ncopies; j++)
2676 vect_get_slp_defs (slp_node, &vec_oprnds0, NULL, -1);
2679 VEC_free (tree, heap, vec_oprnds0);
2680 vec_oprnds0 = VEC_alloc (tree, heap,
2681 (multi_step_cvt ? vect_pow2 (multi_step_cvt) * 2 : 2));
2682 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
2683 vect_pow2 (multi_step_cvt) - 1);
2686 /* Arguments are ready. Create the new vector stmts. */
2687 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
2688 vect_create_vectorized_demotion_stmts (&vec_oprnds0,
2689 multi_step_cvt, stmt, tmp_vec_dsts,
2690 gsi, slp_node, code1,
2694 VEC_free (tree, heap, vec_oprnds0);
2695 VEC_free (tree, heap, vec_dsts);
2696 VEC_free (tree, heap, tmp_vec_dsts);
2697 VEC_free (tree, heap, interm_types);
2699 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2704 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
2705 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
2706 the resulting vectors and call the function recursively. */
2709 vect_create_vectorized_promotion_stmts (VEC (tree, heap) **vec_oprnds0,
2710 VEC (tree, heap) **vec_oprnds1,
2711 int multi_step_cvt, gimple stmt,
2712 VEC (tree, heap) *vec_dsts,
2713 gimple_stmt_iterator *gsi,
2714 slp_tree slp_node, enum tree_code code1,
2715 enum tree_code code2, tree decl1,
2716 tree decl2, int op_type,
2717 stmt_vec_info *prev_stmt_info)
2720 tree vop0, vop1, new_tmp1, new_tmp2, vec_dest;
2721 gimple new_stmt1, new_stmt2;
2722 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2723 VEC (tree, heap) *vec_tmp;
2725 vec_dest = VEC_pop (tree, vec_dsts);
2726 vec_tmp = VEC_alloc (tree, heap, VEC_length (tree, *vec_oprnds0) * 2);
2728 FOR_EACH_VEC_ELT (tree, *vec_oprnds0, i, vop0)
2730 if (op_type == binary_op)
2731 vop1 = VEC_index (tree, *vec_oprnds1, i);
2735 /* Generate the two halves of promotion operation. */
2736 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
2737 op_type, vec_dest, gsi, stmt);
2738 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
2739 op_type, vec_dest, gsi, stmt);
2740 if (is_gimple_call (new_stmt1))
2742 new_tmp1 = gimple_call_lhs (new_stmt1);
2743 new_tmp2 = gimple_call_lhs (new_stmt2);
2747 new_tmp1 = gimple_assign_lhs (new_stmt1);
2748 new_tmp2 = gimple_assign_lhs (new_stmt2);
2753 /* Store the results for the recursive call. */
2754 VEC_quick_push (tree, vec_tmp, new_tmp1);
2755 VEC_quick_push (tree, vec_tmp, new_tmp2);
2759 /* Last step of promotion sequience - store the results. */
2762 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt1);
2763 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt2);
2767 if (!*prev_stmt_info)
2768 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt1;
2770 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt1;
2772 *prev_stmt_info = vinfo_for_stmt (new_stmt1);
2773 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt2;
2774 *prev_stmt_info = vinfo_for_stmt (new_stmt2);
2781 /* For multi-step promotion operation we first generate we call the
2782 function recurcively for every stage. We start from the input type,
2783 create promotion operations to the intermediate types, and then
2784 create promotions to the output type. */
2785 *vec_oprnds0 = VEC_copy (tree, heap, vec_tmp);
2786 VEC_free (tree, heap, vec_tmp);
2787 vect_create_vectorized_promotion_stmts (vec_oprnds0, vec_oprnds1,
2788 multi_step_cvt - 1, stmt,
2789 vec_dsts, gsi, slp_node, code1,
2790 code2, decl2, decl2, op_type,
2796 /* Function vectorizable_type_promotion
2798 Check if STMT performs a binary or unary operation that involves
2799 type promotion, and if it can be vectorized.
2800 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2801 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2802 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2805 vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi,
2806 gimple *vec_stmt, slp_tree slp_node)
2810 tree op0, op1 = NULL;
2811 tree vec_oprnd0=NULL, vec_oprnd1=NULL;
2812 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2813 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2814 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
2815 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
2819 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2820 stmt_vec_info prev_stmt_info;
2827 tree intermediate_type = NULL_TREE;
2828 int multi_step_cvt = 0;
2829 VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
2830 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
2832 /* FORNOW: not supported by basic block SLP vectorization. */
2833 gcc_assert (loop_vinfo);
2835 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2838 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2841 /* Is STMT a vectorizable type-promotion operation? */
2842 if (!is_gimple_assign (stmt))
2845 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2848 code = gimple_assign_rhs_code (stmt);
2849 if (!CONVERT_EXPR_CODE_P (code)
2850 && code != WIDEN_MULT_EXPR)
2853 scalar_dest = gimple_assign_lhs (stmt);
2854 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2856 /* Check the operands of the operation. */
2857 op0 = gimple_assign_rhs1 (stmt);
2858 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
2859 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
2860 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
2861 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
2862 && CONVERT_EXPR_CODE_P (code))))
2864 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
2865 &def_stmt, &def, &dt[0], &vectype_in))
2867 if (vect_print_dump_info (REPORT_DETAILS))
2868 fprintf (vect_dump, "use not simple.");
2871 /* If op0 is an external or constant def use a vector type with
2872 the same size as the output vector type. */
2874 vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2876 gcc_assert (vectype_in);
2879 if (vect_print_dump_info (REPORT_DETAILS))
2881 fprintf (vect_dump, "no vectype for scalar type ");
2882 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2888 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2889 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2890 if (nunits_in <= nunits_out)
2893 /* Multiple types in SLP are handled by creating the appropriate number of
2894 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2899 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2901 gcc_assert (ncopies >= 1);
2903 op_type = TREE_CODE_LENGTH (code);
2904 if (op_type == binary_op)
2906 op1 = gimple_assign_rhs2 (stmt);
2907 if (!vect_is_simple_use (op1, loop_vinfo, NULL, &def_stmt, &def, &dt[1]))
2909 if (vect_print_dump_info (REPORT_DETAILS))
2910 fprintf (vect_dump, "use not simple.");
2915 /* Supportable by target? */
2916 if (!supportable_widening_operation (code, stmt, vectype_out, vectype_in,
2917 &decl1, &decl2, &code1, &code2,
2918 &multi_step_cvt, &interm_types))
2921 /* Binary widening operation can only be supported directly by the
2923 gcc_assert (!(multi_step_cvt && op_type == binary_op));
2925 if (!vec_stmt) /* transformation not required. */
2927 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
2928 if (vect_print_dump_info (REPORT_DETAILS))
2929 fprintf (vect_dump, "=== vectorizable_promotion ===");
2930 vect_model_simple_cost (stmt_info, 2*ncopies, dt, NULL);
2936 if (vect_print_dump_info (REPORT_DETAILS))
2937 fprintf (vect_dump, "transform type promotion operation. ncopies = %d.",
2941 /* In case of multi-step promotion, we first generate promotion operations
2942 to the intermediate types, and then from that types to the final one.
2943 We store vector destination in VEC_DSTS in the correct order for
2944 recursive creation of promotion operations in
2945 vect_create_vectorized_promotion_stmts(). Vector destinations are created
2946 according to TYPES recieved from supportable_widening_operation(). */
2948 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
2950 vec_dsts = VEC_alloc (tree, heap, 1);
2952 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2953 VEC_quick_push (tree, vec_dsts, vec_dest);
2957 for (i = VEC_length (tree, interm_types) - 1;
2958 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
2960 vec_dest = vect_create_destination_var (scalar_dest,
2962 VEC_quick_push (tree, vec_dsts, vec_dest);
2968 vec_oprnds0 = VEC_alloc (tree, heap,
2969 (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
2970 if (op_type == binary_op)
2971 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2974 /* In case the vectorization factor (VF) is bigger than the number
2975 of elements that we can fit in a vectype (nunits), we have to generate
2976 more than one vector stmt - i.e - we need to "unroll" the
2977 vector stmt by a factor VF/nunits. */
2979 prev_stmt_info = NULL;
2980 for (j = 0; j < ncopies; j++)
2986 vect_get_slp_defs (slp_node, &vec_oprnds0, &vec_oprnds1, -1);
2989 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
2990 VEC_quick_push (tree, vec_oprnds0, vec_oprnd0);
2991 if (op_type == binary_op)
2993 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt, NULL);
2994 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
3000 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
3001 VEC_replace (tree, vec_oprnds0, 0, vec_oprnd0);
3002 if (op_type == binary_op)
3004 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd1);
3005 VEC_replace (tree, vec_oprnds1, 0, vec_oprnd1);
3009 /* Arguments are ready. Create the new vector stmts. */
3010 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
3011 vect_create_vectorized_promotion_stmts (&vec_oprnds0, &vec_oprnds1,
3012 multi_step_cvt, stmt,
3014 gsi, slp_node, code1, code2,
3015 decl1, decl2, op_type,
3019 VEC_free (tree, heap, vec_dsts);
3020 VEC_free (tree, heap, tmp_vec_dsts);
3021 VEC_free (tree, heap, interm_types);
3022 VEC_free (tree, heap, vec_oprnds0);
3023 VEC_free (tree, heap, vec_oprnds1);
3025 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3030 /* Function vectorizable_store.
3032 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
3034 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3035 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3036 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3039 vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
3045 tree vec_oprnd = NULL_TREE;
3046 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3047 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
3048 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3049 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3050 struct loop *loop = NULL;
3051 enum machine_mode vec_mode;
3053 enum dr_alignment_support alignment_support_scheme;
3056 enum vect_def_type dt;
3057 stmt_vec_info prev_stmt_info = NULL;
3058 tree dataref_ptr = NULL_TREE;
3059 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3062 gimple next_stmt, first_stmt = NULL;
3063 bool strided_store = false;
3064 unsigned int group_size, i;
3065 VEC(tree,heap) *dr_chain = NULL, *oprnds = NULL, *result_chain = NULL;
3067 VEC(tree,heap) *vec_oprnds = NULL;
3068 bool slp = (slp_node != NULL);
3069 unsigned int vec_num;
3070 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3073 loop = LOOP_VINFO_LOOP (loop_vinfo);
3075 /* Multiple types in SLP are handled by creating the appropriate number of
3076 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3081 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3083 gcc_assert (ncopies >= 1);
3085 /* FORNOW. This restriction should be relaxed. */
3086 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
3088 if (vect_print_dump_info (REPORT_DETAILS))
3089 fprintf (vect_dump, "multiple types in nested loop.");
3093 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3096 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3099 /* Is vectorizable store? */
3101 if (!is_gimple_assign (stmt))
3104 scalar_dest = gimple_assign_lhs (stmt);
3105 if (TREE_CODE (scalar_dest) != ARRAY_REF
3106 && TREE_CODE (scalar_dest) != INDIRECT_REF
3107 && TREE_CODE (scalar_dest) != COMPONENT_REF
3108 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
3109 && TREE_CODE (scalar_dest) != REALPART_EXPR
3110 && TREE_CODE (scalar_dest) != MEM_REF)
3113 gcc_assert (gimple_assign_single_p (stmt));
3114 op = gimple_assign_rhs1 (stmt);
3115 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt))
3117 if (vect_print_dump_info (REPORT_DETAILS))
3118 fprintf (vect_dump, "use not simple.");
3122 /* The scalar rhs type needs to be trivially convertible to the vector
3123 component type. This should always be the case. */
3124 if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (op)))
3126 if (vect_print_dump_info (REPORT_DETAILS))
3127 fprintf (vect_dump, "??? operands of different types");
3131 vec_mode = TYPE_MODE (vectype);
3132 /* FORNOW. In some cases can vectorize even if data-type not supported
3133 (e.g. - array initialization with 0). */
3134 if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing)
3137 if (!STMT_VINFO_DATA_REF (stmt_info))
3140 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
3142 strided_store = true;
3143 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
3144 if (!vect_strided_store_supported (vectype)
3145 && !PURE_SLP_STMT (stmt_info) && !slp)
3148 if (first_stmt == stmt)
3150 /* STMT is the leader of the group. Check the operands of all the
3151 stmts of the group. */
3152 next_stmt = DR_GROUP_NEXT_DR (stmt_info);
3155 gcc_assert (gimple_assign_single_p (next_stmt));
3156 op = gimple_assign_rhs1 (next_stmt);
3157 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt,
3160 if (vect_print_dump_info (REPORT_DETAILS))
3161 fprintf (vect_dump, "use not simple.");
3164 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3169 if (!vec_stmt) /* transformation not required. */
3171 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
3172 vect_model_store_cost (stmt_info, ncopies, dt, NULL);
3180 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3181 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3183 DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
3186 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
3188 /* We vectorize all the stmts of the interleaving group when we
3189 reach the last stmt in the group. */
3190 if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
3191 < DR_GROUP_SIZE (vinfo_for_stmt (first_stmt))
3200 strided_store = false;
3201 /* VEC_NUM is the number of vect stmts to be created for this
3203 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3204 first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
3205 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3208 /* VEC_NUM is the number of vect stmts to be created for this
3210 vec_num = group_size;
3216 group_size = vec_num = 1;
3219 if (vect_print_dump_info (REPORT_DETAILS))
3220 fprintf (vect_dump, "transform store. ncopies = %d",ncopies);
3222 dr_chain = VEC_alloc (tree, heap, group_size);
3223 oprnds = VEC_alloc (tree, heap, group_size);
3225 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
3226 gcc_assert (alignment_support_scheme);
3228 /* In case the vectorization factor (VF) is bigger than the number
3229 of elements that we can fit in a vectype (nunits), we have to generate
3230 more than one vector stmt - i.e - we need to "unroll" the
3231 vector stmt by a factor VF/nunits. For more details see documentation in
3232 vect_get_vec_def_for_copy_stmt. */
3234 /* In case of interleaving (non-unit strided access):
3241 We create vectorized stores starting from base address (the access of the
3242 first stmt in the chain (S2 in the above example), when the last store stmt
3243 of the chain (S4) is reached:
3246 VS2: &base + vec_size*1 = vx0
3247 VS3: &base + vec_size*2 = vx1
3248 VS4: &base + vec_size*3 = vx3
3250 Then permutation statements are generated:
3252 VS5: vx5 = VEC_INTERLEAVE_HIGH_EXPR < vx0, vx3 >
3253 VS6: vx6 = VEC_INTERLEAVE_LOW_EXPR < vx0, vx3 >
3256 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3257 (the order of the data-refs in the output of vect_permute_store_chain
3258 corresponds to the order of scalar stmts in the interleaving chain - see
3259 the documentation of vect_permute_store_chain()).
3261 In case of both multiple types and interleaving, above vector stores and
3262 permutation stmts are created for every copy. The result vector stmts are
3263 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3264 STMT_VINFO_RELATED_STMT for the next copies.
3267 prev_stmt_info = NULL;
3268 for (j = 0; j < ncopies; j++)
3277 /* Get vectorized arguments for SLP_NODE. */
3278 vect_get_slp_defs (slp_node, &vec_oprnds, NULL, -1);
3280 vec_oprnd = VEC_index (tree, vec_oprnds, 0);
3284 /* For interleaved stores we collect vectorized defs for all the
3285 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
3286 used as an input to vect_permute_store_chain(), and OPRNDS as
3287 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
3289 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3290 OPRNDS are of size 1. */
3291 next_stmt = first_stmt;
3292 for (i = 0; i < group_size; i++)
3294 /* Since gaps are not supported for interleaved stores,
3295 GROUP_SIZE is the exact number of stmts in the chain.
3296 Therefore, NEXT_STMT can't be NULL_TREE. In case that
3297 there is no interleaving, GROUP_SIZE is 1, and only one
3298 iteration of the loop will be executed. */
3299 gcc_assert (next_stmt
3300 && gimple_assign_single_p (next_stmt));
3301 op = gimple_assign_rhs1 (next_stmt);
3303 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
3305 VEC_quick_push(tree, dr_chain, vec_oprnd);
3306 VEC_quick_push(tree, oprnds, vec_oprnd);
3307 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3311 /* We should have catched mismatched types earlier. */
3312 gcc_assert (useless_type_conversion_p (vectype,
3313 TREE_TYPE (vec_oprnd)));
3314 dataref_ptr = vect_create_data_ref_ptr (first_stmt, NULL, NULL_TREE,
3315 &dummy, &ptr_incr, false,
3317 gcc_assert (bb_vinfo || !inv_p);
3321 /* For interleaved stores we created vectorized defs for all the
3322 defs stored in OPRNDS in the previous iteration (previous copy).
3323 DR_CHAIN is then used as an input to vect_permute_store_chain(),
3324 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
3326 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3327 OPRNDS are of size 1. */
3328 for (i = 0; i < group_size; i++)
3330 op = VEC_index (tree, oprnds, i);
3331 vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def,
3333 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
3334 VEC_replace(tree, dr_chain, i, vec_oprnd);
3335 VEC_replace(tree, oprnds, i, vec_oprnd);
3338 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
3343 result_chain = VEC_alloc (tree, heap, group_size);
3345 if (!vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
3350 next_stmt = first_stmt;
3351 for (i = 0; i < vec_num; i++)
3353 struct ptr_info_def *pi;
3356 /* Bump the vector pointer. */
3357 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
3361 vec_oprnd = VEC_index (tree, vec_oprnds, i);
3362 else if (strided_store)
3363 /* For strided stores vectorized defs are interleaved in
3364 vect_permute_store_chain(). */
3365 vec_oprnd = VEC_index (tree, result_chain, i);
3367 data_ref = build2 (MEM_REF, TREE_TYPE (vec_oprnd), dataref_ptr,
3368 build_int_cst (reference_alias_ptr_type
3369 (DR_REF (first_dr)), 0));
3370 pi = get_ptr_info (dataref_ptr);
3371 pi->align = TYPE_ALIGN_UNIT (vectype);
3372 if (aligned_access_p (first_dr))
3374 else if (DR_MISALIGNMENT (first_dr) == -1)
3376 TREE_TYPE (data_ref)
3377 = build_aligned_type (TREE_TYPE (data_ref),
3378 TYPE_ALIGN (TREE_TYPE (vectype)));
3379 pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
3384 TREE_TYPE (data_ref)
3385 = build_aligned_type (TREE_TYPE (data_ref),
3386 TYPE_ALIGN (TREE_TYPE (vectype)));
3387 pi->misalign = DR_MISALIGNMENT (first_dr);
3390 /* Arguments are ready. Create the new vector stmt. */
3391 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
3392 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3393 mark_symbols_for_renaming (new_stmt);
3399 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3401 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3403 prev_stmt_info = vinfo_for_stmt (new_stmt);
3404 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3410 VEC_free (tree, heap, dr_chain);
3411 VEC_free (tree, heap, oprnds);
3413 VEC_free (tree, heap, result_chain);
3418 /* vectorizable_load.
3420 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
3422 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3423 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3424 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3427 vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
3428 slp_tree slp_node, slp_instance slp_node_instance)
3431 tree vec_dest = NULL;
3432 tree data_ref = NULL;
3433 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3434 stmt_vec_info prev_stmt_info;
3435 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3436 struct loop *loop = NULL;
3437 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
3438 bool nested_in_vect_loop = false;
3439 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
3440 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3442 enum machine_mode mode;
3443 gimple new_stmt = NULL;
3445 enum dr_alignment_support alignment_support_scheme;
3446 tree dataref_ptr = NULL_TREE;
3448 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3450 int i, j, group_size;
3451 tree msq = NULL_TREE, lsq;
3452 tree offset = NULL_TREE;
3453 tree realignment_token = NULL_TREE;
3455 VEC(tree,heap) *dr_chain = NULL;
3456 bool strided_load = false;
3460 bool compute_in_loop = false;
3461 struct loop *at_loop;
3463 bool slp = (slp_node != NULL);
3464 bool slp_perm = false;
3465 enum tree_code code;
3466 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3471 loop = LOOP_VINFO_LOOP (loop_vinfo);
3472 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
3473 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
3478 /* Multiple types in SLP are handled by creating the appropriate number of
3479 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3484 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3486 gcc_assert (ncopies >= 1);
3488 /* FORNOW. This restriction should be relaxed. */
3489 if (nested_in_vect_loop && ncopies > 1)
3491 if (vect_print_dump_info (REPORT_DETAILS))
3492 fprintf (vect_dump, "multiple types in nested loop.");
3496 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3499 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3502 /* Is vectorizable load? */
3503 if (!is_gimple_assign (stmt))
3506 scalar_dest = gimple_assign_lhs (stmt);
3507 if (TREE_CODE (scalar_dest) != SSA_NAME)
3510 code = gimple_assign_rhs_code (stmt);
3511 if (code != ARRAY_REF
3512 && code != INDIRECT_REF
3513 && code != COMPONENT_REF
3514 && code != IMAGPART_EXPR
3515 && code != REALPART_EXPR
3519 if (!STMT_VINFO_DATA_REF (stmt_info))
3522 scalar_type = TREE_TYPE (DR_REF (dr));
3523 mode = TYPE_MODE (vectype);
3525 /* FORNOW. In some cases can vectorize even if data-type not supported
3526 (e.g. - data copies). */
3527 if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
3529 if (vect_print_dump_info (REPORT_DETAILS))
3530 fprintf (vect_dump, "Aligned load, but unsupported type.");
3534 /* The vector component type needs to be trivially convertible to the
3535 scalar lhs. This should always be the case. */
3536 if (!useless_type_conversion_p (TREE_TYPE (scalar_dest), TREE_TYPE (vectype)))
3538 if (vect_print_dump_info (REPORT_DETAILS))
3539 fprintf (vect_dump, "??? operands of different types");
3543 /* Check if the load is a part of an interleaving chain. */
3544 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
3546 strided_load = true;
3548 gcc_assert (! nested_in_vect_loop);
3550 /* Check if interleaving is supported. */
3551 if (!vect_strided_load_supported (vectype)
3552 && !PURE_SLP_STMT (stmt_info) && !slp)
3556 if (!vec_stmt) /* transformation not required. */
3558 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
3559 vect_model_load_cost (stmt_info, ncopies, NULL);
3563 if (vect_print_dump_info (REPORT_DETAILS))
3564 fprintf (vect_dump, "transform load.");
3570 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
3571 /* Check if the chain of loads is already vectorized. */
3572 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt)))
3574 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3577 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3578 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3580 /* VEC_NUM is the number of vect stmts to be created for this group. */
3583 strided_load = false;
3584 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3585 if (SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance))
3589 vec_num = group_size;
3591 dr_chain = VEC_alloc (tree, heap, vec_num);
3597 group_size = vec_num = 1;
3600 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
3601 gcc_assert (alignment_support_scheme);
3603 /* In case the vectorization factor (VF) is bigger than the number
3604 of elements that we can fit in a vectype (nunits), we have to generate
3605 more than one vector stmt - i.e - we need to "unroll" the
3606 vector stmt by a factor VF/nunits. In doing so, we record a pointer
3607 from one copy of the vector stmt to the next, in the field
3608 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
3609 stages to find the correct vector defs to be used when vectorizing
3610 stmts that use the defs of the current stmt. The example below illustrates
3611 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
3612 4 vectorized stmts):
3614 before vectorization:
3615 RELATED_STMT VEC_STMT
3619 step 1: vectorize stmt S1:
3620 We first create the vector stmt VS1_0, and, as usual, record a
3621 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
3622 Next, we create the vector stmt VS1_1, and record a pointer to
3623 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
3624 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
3626 RELATED_STMT VEC_STMT
3627 VS1_0: vx0 = memref0 VS1_1 -
3628 VS1_1: vx1 = memref1 VS1_2 -
3629 VS1_2: vx2 = memref2 VS1_3 -
3630 VS1_3: vx3 = memref3 - -
3631 S1: x = load - VS1_0
3634 See in documentation in vect_get_vec_def_for_stmt_copy for how the
3635 information we recorded in RELATED_STMT field is used to vectorize
3638 /* In case of interleaving (non-unit strided access):
3645 Vectorized loads are created in the order of memory accesses
3646 starting from the access of the first stmt of the chain:
3649 VS2: vx1 = &base + vec_size*1
3650 VS3: vx3 = &base + vec_size*2
3651 VS4: vx4 = &base + vec_size*3
3653 Then permutation statements are generated:
3655 VS5: vx5 = VEC_EXTRACT_EVEN_EXPR < vx0, vx1 >
3656 VS6: vx6 = VEC_EXTRACT_ODD_EXPR < vx0, vx1 >
3659 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3660 (the order of the data-refs in the output of vect_permute_load_chain
3661 corresponds to the order of scalar stmts in the interleaving chain - see
3662 the documentation of vect_permute_load_chain()).
3663 The generation of permutation stmts and recording them in
3664 STMT_VINFO_VEC_STMT is done in vect_transform_strided_load().
3666 In case of both multiple types and interleaving, the vector loads and
3667 permutation stmts above are created for every copy. The result vector stmts
3668 are put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3669 STMT_VINFO_RELATED_STMT for the next copies. */
3671 /* If the data reference is aligned (dr_aligned) or potentially unaligned
3672 on a target that supports unaligned accesses (dr_unaligned_supported)
3673 we generate the following code:
3677 p = p + indx * vectype_size;
3682 Otherwise, the data reference is potentially unaligned on a target that
3683 does not support unaligned accesses (dr_explicit_realign_optimized) -
3684 then generate the following code, in which the data in each iteration is
3685 obtained by two vector loads, one from the previous iteration, and one
3686 from the current iteration:
3688 msq_init = *(floor(p1))
3689 p2 = initial_addr + VS - 1;
3690 realignment_token = call target_builtin;
3693 p2 = p2 + indx * vectype_size
3695 vec_dest = realign_load (msq, lsq, realignment_token)
3700 /* If the misalignment remains the same throughout the execution of the
3701 loop, we can create the init_addr and permutation mask at the loop
3702 preheader. Otherwise, it needs to be created inside the loop.
3703 This can only occur when vectorizing memory accesses in the inner-loop
3704 nested within an outer-loop that is being vectorized. */
3706 if (loop && nested_in_vect_loop_p (loop, stmt)
3707 && (TREE_INT_CST_LOW (DR_STEP (dr))
3708 % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
3710 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
3711 compute_in_loop = true;
3714 if ((alignment_support_scheme == dr_explicit_realign_optimized
3715 || alignment_support_scheme == dr_explicit_realign)
3716 && !compute_in_loop)
3718 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
3719 alignment_support_scheme, NULL_TREE,
3721 if (alignment_support_scheme == dr_explicit_realign_optimized)
3723 phi = SSA_NAME_DEF_STMT (msq);
3724 offset = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
3730 prev_stmt_info = NULL;
3731 for (j = 0; j < ncopies; j++)
3733 /* 1. Create the vector pointer update chain. */
3735 dataref_ptr = vect_create_data_ref_ptr (first_stmt,
3737 &dummy, &ptr_incr, false,
3741 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
3743 for (i = 0; i < vec_num; i++)
3746 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
3749 /* 2. Create the vector-load in the loop. */
3750 switch (alignment_support_scheme)
3753 case dr_unaligned_supported:
3755 struct ptr_info_def *pi;
3757 = build2 (MEM_REF, vectype, dataref_ptr,
3758 build_int_cst (reference_alias_ptr_type
3759 (DR_REF (first_dr)), 0));
3760 pi = get_ptr_info (dataref_ptr);
3761 pi->align = TYPE_ALIGN_UNIT (vectype);
3762 if (alignment_support_scheme == dr_aligned)
3764 gcc_assert (aligned_access_p (first_dr));
3767 else if (DR_MISALIGNMENT (first_dr) == -1)
3769 TREE_TYPE (data_ref)
3770 = build_aligned_type (TREE_TYPE (data_ref),
3771 TYPE_ALIGN (TREE_TYPE (vectype)));
3772 pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
3777 TREE_TYPE (data_ref)
3778 = build_aligned_type (TREE_TYPE (data_ref),
3779 TYPE_ALIGN (TREE_TYPE (vectype)));
3780 pi->misalign = DR_MISALIGNMENT (first_dr);
3784 case dr_explicit_realign:
3787 tree vs_minus_1 = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
3789 if (compute_in_loop)
3790 msq = vect_setup_realignment (first_stmt, gsi,
3792 dr_explicit_realign,
3795 new_stmt = gimple_build_assign_with_ops
3796 (BIT_AND_EXPR, NULL_TREE, dataref_ptr,
3798 (TREE_TYPE (dataref_ptr),
3799 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
3800 ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
3801 gimple_assign_set_lhs (new_stmt, ptr);
3802 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3804 = build2 (MEM_REF, vectype, ptr,
3805 build_int_cst (reference_alias_ptr_type
3806 (DR_REF (first_dr)), 0));
3807 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3808 new_stmt = gimple_build_assign (vec_dest, data_ref);
3809 new_temp = make_ssa_name (vec_dest, new_stmt);
3810 gimple_assign_set_lhs (new_stmt, new_temp);
3811 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
3812 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
3813 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3816 bump = size_binop (MULT_EXPR, vs_minus_1,
3817 TYPE_SIZE_UNIT (scalar_type));
3818 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
3819 new_stmt = gimple_build_assign_with_ops
3820 (BIT_AND_EXPR, NULL_TREE, ptr,
3823 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
3824 ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
3825 gimple_assign_set_lhs (new_stmt, ptr);
3826 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3828 = build2 (MEM_REF, vectype, ptr,
3829 build_int_cst (reference_alias_ptr_type
3830 (DR_REF (first_dr)), 0));
3833 case dr_explicit_realign_optimized:
3834 new_stmt = gimple_build_assign_with_ops
3835 (BIT_AND_EXPR, NULL_TREE, dataref_ptr,
3837 (TREE_TYPE (dataref_ptr),
3838 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
3839 new_temp = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
3840 gimple_assign_set_lhs (new_stmt, new_temp);
3841 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3843 = build2 (MEM_REF, vectype, new_temp,
3844 build_int_cst (reference_alias_ptr_type
3845 (DR_REF (first_dr)), 0));
3850 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3851 new_stmt = gimple_build_assign (vec_dest, data_ref);
3852 new_temp = make_ssa_name (vec_dest, new_stmt);
3853 gimple_assign_set_lhs (new_stmt, new_temp);
3854 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3855 mark_symbols_for_renaming (new_stmt);
3857 /* 3. Handle explicit realignment if necessary/supported. Create in
3858 loop: vec_dest = realign_load (msq, lsq, realignment_token) */
3859 if (alignment_support_scheme == dr_explicit_realign_optimized
3860 || alignment_support_scheme == dr_explicit_realign)
3864 lsq = gimple_assign_lhs (new_stmt);
3865 if (!realignment_token)
3866 realignment_token = dataref_ptr;
3867 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3868 tmp = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq,
3870 new_stmt = gimple_build_assign (vec_dest, tmp);
3871 new_temp = make_ssa_name (vec_dest, new_stmt);
3872 gimple_assign_set_lhs (new_stmt, new_temp);
3873 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3875 if (alignment_support_scheme == dr_explicit_realign_optimized)
3878 if (i == vec_num - 1 && j == ncopies - 1)
3879 add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
3885 /* 4. Handle invariant-load. */
3886 if (inv_p && !bb_vinfo)
3888 gcc_assert (!strided_load);
3889 gcc_assert (nested_in_vect_loop_p (loop, stmt));
3894 tree vec_inv, bitpos, bitsize = TYPE_SIZE (scalar_type);
3896 /* CHECKME: bitpos depends on endianess? */
3897 bitpos = bitsize_zero_node;
3898 vec_inv = build3 (BIT_FIELD_REF, scalar_type, new_temp,
3901 vect_create_destination_var (scalar_dest, NULL_TREE);
3902 new_stmt = gimple_build_assign (vec_dest, vec_inv);
3903 new_temp = make_ssa_name (vec_dest, new_stmt);
3904 gimple_assign_set_lhs (new_stmt, new_temp);
3905 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3907 for (k = nunits - 1; k >= 0; --k)
3908 t = tree_cons (NULL_TREE, new_temp, t);
3909 /* FIXME: use build_constructor directly. */
3910 vec_inv = build_constructor_from_list (vectype, t);
3911 new_temp = vect_init_vector (stmt, vec_inv, vectype, gsi);
3912 new_stmt = SSA_NAME_DEF_STMT (new_temp);
3915 gcc_unreachable (); /* FORNOW. */
3918 /* Collect vector loads and later create their permutation in
3919 vect_transform_strided_load (). */
3920 if (strided_load || slp_perm)
3921 VEC_quick_push (tree, dr_chain, new_temp);
3923 /* Store vector loads in the corresponding SLP_NODE. */
3924 if (slp && !slp_perm)
3925 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
3928 if (slp && !slp_perm)
3933 if (!vect_transform_slp_perm_load (stmt, dr_chain, gsi, vf,
3934 slp_node_instance, false))
3936 VEC_free (tree, heap, dr_chain);
3944 if (!vect_transform_strided_load (stmt, dr_chain, group_size, gsi))
3947 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3948 VEC_free (tree, heap, dr_chain);
3949 dr_chain = VEC_alloc (tree, heap, group_size);
3954 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3956 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3957 prev_stmt_info = vinfo_for_stmt (new_stmt);
3963 VEC_free (tree, heap, dr_chain);
3968 /* Function vect_is_simple_cond.
3971 LOOP - the loop that is being vectorized.
3972 COND - Condition that is checked for simple use.
3974 Returns whether a COND can be vectorized. Checks whether
3975 condition operands are supportable using vec_is_simple_use. */
3978 vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
3982 enum vect_def_type dt;
3984 if (!COMPARISON_CLASS_P (cond))
3987 lhs = TREE_OPERAND (cond, 0);
3988 rhs = TREE_OPERAND (cond, 1);
3990 if (TREE_CODE (lhs) == SSA_NAME)
3992 gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
3993 if (!vect_is_simple_use (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
3997 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
3998 && TREE_CODE (lhs) != FIXED_CST)
4001 if (TREE_CODE (rhs) == SSA_NAME)
4003 gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
4004 if (!vect_is_simple_use (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
4008 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
4009 && TREE_CODE (rhs) != FIXED_CST)
4015 /* vectorizable_condition.
4017 Check if STMT is conditional modify expression that can be vectorized.
4018 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4019 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
4022 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
4023 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
4024 else caluse if it is 2).
4026 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4029 vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
4030 gimple *vec_stmt, tree reduc_def, int reduc_index)
4032 tree scalar_dest = NULL_TREE;
4033 tree vec_dest = NULL_TREE;
4034 tree op = NULL_TREE;
4035 tree cond_expr, then_clause, else_clause;
4036 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4037 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4038 tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause;
4039 tree vec_compare, vec_cond_expr;
4041 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4042 enum machine_mode vec_mode;
4044 enum vect_def_type dt, dts[4];
4045 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
4046 int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
4047 enum tree_code code;
4048 stmt_vec_info prev_stmt_info = NULL;
4051 /* FORNOW: unsupported in basic block SLP. */
4052 gcc_assert (loop_vinfo);
4054 gcc_assert (ncopies >= 1);
4055 if (reduc_index && ncopies > 1)
4056 return false; /* FORNOW */
4058 if (!STMT_VINFO_RELEVANT_P (stmt_info))
4061 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4062 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
4066 /* FORNOW: SLP not supported. */
4067 if (STMT_SLP_TYPE (stmt_info))
4070 /* FORNOW: not yet supported. */
4071 if (STMT_VINFO_LIVE_P (stmt_info))
4073 if (vect_print_dump_info (REPORT_DETAILS))
4074 fprintf (vect_dump, "value used after loop.");
4078 /* Is vectorizable conditional operation? */
4079 if (!is_gimple_assign (stmt))
4082 code = gimple_assign_rhs_code (stmt);
4084 if (code != COND_EXPR)
4087 gcc_assert (gimple_assign_single_p (stmt));
4088 op = gimple_assign_rhs1 (stmt);
4089 cond_expr = TREE_OPERAND (op, 0);
4090 then_clause = TREE_OPERAND (op, 1);
4091 else_clause = TREE_OPERAND (op, 2);
4093 if (!vect_is_simple_cond (cond_expr, loop_vinfo))
4096 /* We do not handle two different vector types for the condition
4098 if (!types_compatible_p (TREE_TYPE (TREE_OPERAND (cond_expr, 0)),
4099 TREE_TYPE (vectype)))
4102 if (TREE_CODE (then_clause) == SSA_NAME)
4104 gimple then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
4105 if (!vect_is_simple_use (then_clause, loop_vinfo, NULL,
4106 &then_def_stmt, &def, &dt))
4109 else if (TREE_CODE (then_clause) != INTEGER_CST
4110 && TREE_CODE (then_clause) != REAL_CST
4111 && TREE_CODE (then_clause) != FIXED_CST)
4114 if (TREE_CODE (else_clause) == SSA_NAME)
4116 gimple else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
4117 if (!vect_is_simple_use (else_clause, loop_vinfo, NULL,
4118 &else_def_stmt, &def, &dt))
4121 else if (TREE_CODE (else_clause) != INTEGER_CST
4122 && TREE_CODE (else_clause) != REAL_CST
4123 && TREE_CODE (else_clause) != FIXED_CST)
4127 vec_mode = TYPE_MODE (vectype);
4131 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
4132 return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
4138 scalar_dest = gimple_assign_lhs (stmt);
4139 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4141 /* Handle cond expr. */
4142 for (j = 0; j < ncopies; j++)
4149 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
4151 vect_is_simple_use (TREE_OPERAND (cond_expr, 0), loop_vinfo,
4152 NULL, >emp, &def, &dts[0]);
4154 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
4156 vect_is_simple_use (TREE_OPERAND (cond_expr, 1), loop_vinfo,
4157 NULL, >emp, &def, &dts[1]);
4158 if (reduc_index == 1)
4159 vec_then_clause = reduc_def;
4162 vec_then_clause = vect_get_vec_def_for_operand (then_clause,
4164 vect_is_simple_use (then_clause, loop_vinfo,
4165 NULL, >emp, &def, &dts[2]);
4167 if (reduc_index == 2)
4168 vec_else_clause = reduc_def;
4171 vec_else_clause = vect_get_vec_def_for_operand (else_clause,
4173 vect_is_simple_use (else_clause, loop_vinfo,
4174 NULL, >emp, &def, &dts[3]);
4179 vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], vec_cond_lhs);
4180 vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], vec_cond_rhs);
4181 vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
4183 vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
4187 /* Arguments are ready. Create the new vector stmt. */
4188 vec_compare = build2 (TREE_CODE (cond_expr), vectype,
4189 vec_cond_lhs, vec_cond_rhs);
4190 vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
4191 vec_compare, vec_then_clause, vec_else_clause);
4193 new_stmt = gimple_build_assign (vec_dest, vec_cond_expr);
4194 new_temp = make_ssa_name (vec_dest, new_stmt);
4195 gimple_assign_set_lhs (new_stmt, new_temp);
4196 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4198 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4200 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4202 prev_stmt_info = vinfo_for_stmt (new_stmt);
4209 /* Make sure the statement is vectorizable. */
4212 vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
4214 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4215 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4216 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
4218 tree scalar_type, vectype;
4220 if (vect_print_dump_info (REPORT_DETAILS))
4222 fprintf (vect_dump, "==> examining statement: ");
4223 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4226 if (gimple_has_volatile_ops (stmt))
4228 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4229 fprintf (vect_dump, "not vectorized: stmt has volatile operands");
4234 /* Skip stmts that do not need to be vectorized. In loops this is expected
4236 - the COND_EXPR which is the loop exit condition
4237 - any LABEL_EXPRs in the loop
4238 - computations that are used only for array indexing or loop control.
4239 In basic blocks we only analyze statements that are a part of some SLP
4240 instance, therefore, all the statements are relevant. */
4242 if (!STMT_VINFO_RELEVANT_P (stmt_info)
4243 && !STMT_VINFO_LIVE_P (stmt_info))
4245 if (vect_print_dump_info (REPORT_DETAILS))
4246 fprintf (vect_dump, "irrelevant.");
4251 switch (STMT_VINFO_DEF_TYPE (stmt_info))
4253 case vect_internal_def:
4256 case vect_reduction_def:
4257 case vect_nested_cycle:
4258 gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
4259 || relevance == vect_used_in_outer_by_reduction
4260 || relevance == vect_unused_in_scope));
4263 case vect_induction_def:
4264 case vect_constant_def:
4265 case vect_external_def:
4266 case vect_unknown_def_type:
4273 gcc_assert (PURE_SLP_STMT (stmt_info));
4275 scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
4276 if (vect_print_dump_info (REPORT_DETAILS))
4278 fprintf (vect_dump, "get vectype for scalar type: ");
4279 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4282 vectype = get_vectype_for_scalar_type (scalar_type);
4285 if (vect_print_dump_info (REPORT_DETAILS))
4287 fprintf (vect_dump, "not SLPed: unsupported data-type ");
4288 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4293 if (vect_print_dump_info (REPORT_DETAILS))
4295 fprintf (vect_dump, "vectype: ");
4296 print_generic_expr (vect_dump, vectype, TDF_SLIM);
4299 STMT_VINFO_VECTYPE (stmt_info) = vectype;
4302 if (STMT_VINFO_RELEVANT_P (stmt_info))
4304 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
4305 gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
4306 *need_to_vectorize = true;
4311 && (STMT_VINFO_RELEVANT_P (stmt_info)
4312 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
4313 ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
4314 || vectorizable_type_demotion (stmt, NULL, NULL, NULL)
4315 || vectorizable_conversion (stmt, NULL, NULL, NULL)
4316 || vectorizable_operation (stmt, NULL, NULL, NULL)
4317 || vectorizable_assignment (stmt, NULL, NULL, NULL)
4318 || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
4319 || vectorizable_call (stmt, NULL, NULL)
4320 || vectorizable_store (stmt, NULL, NULL, NULL)
4321 || vectorizable_reduction (stmt, NULL, NULL, NULL)
4322 || vectorizable_condition (stmt, NULL, NULL, NULL, 0));
4326 ok = (vectorizable_operation (stmt, NULL, NULL, node)
4327 || vectorizable_assignment (stmt, NULL, NULL, node)
4328 || vectorizable_load (stmt, NULL, NULL, node, NULL)
4329 || vectorizable_store (stmt, NULL, NULL, node));
4334 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4336 fprintf (vect_dump, "not vectorized: relevant stmt not ");
4337 fprintf (vect_dump, "supported: ");
4338 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4347 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
4348 need extra handling, except for vectorizable reductions. */
4349 if (STMT_VINFO_LIVE_P (stmt_info)
4350 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4351 ok = vectorizable_live_operation (stmt, NULL, NULL);
4355 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4357 fprintf (vect_dump, "not vectorized: live stmt not ");
4358 fprintf (vect_dump, "supported: ");
4359 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4365 if (!PURE_SLP_STMT (stmt_info))
4367 /* Groups of strided accesses whose size is not a power of 2 are not
4368 vectorizable yet using loop-vectorization. Therefore, if this stmt
4369 feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
4370 loop-based vectorized), the loop cannot be vectorized. */
4371 if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
4372 && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
4373 DR_GROUP_FIRST_DR (stmt_info)))) == -1)
4375 if (vect_print_dump_info (REPORT_DETAILS))
4377 fprintf (vect_dump, "not vectorized: the size of group "
4378 "of strided accesses is not a power of 2");
4379 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4390 /* Function vect_transform_stmt.
4392 Create a vectorized stmt to replace STMT, and insert it at BSI. */
4395 vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi,
4396 bool *strided_store, slp_tree slp_node,
4397 slp_instance slp_node_instance)
4399 bool is_store = false;
4400 gimple vec_stmt = NULL;
4401 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4402 gimple orig_stmt_in_pattern;
4405 switch (STMT_VINFO_TYPE (stmt_info))
4407 case type_demotion_vec_info_type:
4408 done = vectorizable_type_demotion (stmt, gsi, &vec_stmt, slp_node);
4412 case type_promotion_vec_info_type:
4413 done = vectorizable_type_promotion (stmt, gsi, &vec_stmt, slp_node);
4417 case type_conversion_vec_info_type:
4418 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
4422 case induc_vec_info_type:
4423 gcc_assert (!slp_node);
4424 done = vectorizable_induction (stmt, gsi, &vec_stmt);
4428 case op_vec_info_type:
4429 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
4433 case assignment_vec_info_type:
4434 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
4438 case load_vec_info_type:
4439 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
4444 case store_vec_info_type:
4445 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
4447 if (STMT_VINFO_STRIDED_ACCESS (stmt_info) && !slp_node)
4449 /* In case of interleaving, the whole chain is vectorized when the
4450 last store in the chain is reached. Store stmts before the last
4451 one are skipped, and there vec_stmt_info shouldn't be freed
4453 *strided_store = true;
4454 if (STMT_VINFO_VEC_STMT (stmt_info))
4461 case condition_vec_info_type:
4462 gcc_assert (!slp_node);
4463 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0);
4467 case call_vec_info_type:
4468 gcc_assert (!slp_node);
4469 done = vectorizable_call (stmt, gsi, &vec_stmt);
4472 case reduc_vec_info_type:
4473 done = vectorizable_reduction (stmt, gsi, &vec_stmt, slp_node);
4478 if (!STMT_VINFO_LIVE_P (stmt_info))
4480 if (vect_print_dump_info (REPORT_DETAILS))
4481 fprintf (vect_dump, "stmt not supported.");
4486 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
4487 is being vectorized, but outside the immediately enclosing loop. */
4489 && STMT_VINFO_LOOP_VINFO (stmt_info)
4490 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
4491 STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
4492 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
4493 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
4494 || STMT_VINFO_RELEVANT (stmt_info) ==
4495 vect_used_in_outer_by_reduction))
4497 struct loop *innerloop = LOOP_VINFO_LOOP (
4498 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
4499 imm_use_iterator imm_iter;
4500 use_operand_p use_p;
4504 if (vect_print_dump_info (REPORT_DETAILS))
4505 fprintf (vect_dump, "Record the vdef for outer-loop vectorization.");
4507 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
4508 (to be used when vectorizing outer-loop stmts that use the DEF of
4510 if (gimple_code (stmt) == GIMPLE_PHI)
4511 scalar_dest = PHI_RESULT (stmt);
4513 scalar_dest = gimple_assign_lhs (stmt);
4515 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
4517 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
4519 exit_phi = USE_STMT (use_p);
4520 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
4525 /* Handle stmts whose DEF is used outside the loop-nest that is
4526 being vectorized. */
4527 if (STMT_VINFO_LIVE_P (stmt_info)
4528 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4530 done = vectorizable_live_operation (stmt, gsi, &vec_stmt);
4536 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
4537 orig_stmt_in_pattern = STMT_VINFO_RELATED_STMT (stmt_info);
4538 if (orig_stmt_in_pattern)
4540 stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt_in_pattern);
4541 /* STMT was inserted by the vectorizer to replace a computation idiom.
4542 ORIG_STMT_IN_PATTERN is a stmt in the original sequence that
4543 computed this idiom. We need to record a pointer to VEC_STMT in
4544 the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the
4545 documentation of vect_pattern_recog. */
4546 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
4548 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
4549 STMT_VINFO_VEC_STMT (stmt_vinfo) = vec_stmt;
4558 /* Remove a group of stores (for SLP or interleaving), free their
4562 vect_remove_stores (gimple first_stmt)
4564 gimple next = first_stmt;
4566 gimple_stmt_iterator next_si;
4570 /* Free the attached stmt_vec_info and remove the stmt. */
4571 next_si = gsi_for_stmt (next);
4572 gsi_remove (&next_si, true);
4573 tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
4574 free_stmt_vec_info (next);
4580 /* Function new_stmt_vec_info.
4582 Create and initialize a new stmt_vec_info struct for STMT. */
4585 new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
4586 bb_vec_info bb_vinfo)
4589 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
4591 STMT_VINFO_TYPE (res) = undef_vec_info_type;
4592 STMT_VINFO_STMT (res) = stmt;
4593 STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
4594 STMT_VINFO_BB_VINFO (res) = bb_vinfo;
4595 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
4596 STMT_VINFO_LIVE_P (res) = false;
4597 STMT_VINFO_VECTYPE (res) = NULL;
4598 STMT_VINFO_VEC_STMT (res) = NULL;
4599 STMT_VINFO_VECTORIZABLE (res) = true;
4600 STMT_VINFO_IN_PATTERN_P (res) = false;
4601 STMT_VINFO_RELATED_STMT (res) = NULL;
4602 STMT_VINFO_DATA_REF (res) = NULL;
4604 STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
4605 STMT_VINFO_DR_OFFSET (res) = NULL;
4606 STMT_VINFO_DR_INIT (res) = NULL;
4607 STMT_VINFO_DR_STEP (res) = NULL;
4608 STMT_VINFO_DR_ALIGNED_TO (res) = NULL;
4610 if (gimple_code (stmt) == GIMPLE_PHI
4611 && is_loop_header_bb_p (gimple_bb (stmt)))
4612 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
4614 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
4616 STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5);
4617 STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0;
4618 STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0;
4619 STMT_SLP_TYPE (res) = loop_vect;
4620 DR_GROUP_FIRST_DR (res) = NULL;
4621 DR_GROUP_NEXT_DR (res) = NULL;
4622 DR_GROUP_SIZE (res) = 0;
4623 DR_GROUP_STORE_COUNT (res) = 0;
4624 DR_GROUP_GAP (res) = 0;
4625 DR_GROUP_SAME_DR_STMT (res) = NULL;
4626 DR_GROUP_READ_WRITE_DEPENDENCE (res) = false;
4632 /* Create a hash table for stmt_vec_info. */
4635 init_stmt_vec_info_vec (void)
4637 gcc_assert (!stmt_vec_info_vec);
4638 stmt_vec_info_vec = VEC_alloc (vec_void_p, heap, 50);
4642 /* Free hash table for stmt_vec_info. */
4645 free_stmt_vec_info_vec (void)
4647 gcc_assert (stmt_vec_info_vec);
4648 VEC_free (vec_void_p, heap, stmt_vec_info_vec);
4652 /* Free stmt vectorization related info. */
4655 free_stmt_vec_info (gimple stmt)
4657 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4662 VEC_free (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmt_info));
4663 set_vinfo_for_stmt (stmt, NULL);
4668 /* Function get_vectype_for_scalar_type.
4670 Returns the vector type corresponding to SCALAR_TYPE as supported
4674 get_vectype_for_scalar_type (tree scalar_type)
4676 enum machine_mode inner_mode = TYPE_MODE (scalar_type);
4677 unsigned int nbytes = GET_MODE_SIZE (inner_mode);
4682 || (nbytes >= targetm.vectorize.units_per_simd_word (inner_mode)))
4685 /* We can't build a vector type of elements with alignment bigger than
4687 if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
4690 /* If we'd build a vector type of elements whose mode precision doesn't
4691 match their types precision we'll get mismatched types on vector
4692 extracts via BIT_FIELD_REFs. This effectively means we disable
4693 vectorization of bool and/or enum types in some languages. */
4694 if (INTEGRAL_TYPE_P (scalar_type)
4695 && GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type))
4698 /* FORNOW: Only a single vector size per mode
4699 (TARGET_VECTORIZE_UNITS_PER_SIMD_WORD) is expected. */
4700 nunits = targetm.vectorize.units_per_simd_word (inner_mode) / nbytes;
4702 vectype = build_vector_type (scalar_type, nunits);
4703 if (vect_print_dump_info (REPORT_DETAILS))
4705 fprintf (vect_dump, "get vectype with %d units of type ", nunits);
4706 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4712 if (vect_print_dump_info (REPORT_DETAILS))
4714 fprintf (vect_dump, "vectype: ");
4715 print_generic_expr (vect_dump, vectype, TDF_SLIM);
4718 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
4719 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
4721 if (vect_print_dump_info (REPORT_DETAILS))
4722 fprintf (vect_dump, "mode not supported by target.");
4729 /* Function get_same_sized_vectype
4731 Returns a vector type corresponding to SCALAR_TYPE of size
4732 VECTOR_TYPE if supported by the target. */
4735 get_same_sized_vectype (tree scalar_type, tree vector_type ATTRIBUTE_UNUSED)
4737 return get_vectype_for_scalar_type (scalar_type);
4740 /* Function vect_is_simple_use.
4743 LOOP_VINFO - the vect info of the loop that is being vectorized.
4744 BB_VINFO - the vect info of the basic block that is being vectorized.
4745 OPERAND - operand of a stmt in the loop or bb.
4746 DEF - the defining stmt in case OPERAND is an SSA_NAME.
4748 Returns whether a stmt with OPERAND can be vectorized.
4749 For loops, supportable operands are constants, loop invariants, and operands
4750 that are defined by the current iteration of the loop. Unsupportable
4751 operands are those that are defined by a previous iteration of the loop (as
4752 is the case in reduction/induction computations).
4753 For basic blocks, supportable operands are constants and bb invariants.
4754 For now, operands defined outside the basic block are not supported. */
4757 vect_is_simple_use (tree operand, loop_vec_info loop_vinfo,
4758 bb_vec_info bb_vinfo, gimple *def_stmt,
4759 tree *def, enum vect_def_type *dt)
4762 stmt_vec_info stmt_vinfo;
4763 struct loop *loop = NULL;
4766 loop = LOOP_VINFO_LOOP (loop_vinfo);
4771 if (vect_print_dump_info (REPORT_DETAILS))
4773 fprintf (vect_dump, "vect_is_simple_use: operand ");
4774 print_generic_expr (vect_dump, operand, TDF_SLIM);
4777 if (TREE_CODE (operand) == INTEGER_CST || TREE_CODE (operand) == REAL_CST)
4779 *dt = vect_constant_def;
4783 if (is_gimple_min_invariant (operand))
4786 *dt = vect_external_def;
4790 if (TREE_CODE (operand) == PAREN_EXPR)
4792 if (vect_print_dump_info (REPORT_DETAILS))
4793 fprintf (vect_dump, "non-associatable copy.");
4794 operand = TREE_OPERAND (operand, 0);
4797 if (TREE_CODE (operand) != SSA_NAME)
4799 if (vect_print_dump_info (REPORT_DETAILS))
4800 fprintf (vect_dump, "not ssa-name.");
4804 *def_stmt = SSA_NAME_DEF_STMT (operand);
4805 if (*def_stmt == NULL)
4807 if (vect_print_dump_info (REPORT_DETAILS))
4808 fprintf (vect_dump, "no def_stmt.");
4812 if (vect_print_dump_info (REPORT_DETAILS))
4814 fprintf (vect_dump, "def_stmt: ");
4815 print_gimple_stmt (vect_dump, *def_stmt, 0, TDF_SLIM);
4818 /* Empty stmt is expected only in case of a function argument.
4819 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
4820 if (gimple_nop_p (*def_stmt))
4823 *dt = vect_external_def;
4827 bb = gimple_bb (*def_stmt);
4829 if ((loop && !flow_bb_inside_loop_p (loop, bb))
4830 || (!loop && bb != BB_VINFO_BB (bb_vinfo))
4831 || (!loop && gimple_code (*def_stmt) == GIMPLE_PHI))
4832 *dt = vect_external_def;
4835 stmt_vinfo = vinfo_for_stmt (*def_stmt);
4836 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
4839 if (*dt == vect_unknown_def_type)
4841 if (vect_print_dump_info (REPORT_DETAILS))
4842 fprintf (vect_dump, "Unsupported pattern.");
4846 if (vect_print_dump_info (REPORT_DETAILS))
4847 fprintf (vect_dump, "type of def: %d.",*dt);
4849 switch (gimple_code (*def_stmt))
4852 *def = gimple_phi_result (*def_stmt);
4856 *def = gimple_assign_lhs (*def_stmt);
4860 *def = gimple_call_lhs (*def_stmt);
4865 if (vect_print_dump_info (REPORT_DETAILS))
4866 fprintf (vect_dump, "unsupported defining stmt: ");
4873 /* Function vect_is_simple_use_1.
4875 Same as vect_is_simple_use_1 but also determines the vector operand
4876 type of OPERAND and stores it to *VECTYPE. If the definition of
4877 OPERAND is vect_uninitialized_def, vect_constant_def or
4878 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
4879 is responsible to compute the best suited vector type for the
4883 vect_is_simple_use_1 (tree operand, loop_vec_info loop_vinfo,
4884 bb_vec_info bb_vinfo, gimple *def_stmt,
4885 tree *def, enum vect_def_type *dt, tree *vectype)
4887 if (!vect_is_simple_use (operand, loop_vinfo, bb_vinfo, def_stmt, def, dt))
4890 /* Now get a vector type if the def is internal, otherwise supply
4891 NULL_TREE and leave it up to the caller to figure out a proper
4892 type for the use stmt. */
4893 if (*dt == vect_internal_def
4894 || *dt == vect_induction_def
4895 || *dt == vect_reduction_def
4896 || *dt == vect_double_reduction_def
4897 || *dt == vect_nested_cycle)
4899 stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
4900 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
4901 stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
4902 *vectype = STMT_VINFO_VECTYPE (stmt_info);
4903 gcc_assert (*vectype != NULL_TREE);
4905 else if (*dt == vect_uninitialized_def
4906 || *dt == vect_constant_def
4907 || *dt == vect_external_def)
4908 *vectype = NULL_TREE;
4916 /* Function supportable_widening_operation
4918 Check whether an operation represented by the code CODE is a
4919 widening operation that is supported by the target platform in
4920 vector form (i.e., when operating on arguments of type VECTYPE_IN
4921 producing a result of type VECTYPE_OUT).
4923 Widening operations we currently support are NOP (CONVERT), FLOAT
4924 and WIDEN_MULT. This function checks if these operations are supported
4925 by the target platform either directly (via vector tree-codes), or via
4929 - CODE1 and CODE2 are codes of vector operations to be used when
4930 vectorizing the operation, if available.
4931 - DECL1 and DECL2 are decls of target builtin functions to be used
4932 when vectorizing the operation, if available. In this case,
4933 CODE1 and CODE2 are CALL_EXPR.
4934 - MULTI_STEP_CVT determines the number of required intermediate steps in
4935 case of multi-step conversion (like char->short->int - in that case
4936 MULTI_STEP_CVT will be 1).
4937 - INTERM_TYPES contains the intermediate type required to perform the
4938 widening operation (short in the above example). */
4941 supportable_widening_operation (enum tree_code code, gimple stmt,
4942 tree vectype_out, tree vectype_in,
4943 tree *decl1, tree *decl2,
4944 enum tree_code *code1, enum tree_code *code2,
4945 int *multi_step_cvt,
4946 VEC (tree, heap) **interm_types)
4948 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4949 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
4950 struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
4952 enum machine_mode vec_mode;
4953 enum insn_code icode1, icode2;
4954 optab optab1, optab2;
4955 tree vectype = vectype_in;
4956 tree wide_vectype = vectype_out;
4957 enum tree_code c1, c2;
4959 /* The result of a vectorized widening operation usually requires two vectors
4960 (because the widened results do not fit int one vector). The generated
4961 vector results would normally be expected to be generated in the same
4962 order as in the original scalar computation, i.e. if 8 results are
4963 generated in each vector iteration, they are to be organized as follows:
4964 vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
4966 However, in the special case that the result of the widening operation is
4967 used in a reduction computation only, the order doesn't matter (because
4968 when vectorizing a reduction we change the order of the computation).
4969 Some targets can take advantage of this and generate more efficient code.
4970 For example, targets like Altivec, that support widen_mult using a sequence
4971 of {mult_even,mult_odd} generate the following vectors:
4972 vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8].
4974 When vectorizing outer-loops, we execute the inner-loop sequentially
4975 (each vectorized inner-loop iteration contributes to VF outer-loop
4976 iterations in parallel). We therefore don't allow to change the order
4977 of the computation in the inner-loop during outer-loop vectorization. */
4979 if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
4980 && !nested_in_vect_loop_p (vect_loop, stmt))
4986 && code == WIDEN_MULT_EXPR
4987 && targetm.vectorize.builtin_mul_widen_even
4988 && targetm.vectorize.builtin_mul_widen_even (vectype)
4989 && targetm.vectorize.builtin_mul_widen_odd
4990 && targetm.vectorize.builtin_mul_widen_odd (vectype))
4992 if (vect_print_dump_info (REPORT_DETAILS))
4993 fprintf (vect_dump, "Unordered widening operation detected.");
4995 *code1 = *code2 = CALL_EXPR;
4996 *decl1 = targetm.vectorize.builtin_mul_widen_even (vectype);
4997 *decl2 = targetm.vectorize.builtin_mul_widen_odd (vectype);
5003 case WIDEN_MULT_EXPR:
5004 if (BYTES_BIG_ENDIAN)
5006 c1 = VEC_WIDEN_MULT_HI_EXPR;
5007 c2 = VEC_WIDEN_MULT_LO_EXPR;
5011 c2 = VEC_WIDEN_MULT_HI_EXPR;
5012 c1 = VEC_WIDEN_MULT_LO_EXPR;
5017 if (BYTES_BIG_ENDIAN)
5019 c1 = VEC_UNPACK_HI_EXPR;
5020 c2 = VEC_UNPACK_LO_EXPR;
5024 c2 = VEC_UNPACK_HI_EXPR;
5025 c1 = VEC_UNPACK_LO_EXPR;
5030 if (BYTES_BIG_ENDIAN)
5032 c1 = VEC_UNPACK_FLOAT_HI_EXPR;
5033 c2 = VEC_UNPACK_FLOAT_LO_EXPR;
5037 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
5038 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
5042 case FIX_TRUNC_EXPR:
5043 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
5044 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
5045 computing the operation. */
5052 if (code == FIX_TRUNC_EXPR)
5054 /* The signedness is determined from output operand. */
5055 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
5056 optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
5060 optab1 = optab_for_tree_code (c1, vectype, optab_default);
5061 optab2 = optab_for_tree_code (c2, vectype, optab_default);
5064 if (!optab1 || !optab2)
5067 vec_mode = TYPE_MODE (vectype);
5068 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
5069 || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
5072 /* Check if it's a multi-step conversion that can be done using intermediate
5074 if (insn_data[icode1].operand[0].mode != TYPE_MODE (wide_vectype)
5075 || insn_data[icode2].operand[0].mode != TYPE_MODE (wide_vectype))
5078 tree prev_type = vectype, intermediate_type;
5079 enum machine_mode intermediate_mode, prev_mode = vec_mode;
5080 optab optab3, optab4;
5082 if (!CONVERT_EXPR_CODE_P (code))
5088 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
5089 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
5090 to get to NARROW_VECTYPE, and fail if we do not. */
5091 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
5092 for (i = 0; i < 3; i++)
5094 intermediate_mode = insn_data[icode1].operand[0].mode;
5095 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
5096 TYPE_UNSIGNED (prev_type));
5097 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
5098 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
5100 if (!optab3 || !optab4
5101 || ((icode1 = optab_handler (optab1, prev_mode))
5102 == CODE_FOR_nothing)
5103 || insn_data[icode1].operand[0].mode != intermediate_mode
5104 || ((icode2 = optab_handler (optab2, prev_mode))
5105 == CODE_FOR_nothing)
5106 || insn_data[icode2].operand[0].mode != intermediate_mode
5107 || ((icode1 = optab_handler (optab3, intermediate_mode))
5108 == CODE_FOR_nothing)
5109 || ((icode2 = optab_handler (optab4, intermediate_mode))
5110 == CODE_FOR_nothing))
5113 VEC_quick_push (tree, *interm_types, intermediate_type);
5114 (*multi_step_cvt)++;
5116 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
5117 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
5120 prev_type = intermediate_type;
5121 prev_mode = intermediate_mode;
5133 /* Function supportable_narrowing_operation
5135 Check whether an operation represented by the code CODE is a
5136 narrowing operation that is supported by the target platform in
5137 vector form (i.e., when operating on arguments of type VECTYPE_IN
5138 and producing a result of type VECTYPE_OUT).
5140 Narrowing operations we currently support are NOP (CONVERT) and
5141 FIX_TRUNC. This function checks if these operations are supported by
5142 the target platform directly via vector tree-codes.
5145 - CODE1 is the code of a vector operation to be used when
5146 vectorizing the operation, if available.
5147 - MULTI_STEP_CVT determines the number of required intermediate steps in
5148 case of multi-step conversion (like int->short->char - in that case
5149 MULTI_STEP_CVT will be 1).
5150 - INTERM_TYPES contains the intermediate type required to perform the
5151 narrowing operation (short in the above example). */
5154 supportable_narrowing_operation (enum tree_code code,
5155 tree vectype_out, tree vectype_in,
5156 enum tree_code *code1, int *multi_step_cvt,
5157 VEC (tree, heap) **interm_types)
5159 enum machine_mode vec_mode;
5160 enum insn_code icode1;
5161 optab optab1, interm_optab;
5162 tree vectype = vectype_in;
5163 tree narrow_vectype = vectype_out;
5165 tree intermediate_type, prev_type;
5171 c1 = VEC_PACK_TRUNC_EXPR;
5174 case FIX_TRUNC_EXPR:
5175 c1 = VEC_PACK_FIX_TRUNC_EXPR;
5179 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
5180 tree code and optabs used for computing the operation. */
5187 if (code == FIX_TRUNC_EXPR)
5188 /* The signedness is determined from output operand. */
5189 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
5191 optab1 = optab_for_tree_code (c1, vectype, optab_default);
5196 vec_mode = TYPE_MODE (vectype);
5197 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
5200 /* Check if it's a multi-step conversion that can be done using intermediate
5202 if (insn_data[icode1].operand[0].mode != TYPE_MODE (narrow_vectype))
5204 enum machine_mode intermediate_mode, prev_mode = vec_mode;
5207 prev_type = vectype;
5208 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
5209 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
5210 to get to NARROW_VECTYPE, and fail if we do not. */
5211 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
5212 for (i = 0; i < 3; i++)
5214 intermediate_mode = insn_data[icode1].operand[0].mode;
5215 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
5216 TYPE_UNSIGNED (prev_type));
5217 interm_optab = optab_for_tree_code (c1, intermediate_type,
5220 || ((icode1 = optab_handler (optab1, prev_mode))
5221 == CODE_FOR_nothing)
5222 || insn_data[icode1].operand[0].mode != intermediate_mode
5223 || ((icode1 = optab_handler (interm_optab, intermediate_mode))
5224 == CODE_FOR_nothing))
5227 VEC_quick_push (tree, *interm_types, intermediate_type);
5228 (*multi_step_cvt)++;
5230 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
5233 prev_type = intermediate_type;
5234 prev_mode = intermediate_mode;