1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "tree-pretty-print.h"
32 #include "gimple-pretty-print.h"
33 #include "tree-flow.h"
34 #include "tree-dump.h"
36 #include "cfglayout.h"
40 #include "diagnostic-core.h"
41 #include "tree-vectorizer.h"
42 #include "langhooks.h"
45 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
47 /* Function vect_mark_relevant.
49 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
52 vect_mark_relevant (VEC(gimple,heap) **worklist, gimple stmt,
53 enum vect_relevant relevant, bool live_p)
55 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
56 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
57 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
59 if (vect_print_dump_info (REPORT_DETAILS))
60 fprintf (vect_dump, "mark relevant %d, live %d.", relevant, live_p);
62 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
66 /* This is the last stmt in a sequence that was detected as a
67 pattern that can potentially be vectorized. Don't mark the stmt
68 as relevant/live because it's not going to be vectorized.
69 Instead mark the pattern-stmt that replaces it. */
71 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
73 if (vect_print_dump_info (REPORT_DETAILS))
74 fprintf (vect_dump, "last stmt in pattern. don't mark relevant/live.");
75 stmt_info = vinfo_for_stmt (pattern_stmt);
76 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
77 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
78 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
82 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
83 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
84 STMT_VINFO_RELEVANT (stmt_info) = relevant;
86 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
87 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
89 if (vect_print_dump_info (REPORT_DETAILS))
90 fprintf (vect_dump, "already marked relevant/live.");
94 VEC_safe_push (gimple, heap, *worklist, stmt);
98 /* Function vect_stmt_relevant_p.
100 Return true if STMT in loop that is represented by LOOP_VINFO is
101 "relevant for vectorization".
103 A stmt is considered "relevant for vectorization" if:
104 - it has uses outside the loop.
105 - it has vdefs (it alters memory).
106 - control stmts in the loop (except for the exit condition).
108 CHECKME: what other side effects would the vectorizer allow? */
111 vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo,
112 enum vect_relevant *relevant, bool *live_p)
114 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
116 imm_use_iterator imm_iter;
120 *relevant = vect_unused_in_scope;
123 /* cond stmt other than loop exit cond. */
124 if (is_ctrl_stmt (stmt)
125 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
126 != loop_exit_ctrl_vec_info_type)
127 *relevant = vect_used_in_scope;
129 /* changing memory. */
130 if (gimple_code (stmt) != GIMPLE_PHI)
131 if (gimple_vdef (stmt))
133 if (vect_print_dump_info (REPORT_DETAILS))
134 fprintf (vect_dump, "vec_stmt_relevant_p: stmt has vdefs.");
135 *relevant = vect_used_in_scope;
138 /* uses outside the loop. */
139 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
141 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
143 basic_block bb = gimple_bb (USE_STMT (use_p));
144 if (!flow_bb_inside_loop_p (loop, bb))
146 if (vect_print_dump_info (REPORT_DETAILS))
147 fprintf (vect_dump, "vec_stmt_relevant_p: used out of loop.");
149 if (is_gimple_debug (USE_STMT (use_p)))
152 /* We expect all such uses to be in the loop exit phis
153 (because of loop closed form) */
154 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
155 gcc_assert (bb == single_exit (loop)->dest);
162 return (*live_p || *relevant);
166 /* Function exist_non_indexing_operands_for_use_p
168 USE is one of the uses attached to STMT. Check if USE is
169 used in STMT for anything other than indexing an array. */
172 exist_non_indexing_operands_for_use_p (tree use, gimple stmt)
175 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
177 /* USE corresponds to some operand in STMT. If there is no data
178 reference in STMT, then any operand that corresponds to USE
179 is not indexing an array. */
180 if (!STMT_VINFO_DATA_REF (stmt_info))
183 /* STMT has a data_ref. FORNOW this means that its of one of
187 (This should have been verified in analyze_data_refs).
189 'var' in the second case corresponds to a def, not a use,
190 so USE cannot correspond to any operands that are not used
193 Therefore, all we need to check is if STMT falls into the
194 first case, and whether var corresponds to USE. */
196 if (!gimple_assign_copy_p (stmt))
198 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
200 operand = gimple_assign_rhs1 (stmt);
201 if (TREE_CODE (operand) != SSA_NAME)
212 Function process_use.
215 - a USE in STMT in a loop represented by LOOP_VINFO
216 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
217 that defined USE. This is done by calling mark_relevant and passing it
218 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
221 Generally, LIVE_P and RELEVANT are used to define the liveness and
222 relevance info of the DEF_STMT of this USE:
223 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
224 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
226 - case 1: If USE is used only for address computations (e.g. array indexing),
227 which does not need to be directly vectorized, then the liveness/relevance
228 of the respective DEF_STMT is left unchanged.
229 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
230 skip DEF_STMT cause it had already been processed.
231 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
232 be modified accordingly.
234 Return true if everything is as expected. Return false otherwise. */
237 process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
238 enum vect_relevant relevant, VEC(gimple,heap) **worklist)
240 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
241 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
242 stmt_vec_info dstmt_vinfo;
243 basic_block bb, def_bb;
246 enum vect_def_type dt;
248 /* case 1: we are only interested in uses that need to be vectorized. Uses
249 that are used for address computation are not considered relevant. */
250 if (!exist_non_indexing_operands_for_use_p (use, stmt))
253 if (!vect_is_simple_use (use, loop_vinfo, NULL, &def_stmt, &def, &dt))
255 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
256 fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
260 if (!def_stmt || gimple_nop_p (def_stmt))
263 def_bb = gimple_bb (def_stmt);
264 if (!flow_bb_inside_loop_p (loop, def_bb))
266 if (vect_print_dump_info (REPORT_DETAILS))
267 fprintf (vect_dump, "def_stmt is out of loop.");
271 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
272 DEF_STMT must have already been processed, because this should be the
273 only way that STMT, which is a reduction-phi, was put in the worklist,
274 as there should be no other uses for DEF_STMT in the loop. So we just
275 check that everything is as expected, and we are done. */
276 dstmt_vinfo = vinfo_for_stmt (def_stmt);
277 bb = gimple_bb (stmt);
278 if (gimple_code (stmt) == GIMPLE_PHI
279 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
280 && gimple_code (def_stmt) != GIMPLE_PHI
281 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
282 && bb->loop_father == def_bb->loop_father)
284 if (vect_print_dump_info (REPORT_DETAILS))
285 fprintf (vect_dump, "reduc-stmt defining reduc-phi in the same nest.");
286 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
287 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
288 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
289 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
290 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
294 /* case 3a: outer-loop stmt defining an inner-loop stmt:
295 outer-loop-header-bb:
301 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
303 if (vect_print_dump_info (REPORT_DETAILS))
304 fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt.");
308 case vect_unused_in_scope:
309 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
310 vect_used_in_scope : vect_unused_in_scope;
313 case vect_used_in_outer_by_reduction:
314 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
315 relevant = vect_used_by_reduction;
318 case vect_used_in_outer:
319 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
320 relevant = vect_used_in_scope;
323 case vect_used_in_scope:
331 /* case 3b: inner-loop stmt defining an outer-loop stmt:
332 outer-loop-header-bb:
336 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
338 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
340 if (vect_print_dump_info (REPORT_DETAILS))
341 fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt.");
345 case vect_unused_in_scope:
346 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
347 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
348 vect_used_in_outer_by_reduction : vect_unused_in_scope;
351 case vect_used_by_reduction:
352 relevant = vect_used_in_outer_by_reduction;
355 case vect_used_in_scope:
356 relevant = vect_used_in_outer;
364 vect_mark_relevant (worklist, def_stmt, relevant, live_p);
369 /* Function vect_mark_stmts_to_be_vectorized.
371 Not all stmts in the loop need to be vectorized. For example:
380 Stmt 1 and 3 do not need to be vectorized, because loop control and
381 addressing of vectorized data-refs are handled differently.
383 This pass detects such stmts. */
386 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
388 VEC(gimple,heap) *worklist;
389 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
390 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
391 unsigned int nbbs = loop->num_nodes;
392 gimple_stmt_iterator si;
395 stmt_vec_info stmt_vinfo;
399 enum vect_relevant relevant, tmp_relevant;
400 enum vect_def_type def_type;
402 if (vect_print_dump_info (REPORT_DETAILS))
403 fprintf (vect_dump, "=== vect_mark_stmts_to_be_vectorized ===");
405 worklist = VEC_alloc (gimple, heap, 64);
407 /* 1. Init worklist. */
408 for (i = 0; i < nbbs; i++)
411 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
414 if (vect_print_dump_info (REPORT_DETAILS))
416 fprintf (vect_dump, "init: phi relevant? ");
417 print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
420 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
421 vect_mark_relevant (&worklist, phi, relevant, live_p);
423 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
425 stmt = gsi_stmt (si);
426 if (vect_print_dump_info (REPORT_DETAILS))
428 fprintf (vect_dump, "init: stmt relevant? ");
429 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
432 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
433 vect_mark_relevant (&worklist, stmt, relevant, live_p);
437 /* 2. Process_worklist */
438 while (VEC_length (gimple, worklist) > 0)
443 stmt = VEC_pop (gimple, worklist);
444 if (vect_print_dump_info (REPORT_DETAILS))
446 fprintf (vect_dump, "worklist: examine stmt: ");
447 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
450 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
451 (DEF_STMT) as relevant/irrelevant and live/dead according to the
452 liveness and relevance properties of STMT. */
453 stmt_vinfo = vinfo_for_stmt (stmt);
454 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
455 live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
457 /* Generally, the liveness and relevance properties of STMT are
458 propagated as is to the DEF_STMTs of its USEs:
459 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
460 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
462 One exception is when STMT has been identified as defining a reduction
463 variable; in this case we set the liveness/relevance as follows:
465 relevant = vect_used_by_reduction
466 This is because we distinguish between two kinds of relevant stmts -
467 those that are used by a reduction computation, and those that are
468 (also) used by a regular computation. This allows us later on to
469 identify stmts that are used solely by a reduction, and therefore the
470 order of the results that they produce does not have to be kept. */
472 def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
473 tmp_relevant = relevant;
476 case vect_reduction_def:
477 switch (tmp_relevant)
479 case vect_unused_in_scope:
480 relevant = vect_used_by_reduction;
483 case vect_used_by_reduction:
484 if (gimple_code (stmt) == GIMPLE_PHI)
489 if (vect_print_dump_info (REPORT_DETAILS))
490 fprintf (vect_dump, "unsupported use of reduction.");
492 VEC_free (gimple, heap, worklist);
499 case vect_nested_cycle:
500 if (tmp_relevant != vect_unused_in_scope
501 && tmp_relevant != vect_used_in_outer_by_reduction
502 && tmp_relevant != vect_used_in_outer)
504 if (vect_print_dump_info (REPORT_DETAILS))
505 fprintf (vect_dump, "unsupported use of nested cycle.");
507 VEC_free (gimple, heap, worklist);
514 case vect_double_reduction_def:
515 if (tmp_relevant != vect_unused_in_scope
516 && tmp_relevant != vect_used_by_reduction)
518 if (vect_print_dump_info (REPORT_DETAILS))
519 fprintf (vect_dump, "unsupported use of double reduction.");
521 VEC_free (gimple, heap, worklist);
532 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
534 tree op = USE_FROM_PTR (use_p);
535 if (!process_use (stmt, op, loop_vinfo, live_p, relevant, &worklist))
537 VEC_free (gimple, heap, worklist);
541 } /* while worklist */
543 VEC_free (gimple, heap, worklist);
548 /* Get cost by calling cost target builtin. */
551 int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost)
553 tree dummy_type = NULL;
556 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
561 /* Get cost for STMT. */
564 cost_for_stmt (gimple stmt)
566 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
568 switch (STMT_VINFO_TYPE (stmt_info))
570 case load_vec_info_type:
571 return vect_get_stmt_cost (scalar_load);
572 case store_vec_info_type:
573 return vect_get_stmt_cost (scalar_store);
574 case op_vec_info_type:
575 case condition_vec_info_type:
576 case assignment_vec_info_type:
577 case reduc_vec_info_type:
578 case induc_vec_info_type:
579 case type_promotion_vec_info_type:
580 case type_demotion_vec_info_type:
581 case type_conversion_vec_info_type:
582 case call_vec_info_type:
583 return vect_get_stmt_cost (scalar_stmt);
584 case undef_vec_info_type:
590 /* Function vect_model_simple_cost.
592 Models cost for simple operations, i.e. those that only emit ncopies of a
593 single op. Right now, this does not account for multiple insns that could
594 be generated for the single vector op. We will handle that shortly. */
597 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
598 enum vect_def_type *dt, slp_tree slp_node)
601 int inside_cost = 0, outside_cost = 0;
603 /* The SLP costs were already calculated during SLP tree build. */
604 if (PURE_SLP_STMT (stmt_info))
607 inside_cost = ncopies * vect_get_stmt_cost (vector_stmt);
609 /* FORNOW: Assuming maximum 2 args per stmts. */
610 for (i = 0; i < 2; i++)
612 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
613 outside_cost += vect_get_stmt_cost (vector_stmt);
616 if (vect_print_dump_info (REPORT_COST))
617 fprintf (vect_dump, "vect_model_simple_cost: inside_cost = %d, "
618 "outside_cost = %d .", inside_cost, outside_cost);
620 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
621 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
622 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
626 /* Function vect_cost_strided_group_size
628 For strided load or store, return the group_size only if it is the first
629 load or store of a group, else return 1. This ensures that group size is
630 only returned once per group. */
633 vect_cost_strided_group_size (stmt_vec_info stmt_info)
635 gimple first_stmt = DR_GROUP_FIRST_DR (stmt_info);
637 if (first_stmt == STMT_VINFO_STMT (stmt_info))
638 return DR_GROUP_SIZE (stmt_info);
644 /* Function vect_model_store_cost
646 Models cost for stores. In the case of strided accesses, one access
647 has the overhead of the strided access attributed to it. */
650 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
651 enum vect_def_type dt, slp_tree slp_node)
654 unsigned int inside_cost = 0, outside_cost = 0;
655 struct data_reference *first_dr;
658 /* The SLP costs were already calculated during SLP tree build. */
659 if (PURE_SLP_STMT (stmt_info))
662 if (dt == vect_constant_def || dt == vect_external_def)
663 outside_cost = vect_get_stmt_cost (scalar_to_vec);
665 /* Strided access? */
666 if (DR_GROUP_FIRST_DR (stmt_info))
670 first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
675 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
676 group_size = vect_cost_strided_group_size (stmt_info);
679 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
681 /* Not a strided access. */
685 first_dr = STMT_VINFO_DATA_REF (stmt_info);
688 /* Is this an access in a group of stores, which provide strided access?
689 If so, add in the cost of the permutes. */
692 /* Uses a high and low interleave operation for each needed permute. */
693 inside_cost = ncopies * exact_log2(group_size) * group_size
694 * vect_get_stmt_cost (vector_stmt);
696 if (vect_print_dump_info (REPORT_COST))
697 fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .",
702 /* Costs of the stores. */
703 vect_get_store_cost (first_dr, ncopies, &inside_cost);
705 if (vect_print_dump_info (REPORT_COST))
706 fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, "
707 "outside_cost = %d .", inside_cost, outside_cost);
709 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
710 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
711 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
715 /* Calculate cost of DR's memory access. */
717 vect_get_store_cost (struct data_reference *dr, int ncopies,
718 unsigned int *inside_cost)
720 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
722 switch (alignment_support_scheme)
726 *inside_cost += ncopies * vect_get_stmt_cost (vector_store);
728 if (vect_print_dump_info (REPORT_COST))
729 fprintf (vect_dump, "vect_model_store_cost: aligned.");
734 case dr_unaligned_supported:
736 gimple stmt = DR_STMT (dr);
737 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
738 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
740 /* Here, we assign an additional cost for the unaligned store. */
741 *inside_cost += ncopies
742 * targetm.vectorize.builtin_vectorization_cost (unaligned_store,
743 vectype, DR_MISALIGNMENT (dr));
745 if (vect_print_dump_info (REPORT_COST))
746 fprintf (vect_dump, "vect_model_store_cost: unaligned supported by "
758 /* Function vect_model_load_cost
760 Models cost for loads. In the case of strided accesses, the last access
761 has the overhead of the strided access attributed to it. Since unaligned
762 accesses are supported for loads, we also account for the costs of the
763 access scheme chosen. */
766 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node)
771 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
772 unsigned int inside_cost = 0, outside_cost = 0;
774 /* The SLP costs were already calculated during SLP tree build. */
775 if (PURE_SLP_STMT (stmt_info))
778 /* Strided accesses? */
779 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
780 if (first_stmt && !slp_node)
782 group_size = vect_cost_strided_group_size (stmt_info);
783 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
785 /* Not a strided access. */
792 /* Is this an access in a group of loads providing strided access?
793 If so, add in the cost of the permutes. */
796 /* Uses an even and odd extract operations for each needed permute. */
797 inside_cost = ncopies * exact_log2(group_size) * group_size
798 * vect_get_stmt_cost (vector_stmt);
800 if (vect_print_dump_info (REPORT_COST))
801 fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .",
805 /* The loads themselves. */
806 vect_get_load_cost (first_dr, ncopies,
807 ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node),
808 &inside_cost, &outside_cost);
810 if (vect_print_dump_info (REPORT_COST))
811 fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, "
812 "outside_cost = %d .", inside_cost, outside_cost);
814 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
815 stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
816 stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
820 /* Calculate cost of DR's memory access. */
822 vect_get_load_cost (struct data_reference *dr, int ncopies,
823 bool add_realign_cost, unsigned int *inside_cost,
824 unsigned int *outside_cost)
826 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
828 switch (alignment_support_scheme)
832 *inside_cost += ncopies * vect_get_stmt_cost (vector_load);
834 if (vect_print_dump_info (REPORT_COST))
835 fprintf (vect_dump, "vect_model_load_cost: aligned.");
839 case dr_unaligned_supported:
841 gimple stmt = DR_STMT (dr);
842 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
843 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
845 /* Here, we assign an additional cost for the unaligned load. */
846 *inside_cost += ncopies
847 * targetm.vectorize.builtin_vectorization_cost (unaligned_load,
848 vectype, DR_MISALIGNMENT (dr));
849 if (vect_print_dump_info (REPORT_COST))
850 fprintf (vect_dump, "vect_model_load_cost: unaligned supported by "
855 case dr_explicit_realign:
857 *inside_cost += ncopies * (2 * vect_get_stmt_cost (vector_load)
858 + vect_get_stmt_cost (vector_stmt));
860 /* FIXME: If the misalignment remains fixed across the iterations of
861 the containing loop, the following cost should be added to the
863 if (targetm.vectorize.builtin_mask_for_load)
864 *inside_cost += vect_get_stmt_cost (vector_stmt);
868 case dr_explicit_realign_optimized:
870 if (vect_print_dump_info (REPORT_COST))
871 fprintf (vect_dump, "vect_model_load_cost: unaligned software "
874 /* Unaligned software pipeline has a load of an address, an initial
875 load, and possibly a mask operation to "prime" the loop. However,
876 if this is an access in a group of loads, which provide strided
877 access, then the above cost should only be considered for one
878 access in the group. Inside the loop, there is a load op
879 and a realignment op. */
881 if (add_realign_cost)
883 *outside_cost = 2 * vect_get_stmt_cost (vector_stmt);
884 if (targetm.vectorize.builtin_mask_for_load)
885 *outside_cost += vect_get_stmt_cost (vector_stmt);
888 *inside_cost += ncopies * (vect_get_stmt_cost (vector_load)
889 + vect_get_stmt_cost (vector_stmt));
899 /* Function vect_init_vector.
901 Insert a new stmt (INIT_STMT) that initializes a new vector variable with
902 the vector elements of VECTOR_VAR. Place the initialization at BSI if it
903 is not NULL. Otherwise, place the initialization at the loop preheader.
904 Return the DEF of INIT_STMT.
905 It will be used in the vectorization of STMT. */
908 vect_init_vector (gimple stmt, tree vector_var, tree vector_type,
909 gimple_stmt_iterator *gsi)
911 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
919 new_var = vect_get_new_vect_var (vector_type, vect_simple_var, "cst_");
920 add_referenced_var (new_var);
921 init_stmt = gimple_build_assign (new_var, vector_var);
922 new_temp = make_ssa_name (new_var, init_stmt);
923 gimple_assign_set_lhs (init_stmt, new_temp);
926 vect_finish_stmt_generation (stmt, init_stmt, gsi);
929 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
933 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
935 if (nested_in_vect_loop_p (loop, stmt))
938 pe = loop_preheader_edge (loop);
939 new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
940 gcc_assert (!new_bb);
944 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
946 gimple_stmt_iterator gsi_bb_start;
948 gcc_assert (bb_vinfo);
949 bb = BB_VINFO_BB (bb_vinfo);
950 gsi_bb_start = gsi_after_labels (bb);
951 gsi_insert_before (&gsi_bb_start, init_stmt, GSI_SAME_STMT);
955 if (vect_print_dump_info (REPORT_DETAILS))
957 fprintf (vect_dump, "created new init_stmt: ");
958 print_gimple_stmt (vect_dump, init_stmt, 0, TDF_SLIM);
961 vec_oprnd = gimple_assign_lhs (init_stmt);
966 /* Function vect_get_vec_def_for_operand.
968 OP is an operand in STMT. This function returns a (vector) def that will be
969 used in the vectorized stmt for STMT.
971 In the case that OP is an SSA_NAME which is defined in the loop, then
972 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
974 In case OP is an invariant or constant, a new stmt that creates a vector def
975 needs to be introduced. */
978 vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def)
983 stmt_vec_info def_stmt_info = NULL;
984 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
986 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
992 enum vect_def_type dt;
996 if (vect_print_dump_info (REPORT_DETAILS))
998 fprintf (vect_dump, "vect_get_vec_def_for_operand: ");
999 print_generic_expr (vect_dump, op, TDF_SLIM);
1002 is_simple_use = vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def,
1004 gcc_assert (is_simple_use);
1005 if (vect_print_dump_info (REPORT_DETAILS))
1009 fprintf (vect_dump, "def = ");
1010 print_generic_expr (vect_dump, def, TDF_SLIM);
1014 fprintf (vect_dump, " def_stmt = ");
1015 print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
1021 /* Case 1: operand is a constant. */
1022 case vect_constant_def:
1024 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
1025 gcc_assert (vector_type);
1026 nunits = TYPE_VECTOR_SUBPARTS (vector_type);
1031 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1032 if (vect_print_dump_info (REPORT_DETAILS))
1033 fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
1035 vec_cst = build_vector_from_val (vector_type, op);
1036 return vect_init_vector (stmt, vec_cst, vector_type, NULL);
1039 /* Case 2: operand is defined outside the loop - loop invariant. */
1040 case vect_external_def:
1042 vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
1043 gcc_assert (vector_type);
1044 nunits = TYPE_VECTOR_SUBPARTS (vector_type);
1049 /* Create 'vec_inv = {inv,inv,..,inv}' */
1050 if (vect_print_dump_info (REPORT_DETAILS))
1051 fprintf (vect_dump, "Create vector_inv.");
1053 for (i = nunits - 1; i >= 0; --i)
1055 t = tree_cons (NULL_TREE, def, t);
1058 /* FIXME: use build_constructor directly. */
1059 vec_inv = build_constructor_from_list (vector_type, t);
1060 return vect_init_vector (stmt, vec_inv, vector_type, NULL);
1063 /* Case 3: operand is defined inside the loop. */
1064 case vect_internal_def:
1067 *scalar_def = NULL/* FIXME tuples: def_stmt*/;
1069 /* Get the def from the vectorized stmt. */
1070 def_stmt_info = vinfo_for_stmt (def_stmt);
1071 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1072 gcc_assert (vec_stmt);
1073 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1074 vec_oprnd = PHI_RESULT (vec_stmt);
1075 else if (is_gimple_call (vec_stmt))
1076 vec_oprnd = gimple_call_lhs (vec_stmt);
1078 vec_oprnd = gimple_assign_lhs (vec_stmt);
1082 /* Case 4: operand is defined by a loop header phi - reduction */
1083 case vect_reduction_def:
1084 case vect_double_reduction_def:
1085 case vect_nested_cycle:
1089 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1090 loop = (gimple_bb (def_stmt))->loop_father;
1092 /* Get the def before the loop */
1093 op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));
1094 return get_initial_def_for_reduction (stmt, op, scalar_def);
1097 /* Case 5: operand is defined by loop-header phi - induction. */
1098 case vect_induction_def:
1100 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1102 /* Get the def from the vectorized stmt. */
1103 def_stmt_info = vinfo_for_stmt (def_stmt);
1104 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1105 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1106 vec_oprnd = PHI_RESULT (vec_stmt);
1108 vec_oprnd = gimple_get_lhs (vec_stmt);
1118 /* Function vect_get_vec_def_for_stmt_copy
1120 Return a vector-def for an operand. This function is used when the
1121 vectorized stmt to be created (by the caller to this function) is a "copy"
1122 created in case the vectorized result cannot fit in one vector, and several
1123 copies of the vector-stmt are required. In this case the vector-def is
1124 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1125 of the stmt that defines VEC_OPRND.
1126 DT is the type of the vector def VEC_OPRND.
1129 In case the vectorization factor (VF) is bigger than the number
1130 of elements that can fit in a vectype (nunits), we have to generate
1131 more than one vector stmt to vectorize the scalar stmt. This situation
1132 arises when there are multiple data-types operated upon in the loop; the
1133 smallest data-type determines the VF, and as a result, when vectorizing
1134 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1135 vector stmt (each computing a vector of 'nunits' results, and together
1136 computing 'VF' results in each iteration). This function is called when
1137 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1138 which VF=16 and nunits=4, so the number of copies required is 4):
1140 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1142 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1143 VS1.1: vx.1 = memref1 VS1.2
1144 VS1.2: vx.2 = memref2 VS1.3
1145 VS1.3: vx.3 = memref3
1147 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1148 VSnew.1: vz1 = vx.1 + ... VSnew.2
1149 VSnew.2: vz2 = vx.2 + ... VSnew.3
1150 VSnew.3: vz3 = vx.3 + ...
1152 The vectorization of S1 is explained in vectorizable_load.
1153 The vectorization of S2:
1154 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1155 the function 'vect_get_vec_def_for_operand' is called to
1156 get the relevant vector-def for each operand of S2. For operand x it
1157 returns the vector-def 'vx.0'.
1159 To create the remaining copies of the vector-stmt (VSnew.j), this
1160 function is called to get the relevant vector-def for each operand. It is
1161 obtained from the respective VS1.j stmt, which is recorded in the
1162 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1164 For example, to obtain the vector-def 'vx.1' in order to create the
1165 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1166 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1167 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1168 and return its def ('vx.1').
1169 Overall, to create the above sequence this function will be called 3 times:
1170 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1171 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1172 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1175 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1177 gimple vec_stmt_for_operand;
1178 stmt_vec_info def_stmt_info;
1180 /* Do nothing; can reuse same def. */
1181 if (dt == vect_external_def || dt == vect_constant_def )
1184 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1185 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1186 gcc_assert (def_stmt_info);
1187 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1188 gcc_assert (vec_stmt_for_operand);
1189 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1190 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1191 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1193 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1198 /* Get vectorized definitions for the operands to create a copy of an original
1199 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1202 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1203 VEC(tree,heap) **vec_oprnds0,
1204 VEC(tree,heap) **vec_oprnds1)
1206 tree vec_oprnd = VEC_pop (tree, *vec_oprnds0);
1208 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1209 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1211 if (vec_oprnds1 && *vec_oprnds1)
1213 vec_oprnd = VEC_pop (tree, *vec_oprnds1);
1214 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1215 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1220 /* Get vectorized definitions for OP0 and OP1, or SLP_NODE if it is not
1224 vect_get_vec_defs (tree op0, tree op1, gimple stmt,
1225 VEC(tree,heap) **vec_oprnds0, VEC(tree,heap) **vec_oprnds1,
1229 vect_get_slp_defs (op0, op1, slp_node, vec_oprnds0, vec_oprnds1, -1);
1234 *vec_oprnds0 = VEC_alloc (tree, heap, 1);
1235 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt, NULL);
1236 VEC_quick_push (tree, *vec_oprnds0, vec_oprnd);
1240 *vec_oprnds1 = VEC_alloc (tree, heap, 1);
1241 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt, NULL);
1242 VEC_quick_push (tree, *vec_oprnds1, vec_oprnd);
1248 /* Function vect_finish_stmt_generation.
1250 Insert a new stmt. */
1253 vect_finish_stmt_generation (gimple stmt, gimple vec_stmt,
1254 gimple_stmt_iterator *gsi)
1256 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1257 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1258 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1260 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1262 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1264 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo,
1267 if (vect_print_dump_info (REPORT_DETAILS))
1269 fprintf (vect_dump, "add new stmt: ");
1270 print_gimple_stmt (vect_dump, vec_stmt, 0, TDF_SLIM);
1273 gimple_set_location (vec_stmt, gimple_location (gsi_stmt (*gsi)));
1276 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1277 a function declaration if the target has a vectorized version
1278 of the function, or NULL_TREE if the function cannot be vectorized. */
1281 vectorizable_function (gimple call, tree vectype_out, tree vectype_in)
1283 tree fndecl = gimple_call_fndecl (call);
1285 /* We only handle functions that do not read or clobber memory -- i.e.
1286 const or novops ones. */
1287 if (!(gimple_call_flags (call) & (ECF_CONST | ECF_NOVOPS)))
1291 || TREE_CODE (fndecl) != FUNCTION_DECL
1292 || !DECL_BUILT_IN (fndecl))
1295 return targetm.vectorize.builtin_vectorized_function (fndecl, vectype_out,
1299 /* Function vectorizable_call.
1301 Check if STMT performs a function call that can be vectorized.
1302 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1303 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1304 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1307 vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt)
1312 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1313 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
1314 tree vectype_out, vectype_in;
1317 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1318 tree fndecl, new_temp, def, rhs_type;
1320 enum vect_def_type dt[3]
1321 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
1322 gimple new_stmt = NULL;
1324 VEC(tree, heap) *vargs = NULL;
1325 enum { NARROW, NONE, WIDEN } modifier;
1328 /* FORNOW: unsupported in basic block SLP. */
1329 gcc_assert (loop_vinfo);
1331 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1334 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1337 /* FORNOW: SLP not supported. */
1338 if (STMT_SLP_TYPE (stmt_info))
1341 /* Is STMT a vectorizable call? */
1342 if (!is_gimple_call (stmt))
1345 if (TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
1348 if (stmt_can_throw_internal (stmt))
1351 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
1353 /* Process function arguments. */
1354 rhs_type = NULL_TREE;
1355 vectype_in = NULL_TREE;
1356 nargs = gimple_call_num_args (stmt);
1358 /* Bail out if the function has more than three arguments, we do not have
1359 interesting builtin functions to vectorize with more than two arguments
1360 except for fma. No arguments is also not good. */
1361 if (nargs == 0 || nargs > 3)
1364 for (i = 0; i < nargs; i++)
1368 op = gimple_call_arg (stmt, i);
1370 /* We can only handle calls with arguments of the same type. */
1372 && !types_compatible_p (rhs_type, TREE_TYPE (op)))
1374 if (vect_print_dump_info (REPORT_DETAILS))
1375 fprintf (vect_dump, "argument types differ.");
1379 rhs_type = TREE_TYPE (op);
1381 if (!vect_is_simple_use_1 (op, loop_vinfo, NULL,
1382 &def_stmt, &def, &dt[i], &opvectype))
1384 if (vect_print_dump_info (REPORT_DETAILS))
1385 fprintf (vect_dump, "use not simple.");
1390 vectype_in = opvectype;
1392 && opvectype != vectype_in)
1394 if (vect_print_dump_info (REPORT_DETAILS))
1395 fprintf (vect_dump, "argument vector types differ.");
1399 /* If all arguments are external or constant defs use a vector type with
1400 the same size as the output vector type. */
1402 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
1404 gcc_assert (vectype_in);
1407 if (vect_print_dump_info (REPORT_DETAILS))
1409 fprintf (vect_dump, "no vectype for scalar type ");
1410 print_generic_expr (vect_dump, rhs_type, TDF_SLIM);
1417 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1418 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1419 if (nunits_in == nunits_out / 2)
1421 else if (nunits_out == nunits_in)
1423 else if (nunits_out == nunits_in / 2)
1428 /* For now, we only vectorize functions if a target specific builtin
1429 is available. TODO -- in some cases, it might be profitable to
1430 insert the calls for pieces of the vector, in order to be able
1431 to vectorize other operations in the loop. */
1432 fndecl = vectorizable_function (stmt, vectype_out, vectype_in);
1433 if (fndecl == NULL_TREE)
1435 if (vect_print_dump_info (REPORT_DETAILS))
1436 fprintf (vect_dump, "function is not vectorizable.");
1441 gcc_assert (!gimple_vuse (stmt));
1443 if (modifier == NARROW)
1444 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1446 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1448 /* Sanity check: make sure that at least one copy of the vectorized stmt
1449 needs to be generated. */
1450 gcc_assert (ncopies >= 1);
1452 if (!vec_stmt) /* transformation not required. */
1454 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
1455 if (vect_print_dump_info (REPORT_DETAILS))
1456 fprintf (vect_dump, "=== vectorizable_call ===");
1457 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
1463 if (vect_print_dump_info (REPORT_DETAILS))
1464 fprintf (vect_dump, "transform operation.");
1467 scalar_dest = gimple_call_lhs (stmt);
1468 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1470 prev_stmt_info = NULL;
1474 for (j = 0; j < ncopies; ++j)
1476 /* Build argument list for the vectorized call. */
1478 vargs = VEC_alloc (tree, heap, nargs);
1480 VEC_truncate (tree, vargs, 0);
1482 for (i = 0; i < nargs; i++)
1484 op = gimple_call_arg (stmt, i);
1487 = vect_get_vec_def_for_operand (op, stmt, NULL);
1490 vec_oprnd0 = gimple_call_arg (new_stmt, i);
1492 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1495 VEC_quick_push (tree, vargs, vec_oprnd0);
1498 new_stmt = gimple_build_call_vec (fndecl, vargs);
1499 new_temp = make_ssa_name (vec_dest, new_stmt);
1500 gimple_call_set_lhs (new_stmt, new_temp);
1502 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1503 mark_symbols_for_renaming (new_stmt);
1506 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1508 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1510 prev_stmt_info = vinfo_for_stmt (new_stmt);
1516 for (j = 0; j < ncopies; ++j)
1518 /* Build argument list for the vectorized call. */
1520 vargs = VEC_alloc (tree, heap, nargs * 2);
1522 VEC_truncate (tree, vargs, 0);
1524 for (i = 0; i < nargs; i++)
1526 op = gimple_call_arg (stmt, i);
1530 = vect_get_vec_def_for_operand (op, stmt, NULL);
1532 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1536 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1);
1538 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
1540 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
1543 VEC_quick_push (tree, vargs, vec_oprnd0);
1544 VEC_quick_push (tree, vargs, vec_oprnd1);
1547 new_stmt = gimple_build_call_vec (fndecl, vargs);
1548 new_temp = make_ssa_name (vec_dest, new_stmt);
1549 gimple_call_set_lhs (new_stmt, new_temp);
1551 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1552 mark_symbols_for_renaming (new_stmt);
1555 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1557 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1559 prev_stmt_info = vinfo_for_stmt (new_stmt);
1562 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1567 /* No current target implements this case. */
1571 VEC_free (tree, heap, vargs);
1573 /* Update the exception handling table with the vector stmt if necessary. */
1574 if (maybe_clean_or_replace_eh_stmt (stmt, *vec_stmt))
1575 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1577 /* The call in STMT might prevent it from being removed in dce.
1578 We however cannot remove it here, due to the way the ssa name
1579 it defines is mapped to the new definition. So just replace
1580 rhs of the statement with something harmless. */
1582 type = TREE_TYPE (scalar_dest);
1583 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
1584 build_zero_cst (type));
1585 set_vinfo_for_stmt (new_stmt, stmt_info);
1586 /* For pattern statements make the related statement to point to
1587 NEW_STMT in order to be able to retrieve the original statement
1588 information later. */
1589 if (is_pattern_stmt_p (stmt_info))
1591 gimple related = STMT_VINFO_RELATED_STMT (stmt_info);
1592 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (related)) = new_stmt;
1594 set_vinfo_for_stmt (stmt, NULL);
1595 STMT_VINFO_STMT (stmt_info) = new_stmt;
1596 gsi_replace (gsi, new_stmt, false);
1597 SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt)) = new_stmt;
1603 /* Function vect_gen_widened_results_half
1605 Create a vector stmt whose code, type, number of arguments, and result
1606 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
1607 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
1608 In the case that CODE is a CALL_EXPR, this means that a call to DECL
1609 needs to be created (DECL is a function-decl of a target-builtin).
1610 STMT is the original scalar stmt that we are vectorizing. */
1613 vect_gen_widened_results_half (enum tree_code code,
1615 tree vec_oprnd0, tree vec_oprnd1, int op_type,
1616 tree vec_dest, gimple_stmt_iterator *gsi,
1622 /* Generate half of the widened result: */
1623 if (code == CALL_EXPR)
1625 /* Target specific support */
1626 if (op_type == binary_op)
1627 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
1629 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
1630 new_temp = make_ssa_name (vec_dest, new_stmt);
1631 gimple_call_set_lhs (new_stmt, new_temp);
1635 /* Generic support */
1636 gcc_assert (op_type == TREE_CODE_LENGTH (code));
1637 if (op_type != binary_op)
1639 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vec_oprnd0,
1641 new_temp = make_ssa_name (vec_dest, new_stmt);
1642 gimple_assign_set_lhs (new_stmt, new_temp);
1644 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1650 /* Check if STMT performs a conversion operation, that can be vectorized.
1651 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1652 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1653 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1656 vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
1657 gimple *vec_stmt, slp_tree slp_node)
1662 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
1663 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1664 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1665 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
1666 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
1670 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1671 gimple new_stmt = NULL;
1672 stmt_vec_info prev_stmt_info;
1675 tree vectype_out, vectype_in;
1679 enum { NARROW, NONE, WIDEN } modifier;
1681 VEC(tree,heap) *vec_oprnds0 = NULL;
1683 VEC(tree,heap) *dummy = NULL;
1686 /* Is STMT a vectorizable conversion? */
1688 /* FORNOW: unsupported in basic block SLP. */
1689 gcc_assert (loop_vinfo);
1691 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1694 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1697 if (!is_gimple_assign (stmt))
1700 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
1703 code = gimple_assign_rhs_code (stmt);
1704 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
1707 /* Check types of lhs and rhs. */
1708 scalar_dest = gimple_assign_lhs (stmt);
1709 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
1711 op0 = gimple_assign_rhs1 (stmt);
1712 rhs_type = TREE_TYPE (op0);
1713 /* Check the operands of the operation. */
1714 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
1715 &def_stmt, &def, &dt[0], &vectype_in))
1717 if (vect_print_dump_info (REPORT_DETAILS))
1718 fprintf (vect_dump, "use not simple.");
1721 /* If op0 is an external or constant defs use a vector type of
1722 the same size as the output vector type. */
1724 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
1726 gcc_assert (vectype_in);
1729 if (vect_print_dump_info (REPORT_DETAILS))
1731 fprintf (vect_dump, "no vectype for scalar type ");
1732 print_generic_expr (vect_dump, rhs_type, TDF_SLIM);
1739 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
1740 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
1741 if (nunits_in == nunits_out / 2)
1743 else if (nunits_out == nunits_in)
1745 else if (nunits_out == nunits_in / 2)
1750 if (modifier == NARROW)
1751 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
1753 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
1755 /* Multiple types in SLP are handled by creating the appropriate number of
1756 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1761 /* Sanity check: make sure that at least one copy of the vectorized stmt
1762 needs to be generated. */
1763 gcc_assert (ncopies >= 1);
1765 /* Supportable by target? */
1766 if ((modifier == NONE
1767 && !targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
1768 || (modifier == WIDEN
1769 && !supportable_widening_operation (code, stmt,
1770 vectype_out, vectype_in,
1773 &dummy_int, &dummy))
1774 || (modifier == NARROW
1775 && !supportable_narrowing_operation (code, vectype_out, vectype_in,
1776 &code1, &dummy_int, &dummy)))
1778 if (vect_print_dump_info (REPORT_DETAILS))
1779 fprintf (vect_dump, "conversion not supported by target.");
1783 if (modifier != NONE)
1785 /* FORNOW: SLP not supported. */
1786 if (STMT_SLP_TYPE (stmt_info))
1790 if (!vec_stmt) /* transformation not required. */
1792 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
1797 if (vect_print_dump_info (REPORT_DETAILS))
1798 fprintf (vect_dump, "transform conversion.");
1801 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
1803 if (modifier == NONE && !slp_node)
1804 vec_oprnds0 = VEC_alloc (tree, heap, 1);
1806 prev_stmt_info = NULL;
1810 for (j = 0; j < ncopies; j++)
1813 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node);
1815 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
1818 targetm.vectorize.builtin_conversion (code,
1819 vectype_out, vectype_in);
1820 FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
1822 /* Arguments are ready. create the new vector stmt. */
1823 new_stmt = gimple_build_call (builtin_decl, 1, vop0);
1824 new_temp = make_ssa_name (vec_dest, new_stmt);
1825 gimple_call_set_lhs (new_stmt, new_temp);
1826 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1828 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
1832 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1834 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1835 prev_stmt_info = vinfo_for_stmt (new_stmt);
1840 /* In case the vectorization factor (VF) is bigger than the number
1841 of elements that we can fit in a vectype (nunits), we have to
1842 generate more than one vector stmt - i.e - we need to "unroll"
1843 the vector stmt by a factor VF/nunits. */
1844 for (j = 0; j < ncopies; j++)
1847 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1849 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1851 /* Generate first half of the widened result: */
1853 = vect_gen_widened_results_half (code1, decl1,
1854 vec_oprnd0, vec_oprnd1,
1855 unary_op, vec_dest, gsi, stmt);
1857 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1859 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1860 prev_stmt_info = vinfo_for_stmt (new_stmt);
1862 /* Generate second half of the widened result: */
1864 = vect_gen_widened_results_half (code2, decl2,
1865 vec_oprnd0, vec_oprnd1,
1866 unary_op, vec_dest, gsi, stmt);
1867 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1868 prev_stmt_info = vinfo_for_stmt (new_stmt);
1873 /* In case the vectorization factor (VF) is bigger than the number
1874 of elements that we can fit in a vectype (nunits), we have to
1875 generate more than one vector stmt - i.e - we need to "unroll"
1876 the vector stmt by a factor VF/nunits. */
1877 for (j = 0; j < ncopies; j++)
1882 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
1883 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1887 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd1);
1888 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
1891 /* Arguments are ready. Create the new vector stmt. */
1892 new_stmt = gimple_build_assign_with_ops (code1, vec_dest, vec_oprnd0,
1894 new_temp = make_ssa_name (vec_dest, new_stmt);
1895 gimple_assign_set_lhs (new_stmt, new_temp);
1896 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1899 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
1901 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1903 prev_stmt_info = vinfo_for_stmt (new_stmt);
1906 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
1910 VEC_free (tree, heap, vec_oprnds0);
1916 /* Function vectorizable_assignment.
1918 Check if STMT performs an assignment (copy) that can be vectorized.
1919 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1920 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1921 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1924 vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi,
1925 gimple *vec_stmt, slp_tree slp_node)
1930 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1931 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1932 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1936 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
1937 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
1940 VEC(tree,heap) *vec_oprnds = NULL;
1942 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1943 gimple new_stmt = NULL;
1944 stmt_vec_info prev_stmt_info = NULL;
1945 enum tree_code code;
1948 /* Multiple types in SLP are handled by creating the appropriate number of
1949 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1954 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
1956 gcc_assert (ncopies >= 1);
1958 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
1961 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1964 /* Is vectorizable assignment? */
1965 if (!is_gimple_assign (stmt))
1968 scalar_dest = gimple_assign_lhs (stmt);
1969 if (TREE_CODE (scalar_dest) != SSA_NAME)
1972 code = gimple_assign_rhs_code (stmt);
1973 if (gimple_assign_single_p (stmt)
1974 || code == PAREN_EXPR
1975 || CONVERT_EXPR_CODE_P (code))
1976 op = gimple_assign_rhs1 (stmt);
1980 if (!vect_is_simple_use_1 (op, loop_vinfo, bb_vinfo,
1981 &def_stmt, &def, &dt[0], &vectype_in))
1983 if (vect_print_dump_info (REPORT_DETAILS))
1984 fprintf (vect_dump, "use not simple.");
1988 /* We can handle NOP_EXPR conversions that do not change the number
1989 of elements or the vector size. */
1990 if (CONVERT_EXPR_CODE_P (code)
1992 || TYPE_VECTOR_SUBPARTS (vectype_in) != nunits
1993 || (GET_MODE_SIZE (TYPE_MODE (vectype))
1994 != GET_MODE_SIZE (TYPE_MODE (vectype_in)))))
1997 if (!vec_stmt) /* transformation not required. */
1999 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
2000 if (vect_print_dump_info (REPORT_DETAILS))
2001 fprintf (vect_dump, "=== vectorizable_assignment ===");
2002 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2007 if (vect_print_dump_info (REPORT_DETAILS))
2008 fprintf (vect_dump, "transform assignment.");
2011 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2014 for (j = 0; j < ncopies; j++)
2018 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node);
2020 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
2022 /* Arguments are ready. create the new vector stmt. */
2023 FOR_EACH_VEC_ELT (tree, vec_oprnds, i, vop)
2025 if (CONVERT_EXPR_CODE_P (code))
2026 vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
2027 new_stmt = gimple_build_assign (vec_dest, vop);
2028 new_temp = make_ssa_name (vec_dest, new_stmt);
2029 gimple_assign_set_lhs (new_stmt, new_temp);
2030 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2032 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2039 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2041 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2043 prev_stmt_info = vinfo_for_stmt (new_stmt);
2046 VEC_free (tree, heap, vec_oprnds);
2051 /* Function vectorizable_shift.
2053 Check if STMT performs a shift operation that can be vectorized.
2054 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2055 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2056 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2059 vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi,
2060 gimple *vec_stmt, slp_tree slp_node)
2064 tree op0, op1 = NULL;
2065 tree vec_oprnd1 = NULL_TREE;
2066 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2068 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2069 enum tree_code code;
2070 enum machine_mode vec_mode;
2074 enum machine_mode optab_op2_mode;
2077 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2078 gimple new_stmt = NULL;
2079 stmt_vec_info prev_stmt_info;
2085 VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
2088 bool scalar_shift_arg = true;
2089 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2092 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2095 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2098 /* Is STMT a vectorizable binary/unary operation? */
2099 if (!is_gimple_assign (stmt))
2102 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2105 code = gimple_assign_rhs_code (stmt);
2107 if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
2108 || code == RROTATE_EXPR))
2111 scalar_dest = gimple_assign_lhs (stmt);
2112 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2114 op0 = gimple_assign_rhs1 (stmt);
2115 if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
2116 &def_stmt, &def, &dt[0], &vectype))
2118 if (vect_print_dump_info (REPORT_DETAILS))
2119 fprintf (vect_dump, "use not simple.");
2122 /* If op0 is an external or constant def use a vector type with
2123 the same size as the output vector type. */
2125 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2127 gcc_assert (vectype);
2130 if (vect_print_dump_info (REPORT_DETAILS))
2132 fprintf (vect_dump, "no vectype for scalar type ");
2133 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2139 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2140 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
2141 if (nunits_out != nunits_in)
2144 op1 = gimple_assign_rhs2 (stmt);
2145 if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[1]))
2147 if (vect_print_dump_info (REPORT_DETAILS))
2148 fprintf (vect_dump, "use not simple.");
2153 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2157 /* Multiple types in SLP are handled by creating the appropriate number of
2158 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2163 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2165 gcc_assert (ncopies >= 1);
2167 /* Determine whether the shift amount is a vector, or scalar. If the
2168 shift/rotate amount is a vector, use the vector/vector shift optabs. */
2170 if (dt[1] == vect_internal_def && !slp_node)
2171 scalar_shift_arg = false;
2172 else if (dt[1] == vect_constant_def
2173 || dt[1] == vect_external_def
2174 || dt[1] == vect_internal_def)
2176 /* In SLP, need to check whether the shift count is the same,
2177 in loops if it is a constant or invariant, it is always
2181 VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (slp_node);
2184 FOR_EACH_VEC_ELT (gimple, stmts, k, slpstmt)
2185 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
2186 scalar_shift_arg = false;
2191 if (vect_print_dump_info (REPORT_DETAILS))
2192 fprintf (vect_dump, "operand mode requires invariant argument.");
2196 /* Vector shifted by vector. */
2197 if (!scalar_shift_arg)
2199 optab = optab_for_tree_code (code, vectype, optab_vector);
2200 if (vect_print_dump_info (REPORT_DETAILS))
2201 fprintf (vect_dump, "vector/vector shift/rotate found.");
2203 /* See if the machine has a vector shifted by scalar insn and if not
2204 then see if it has a vector shifted by vector insn. */
2207 optab = optab_for_tree_code (code, vectype, optab_scalar);
2209 && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
2211 if (vect_print_dump_info (REPORT_DETAILS))
2212 fprintf (vect_dump, "vector/scalar shift/rotate found.");
2216 optab = optab_for_tree_code (code, vectype, optab_vector);
2218 && (optab_handler (optab, TYPE_MODE (vectype))
2219 != CODE_FOR_nothing))
2221 scalar_shift_arg = false;
2223 if (vect_print_dump_info (REPORT_DETAILS))
2224 fprintf (vect_dump, "vector/vector shift/rotate found.");
2226 /* Unlike the other binary operators, shifts/rotates have
2227 the rhs being int, instead of the same type as the lhs,
2228 so make sure the scalar is the right type if we are
2229 dealing with vectors of short/char. */
2230 if (dt[1] == vect_constant_def)
2231 op1 = fold_convert (TREE_TYPE (vectype), op1);
2236 /* Supportable by target? */
2239 if (vect_print_dump_info (REPORT_DETAILS))
2240 fprintf (vect_dump, "no optab.");
2243 vec_mode = TYPE_MODE (vectype);
2244 icode = (int) optab_handler (optab, vec_mode);
2245 if (icode == CODE_FOR_nothing)
2247 if (vect_print_dump_info (REPORT_DETAILS))
2248 fprintf (vect_dump, "op not supported by target.");
2249 /* Check only during analysis. */
2250 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
2251 || (vf < vect_min_worthwhile_factor (code)
2254 if (vect_print_dump_info (REPORT_DETAILS))
2255 fprintf (vect_dump, "proceeding using word mode.");
2258 /* Worthwhile without SIMD support? Check only during analysis. */
2259 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
2260 && vf < vect_min_worthwhile_factor (code)
2263 if (vect_print_dump_info (REPORT_DETAILS))
2264 fprintf (vect_dump, "not worthwhile without SIMD support.");
2268 if (!vec_stmt) /* transformation not required. */
2270 STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
2271 if (vect_print_dump_info (REPORT_DETAILS))
2272 fprintf (vect_dump, "=== vectorizable_shift ===");
2273 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2279 if (vect_print_dump_info (REPORT_DETAILS))
2280 fprintf (vect_dump, "transform binary/unary operation.");
2283 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2285 /* Allocate VECs for vector operands. In case of SLP, vector operands are
2286 created in the previous stages of the recursion, so no allocation is
2287 needed, except for the case of shift with scalar shift argument. In that
2288 case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
2289 be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
2290 In case of loop-based vectorization we allocate VECs of size 1. We
2291 allocate VEC_OPRNDS1 only in case of binary operation. */
2294 vec_oprnds0 = VEC_alloc (tree, heap, 1);
2295 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2297 else if (scalar_shift_arg)
2298 vec_oprnds1 = VEC_alloc (tree, heap, slp_node->vec_stmts_size);
2300 prev_stmt_info = NULL;
2301 for (j = 0; j < ncopies; j++)
2306 if (scalar_shift_arg)
2308 /* Vector shl and shr insn patterns can be defined with scalar
2309 operand 2 (shift operand). In this case, use constant or loop
2310 invariant op1 directly, without extending it to vector mode
2312 optab_op2_mode = insn_data[icode].operand[2].mode;
2313 if (!VECTOR_MODE_P (optab_op2_mode))
2315 if (vect_print_dump_info (REPORT_DETAILS))
2316 fprintf (vect_dump, "operand 1 using scalar mode.");
2318 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2321 /* Store vec_oprnd1 for every vector stmt to be created
2322 for SLP_NODE. We check during the analysis that all
2323 the shift arguments are the same.
2324 TODO: Allow different constants for different vector
2325 stmts generated for an SLP instance. */
2326 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
2327 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
2332 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
2333 (a special case for certain kind of vector shifts); otherwise,
2334 operand 1 should be of a vector type (the usual case). */
2336 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
2339 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
2343 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
2345 /* Arguments are ready. Create the new vector stmt. */
2346 FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
2348 vop1 = VEC_index (tree, vec_oprnds1, i);
2349 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2350 new_temp = make_ssa_name (vec_dest, new_stmt);
2351 gimple_assign_set_lhs (new_stmt, new_temp);
2352 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2354 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2361 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2363 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2364 prev_stmt_info = vinfo_for_stmt (new_stmt);
2367 VEC_free (tree, heap, vec_oprnds0);
2368 VEC_free (tree, heap, vec_oprnds1);
2374 /* Function vectorizable_operation.
2376 Check if STMT performs a binary, unary or ternary operation that can
2378 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2379 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2380 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2383 vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
2384 gimple *vec_stmt, slp_tree slp_node)
2388 tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
2389 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2391 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2392 enum tree_code code;
2393 enum machine_mode vec_mode;
2400 enum vect_def_type dt[3]
2401 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
2402 gimple new_stmt = NULL;
2403 stmt_vec_info prev_stmt_info;
2409 VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vec_oprnds2 = NULL;
2410 tree vop0, vop1, vop2;
2411 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2414 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2417 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2420 /* Is STMT a vectorizable binary/unary operation? */
2421 if (!is_gimple_assign (stmt))
2424 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2427 code = gimple_assign_rhs_code (stmt);
2429 /* For pointer addition, we should use the normal plus for
2430 the vector addition. */
2431 if (code == POINTER_PLUS_EXPR)
2434 /* Support only unary or binary operations. */
2435 op_type = TREE_CODE_LENGTH (code);
2436 if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
2438 if (vect_print_dump_info (REPORT_DETAILS))
2439 fprintf (vect_dump, "num. args = %d (not unary/binary/ternary op).",
2444 scalar_dest = gimple_assign_lhs (stmt);
2445 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2447 op0 = gimple_assign_rhs1 (stmt);
2448 if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo,
2449 &def_stmt, &def, &dt[0], &vectype))
2451 if (vect_print_dump_info (REPORT_DETAILS))
2452 fprintf (vect_dump, "use not simple.");
2455 /* If op0 is an external or constant def use a vector type with
2456 the same size as the output vector type. */
2458 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2460 gcc_assert (vectype);
2463 if (vect_print_dump_info (REPORT_DETAILS))
2465 fprintf (vect_dump, "no vectype for scalar type ");
2466 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2472 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2473 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
2474 if (nunits_out != nunits_in)
2477 if (op_type == binary_op || op_type == ternary_op)
2479 op1 = gimple_assign_rhs2 (stmt);
2480 if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def,
2483 if (vect_print_dump_info (REPORT_DETAILS))
2484 fprintf (vect_dump, "use not simple.");
2488 if (op_type == ternary_op)
2490 op2 = gimple_assign_rhs3 (stmt);
2491 if (!vect_is_simple_use (op2, loop_vinfo, bb_vinfo, &def_stmt, &def,
2494 if (vect_print_dump_info (REPORT_DETAILS))
2495 fprintf (vect_dump, "use not simple.");
2501 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2505 /* Multiple types in SLP are handled by creating the appropriate number of
2506 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2511 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2513 gcc_assert (ncopies >= 1);
2515 /* Shifts are handled in vectorizable_shift (). */
2516 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
2517 || code == RROTATE_EXPR)
2520 optab = optab_for_tree_code (code, vectype, optab_default);
2522 /* Supportable by target? */
2525 if (vect_print_dump_info (REPORT_DETAILS))
2526 fprintf (vect_dump, "no optab.");
2529 vec_mode = TYPE_MODE (vectype);
2530 icode = (int) optab_handler (optab, vec_mode);
2531 if (icode == CODE_FOR_nothing)
2533 if (vect_print_dump_info (REPORT_DETAILS))
2534 fprintf (vect_dump, "op not supported by target.");
2535 /* Check only during analysis. */
2536 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
2537 || (vf < vect_min_worthwhile_factor (code)
2540 if (vect_print_dump_info (REPORT_DETAILS))
2541 fprintf (vect_dump, "proceeding using word mode.");
2544 /* Worthwhile without SIMD support? Check only during analysis. */
2545 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
2546 && vf < vect_min_worthwhile_factor (code)
2549 if (vect_print_dump_info (REPORT_DETAILS))
2550 fprintf (vect_dump, "not worthwhile without SIMD support.");
2554 if (!vec_stmt) /* transformation not required. */
2556 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
2557 if (vect_print_dump_info (REPORT_DETAILS))
2558 fprintf (vect_dump, "=== vectorizable_operation ===");
2559 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2565 if (vect_print_dump_info (REPORT_DETAILS))
2566 fprintf (vect_dump, "transform binary/unary operation.");
2569 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2571 /* Allocate VECs for vector operands. In case of SLP, vector operands are
2572 created in the previous stages of the recursion, so no allocation is
2573 needed, except for the case of shift with scalar shift argument. In that
2574 case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
2575 be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
2576 In case of loop-based vectorization we allocate VECs of size 1. We
2577 allocate VEC_OPRNDS1 only in case of binary operation. */
2580 vec_oprnds0 = VEC_alloc (tree, heap, 1);
2581 if (op_type == binary_op || op_type == ternary_op)
2582 vec_oprnds1 = VEC_alloc (tree, heap, 1);
2583 if (op_type == ternary_op)
2584 vec_oprnds2 = VEC_alloc (tree, heap, 1);
2587 /* In case the vectorization factor (VF) is bigger than the number
2588 of elements that we can fit in a vectype (nunits), we have to generate
2589 more than one vector stmt - i.e - we need to "unroll" the
2590 vector stmt by a factor VF/nunits. In doing so, we record a pointer
2591 from one copy of the vector stmt to the next, in the field
2592 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
2593 stages to find the correct vector defs to be used when vectorizing
2594 stmts that use the defs of the current stmt. The example below
2595 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
2596 we need to create 4 vectorized stmts):
2598 before vectorization:
2599 RELATED_STMT VEC_STMT
2603 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
2605 RELATED_STMT VEC_STMT
2606 VS1_0: vx0 = memref0 VS1_1 -
2607 VS1_1: vx1 = memref1 VS1_2 -
2608 VS1_2: vx2 = memref2 VS1_3 -
2609 VS1_3: vx3 = memref3 - -
2610 S1: x = load - VS1_0
2613 step2: vectorize stmt S2 (done here):
2614 To vectorize stmt S2 we first need to find the relevant vector
2615 def for the first operand 'x'. This is, as usual, obtained from
2616 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
2617 that defines 'x' (S1). This way we find the stmt VS1_0, and the
2618 relevant vector def 'vx0'. Having found 'vx0' we can generate
2619 the vector stmt VS2_0, and as usual, record it in the
2620 STMT_VINFO_VEC_STMT of stmt S2.
2621 When creating the second copy (VS2_1), we obtain the relevant vector
2622 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
2623 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
2624 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
2625 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
2626 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
2627 chain of stmts and pointers:
2628 RELATED_STMT VEC_STMT
2629 VS1_0: vx0 = memref0 VS1_1 -
2630 VS1_1: vx1 = memref1 VS1_2 -
2631 VS1_2: vx2 = memref2 VS1_3 -
2632 VS1_3: vx3 = memref3 - -
2633 S1: x = load - VS1_0
2634 VS2_0: vz0 = vx0 + v1 VS2_1 -
2635 VS2_1: vz1 = vx1 + v1 VS2_2 -
2636 VS2_2: vz2 = vx2 + v1 VS2_3 -
2637 VS2_3: vz3 = vx3 + v1 - -
2638 S2: z = x + 1 - VS2_0 */
2640 prev_stmt_info = NULL;
2641 for (j = 0; j < ncopies; j++)
2646 if (op_type == binary_op || op_type == ternary_op)
2647 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
2650 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
2652 if (op_type == ternary_op)
2654 vec_oprnds2 = VEC_alloc (tree, heap, 1);
2655 VEC_quick_push (tree, vec_oprnds2,
2656 vect_get_vec_def_for_operand (op2, stmt, NULL));
2661 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
2662 if (op_type == ternary_op)
2664 tree vec_oprnd = VEC_pop (tree, vec_oprnds2);
2665 VEC_quick_push (tree, vec_oprnds2,
2666 vect_get_vec_def_for_stmt_copy (dt[2],
2671 /* Arguments are ready. Create the new vector stmt. */
2672 FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0)
2674 vop1 = ((op_type == binary_op || op_type == ternary_op)
2675 ? VEC_index (tree, vec_oprnds1, i) : NULL_TREE);
2676 vop2 = ((op_type == ternary_op)
2677 ? VEC_index (tree, vec_oprnds2, i) : NULL_TREE);
2678 new_stmt = gimple_build_assign_with_ops3 (code, vec_dest,
2680 new_temp = make_ssa_name (vec_dest, new_stmt);
2681 gimple_assign_set_lhs (new_stmt, new_temp);
2682 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2684 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2691 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2693 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2694 prev_stmt_info = vinfo_for_stmt (new_stmt);
2697 VEC_free (tree, heap, vec_oprnds0);
2699 VEC_free (tree, heap, vec_oprnds1);
2701 VEC_free (tree, heap, vec_oprnds2);
2707 /* Get vectorized definitions for loop-based vectorization. For the first
2708 operand we call vect_get_vec_def_for_operand() (with OPRND containing
2709 scalar operand), and for the rest we get a copy with
2710 vect_get_vec_def_for_stmt_copy() using the previous vector definition
2711 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
2712 The vectors are collected into VEC_OPRNDS. */
2715 vect_get_loop_based_defs (tree *oprnd, gimple stmt, enum vect_def_type dt,
2716 VEC (tree, heap) **vec_oprnds, int multi_step_cvt)
2720 /* Get first vector operand. */
2721 /* All the vector operands except the very first one (that is scalar oprnd)
2723 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
2724 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt, NULL);
2726 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
2728 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2730 /* Get second vector operand. */
2731 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
2732 VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
2736 /* For conversion in multiple steps, continue to get operands
2739 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
2743 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
2744 For multi-step conversions store the resulting vectors and call the function
2748 vect_create_vectorized_demotion_stmts (VEC (tree, heap) **vec_oprnds,
2749 int multi_step_cvt, gimple stmt,
2750 VEC (tree, heap) *vec_dsts,
2751 gimple_stmt_iterator *gsi,
2752 slp_tree slp_node, enum tree_code code,
2753 stmt_vec_info *prev_stmt_info)
2756 tree vop0, vop1, new_tmp, vec_dest;
2758 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2760 vec_dest = VEC_pop (tree, vec_dsts);
2762 for (i = 0; i < VEC_length (tree, *vec_oprnds); i += 2)
2764 /* Create demotion operation. */
2765 vop0 = VEC_index (tree, *vec_oprnds, i);
2766 vop1 = VEC_index (tree, *vec_oprnds, i + 1);
2767 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
2768 new_tmp = make_ssa_name (vec_dest, new_stmt);
2769 gimple_assign_set_lhs (new_stmt, new_tmp);
2770 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2773 /* Store the resulting vector for next recursive call. */
2774 VEC_replace (tree, *vec_oprnds, i/2, new_tmp);
2777 /* This is the last step of the conversion sequence. Store the
2778 vectors in SLP_NODE or in vector info of the scalar statement
2779 (or in STMT_VINFO_RELATED_STMT chain). */
2781 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
2784 if (!*prev_stmt_info)
2785 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
2787 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
2789 *prev_stmt_info = vinfo_for_stmt (new_stmt);
2794 /* For multi-step demotion operations we first generate demotion operations
2795 from the source type to the intermediate types, and then combine the
2796 results (stored in VEC_OPRNDS) in demotion operation to the destination
2800 /* At each level of recursion we have have of the operands we had at the
2802 VEC_truncate (tree, *vec_oprnds, (i+1)/2);
2803 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
2804 stmt, vec_dsts, gsi, slp_node,
2805 code, prev_stmt_info);
2810 /* Function vectorizable_type_demotion
2812 Check if STMT performs a binary or unary operation that involves
2813 type demotion, and if it can be vectorized.
2814 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2815 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2816 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2819 vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi,
2820 gimple *vec_stmt, slp_tree slp_node)
2825 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2826 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2827 enum tree_code code, code1 = ERROR_MARK;
2830 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
2831 stmt_vec_info prev_stmt_info;
2838 int multi_step_cvt = 0;
2839 VEC (tree, heap) *vec_oprnds0 = NULL;
2840 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
2841 tree last_oprnd, intermediate_type;
2843 /* FORNOW: not supported by basic block SLP vectorization. */
2844 gcc_assert (loop_vinfo);
2846 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2849 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2852 /* Is STMT a vectorizable type-demotion operation? */
2853 if (!is_gimple_assign (stmt))
2856 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
2859 code = gimple_assign_rhs_code (stmt);
2860 if (!CONVERT_EXPR_CODE_P (code))
2863 scalar_dest = gimple_assign_lhs (stmt);
2864 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2866 /* Check the operands of the operation. */
2867 op0 = gimple_assign_rhs1 (stmt);
2868 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
2869 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
2870 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
2871 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
2872 && CONVERT_EXPR_CODE_P (code))))
2874 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
2875 &def_stmt, &def, &dt[0], &vectype_in))
2877 if (vect_print_dump_info (REPORT_DETAILS))
2878 fprintf (vect_dump, "use not simple.");
2881 /* If op0 is an external def use a vector type with the
2882 same size as the output vector type if possible. */
2884 vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
2886 gcc_assert (vectype_in);
2889 if (vect_print_dump_info (REPORT_DETAILS))
2891 fprintf (vect_dump, "no vectype for scalar type ");
2892 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
2898 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2899 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2900 if (nunits_in >= nunits_out)
2903 /* Multiple types in SLP are handled by creating the appropriate number of
2904 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2909 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
2910 gcc_assert (ncopies >= 1);
2912 /* Supportable by target? */
2913 if (!supportable_narrowing_operation (code, vectype_out, vectype_in,
2914 &code1, &multi_step_cvt, &interm_types))
2917 if (!vec_stmt) /* transformation not required. */
2919 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
2920 if (vect_print_dump_info (REPORT_DETAILS))
2921 fprintf (vect_dump, "=== vectorizable_demotion ===");
2922 vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
2927 if (vect_print_dump_info (REPORT_DETAILS))
2928 fprintf (vect_dump, "transform type demotion operation. ncopies = %d.",
2931 /* In case of multi-step demotion, we first generate demotion operations to
2932 the intermediate types, and then from that types to the final one.
2933 We create vector destinations for the intermediate type (TYPES) received
2934 from supportable_narrowing_operation, and store them in the correct order
2935 for future use in vect_create_vectorized_demotion_stmts(). */
2937 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
2939 vec_dsts = VEC_alloc (tree, heap, 1);
2941 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2942 VEC_quick_push (tree, vec_dsts, vec_dest);
2946 for (i = VEC_length (tree, interm_types) - 1;
2947 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
2949 vec_dest = vect_create_destination_var (scalar_dest,
2951 VEC_quick_push (tree, vec_dsts, vec_dest);
2955 /* In case the vectorization factor (VF) is bigger than the number
2956 of elements that we can fit in a vectype (nunits), we have to generate
2957 more than one vector stmt - i.e - we need to "unroll" the
2958 vector stmt by a factor VF/nunits. */
2960 prev_stmt_info = NULL;
2961 for (j = 0; j < ncopies; j++)
2965 vect_get_slp_defs (op0, NULL_TREE, slp_node, &vec_oprnds0, NULL, -1);
2968 VEC_free (tree, heap, vec_oprnds0);
2969 vec_oprnds0 = VEC_alloc (tree, heap,
2970 (multi_step_cvt ? vect_pow2 (multi_step_cvt) * 2 : 2));
2971 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
2972 vect_pow2 (multi_step_cvt) - 1);
2975 /* Arguments are ready. Create the new vector stmts. */
2976 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
2977 vect_create_vectorized_demotion_stmts (&vec_oprnds0,
2978 multi_step_cvt, stmt, tmp_vec_dsts,
2979 gsi, slp_node, code1,
2983 VEC_free (tree, heap, vec_oprnds0);
2984 VEC_free (tree, heap, vec_dsts);
2985 VEC_free (tree, heap, tmp_vec_dsts);
2986 VEC_free (tree, heap, interm_types);
2988 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2993 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
2994 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
2995 the resulting vectors and call the function recursively. */
2998 vect_create_vectorized_promotion_stmts (VEC (tree, heap) **vec_oprnds0,
2999 VEC (tree, heap) **vec_oprnds1,
3000 int multi_step_cvt, gimple stmt,
3001 VEC (tree, heap) *vec_dsts,
3002 gimple_stmt_iterator *gsi,
3003 slp_tree slp_node, enum tree_code code1,
3004 enum tree_code code2, tree decl1,
3005 tree decl2, int op_type,
3006 stmt_vec_info *prev_stmt_info)
3009 tree vop0, vop1, new_tmp1, new_tmp2, vec_dest;
3010 gimple new_stmt1, new_stmt2;
3011 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3012 VEC (tree, heap) *vec_tmp;
3014 vec_dest = VEC_pop (tree, vec_dsts);
3015 vec_tmp = VEC_alloc (tree, heap, VEC_length (tree, *vec_oprnds0) * 2);
3017 FOR_EACH_VEC_ELT (tree, *vec_oprnds0, i, vop0)
3019 if (op_type == binary_op)
3020 vop1 = VEC_index (tree, *vec_oprnds1, i);
3024 /* Generate the two halves of promotion operation. */
3025 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
3026 op_type, vec_dest, gsi, stmt);
3027 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
3028 op_type, vec_dest, gsi, stmt);
3029 if (is_gimple_call (new_stmt1))
3031 new_tmp1 = gimple_call_lhs (new_stmt1);
3032 new_tmp2 = gimple_call_lhs (new_stmt2);
3036 new_tmp1 = gimple_assign_lhs (new_stmt1);
3037 new_tmp2 = gimple_assign_lhs (new_stmt2);
3042 /* Store the results for the recursive call. */
3043 VEC_quick_push (tree, vec_tmp, new_tmp1);
3044 VEC_quick_push (tree, vec_tmp, new_tmp2);
3048 /* Last step of promotion sequience - store the results. */
3051 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt1);
3052 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt2);
3056 if (!*prev_stmt_info)
3057 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt1;
3059 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt1;
3061 *prev_stmt_info = vinfo_for_stmt (new_stmt1);
3062 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt2;
3063 *prev_stmt_info = vinfo_for_stmt (new_stmt2);
3070 /* For multi-step promotion operation we first generate we call the
3071 function recurcively for every stage. We start from the input type,
3072 create promotion operations to the intermediate types, and then
3073 create promotions to the output type. */
3074 *vec_oprnds0 = VEC_copy (tree, heap, vec_tmp);
3075 vect_create_vectorized_promotion_stmts (vec_oprnds0, vec_oprnds1,
3076 multi_step_cvt - 1, stmt,
3077 vec_dsts, gsi, slp_node, code1,
3078 code2, decl2, decl2, op_type,
3082 VEC_free (tree, heap, vec_tmp);
3086 /* Function vectorizable_type_promotion
3088 Check if STMT performs a binary or unary operation that involves
3089 type promotion, and if it can be vectorized.
3090 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3091 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3092 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3095 vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi,
3096 gimple *vec_stmt, slp_tree slp_node)
3100 tree op0, op1 = NULL;
3101 tree vec_oprnd0=NULL, vec_oprnd1=NULL;
3102 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3103 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3104 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
3105 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
3109 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
3110 stmt_vec_info prev_stmt_info;
3117 tree intermediate_type = NULL_TREE;
3118 int multi_step_cvt = 0;
3119 VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
3120 VEC (tree, heap) *vec_dsts = NULL, *interm_types = NULL, *tmp_vec_dsts = NULL;
3122 /* FORNOW: not supported by basic block SLP vectorization. */
3123 gcc_assert (loop_vinfo);
3125 if (!STMT_VINFO_RELEVANT_P (stmt_info))
3128 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3131 /* Is STMT a vectorizable type-promotion operation? */
3132 if (!is_gimple_assign (stmt))
3135 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
3138 code = gimple_assign_rhs_code (stmt);
3139 if (!CONVERT_EXPR_CODE_P (code)
3140 && code != WIDEN_MULT_EXPR)
3143 scalar_dest = gimple_assign_lhs (stmt);
3144 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
3146 /* Check the operands of the operation. */
3147 op0 = gimple_assign_rhs1 (stmt);
3148 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
3149 && INTEGRAL_TYPE_P (TREE_TYPE (op0)))
3150 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest))
3151 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0))
3152 && CONVERT_EXPR_CODE_P (code))))
3154 if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL,
3155 &def_stmt, &def, &dt[0], &vectype_in))
3157 if (vect_print_dump_info (REPORT_DETAILS))
3158 fprintf (vect_dump, "use not simple.");
3161 /* If op0 is an external or constant def use a vector type with
3162 the same size as the output vector type. */
3164 vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
3166 gcc_assert (vectype_in);
3169 if (vect_print_dump_info (REPORT_DETAILS))
3171 fprintf (vect_dump, "no vectype for scalar type ");
3172 print_generic_expr (vect_dump, TREE_TYPE (op0), TDF_SLIM);
3178 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3179 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3180 if (nunits_in <= nunits_out)
3183 /* Multiple types in SLP are handled by creating the appropriate number of
3184 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3189 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
3191 gcc_assert (ncopies >= 1);
3193 op_type = TREE_CODE_LENGTH (code);
3194 if (op_type == binary_op)
3196 op1 = gimple_assign_rhs2 (stmt);
3197 if (!vect_is_simple_use (op1, loop_vinfo, NULL, &def_stmt, &def, &dt[1]))
3199 if (vect_print_dump_info (REPORT_DETAILS))
3200 fprintf (vect_dump, "use not simple.");
3205 /* Supportable by target? */
3206 if (!supportable_widening_operation (code, stmt, vectype_out, vectype_in,
3207 &decl1, &decl2, &code1, &code2,
3208 &multi_step_cvt, &interm_types))
3211 /* Binary widening operation can only be supported directly by the
3213 gcc_assert (!(multi_step_cvt && op_type == binary_op));
3215 if (!vec_stmt) /* transformation not required. */
3217 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
3218 if (vect_print_dump_info (REPORT_DETAILS))
3219 fprintf (vect_dump, "=== vectorizable_promotion ===");
3220 vect_model_simple_cost (stmt_info, 2*ncopies, dt, NULL);
3226 if (vect_print_dump_info (REPORT_DETAILS))
3227 fprintf (vect_dump, "transform type promotion operation. ncopies = %d.",
3231 /* In case of multi-step promotion, we first generate promotion operations
3232 to the intermediate types, and then from that types to the final one.
3233 We store vector destination in VEC_DSTS in the correct order for
3234 recursive creation of promotion operations in
3235 vect_create_vectorized_promotion_stmts(). Vector destinations are created
3236 according to TYPES recieved from supportable_widening_operation(). */
3238 vec_dsts = VEC_alloc (tree, heap, multi_step_cvt + 1);
3240 vec_dsts = VEC_alloc (tree, heap, 1);
3242 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3243 VEC_quick_push (tree, vec_dsts, vec_dest);
3247 for (i = VEC_length (tree, interm_types) - 1;
3248 VEC_iterate (tree, interm_types, i, intermediate_type); i--)
3250 vec_dest = vect_create_destination_var (scalar_dest,
3252 VEC_quick_push (tree, vec_dsts, vec_dest);
3258 vec_oprnds0 = VEC_alloc (tree, heap,
3259 (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
3260 if (op_type == binary_op)
3261 vec_oprnds1 = VEC_alloc (tree, heap, 1);
3264 /* In case the vectorization factor (VF) is bigger than the number
3265 of elements that we can fit in a vectype (nunits), we have to generate
3266 more than one vector stmt - i.e - we need to "unroll" the
3267 vector stmt by a factor VF/nunits. */
3269 prev_stmt_info = NULL;
3270 for (j = 0; j < ncopies; j++)
3276 vect_get_slp_defs (op0, op1, slp_node, &vec_oprnds0,
3280 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
3281 VEC_quick_push (tree, vec_oprnds0, vec_oprnd0);
3282 if (op_type == binary_op)
3284 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt, NULL);
3285 VEC_quick_push (tree, vec_oprnds1, vec_oprnd1);
3291 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
3292 VEC_replace (tree, vec_oprnds0, 0, vec_oprnd0);
3293 if (op_type == binary_op)
3295 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd1);
3296 VEC_replace (tree, vec_oprnds1, 0, vec_oprnd1);
3300 /* Arguments are ready. Create the new vector stmts. */
3301 tmp_vec_dsts = VEC_copy (tree, heap, vec_dsts);
3302 vect_create_vectorized_promotion_stmts (&vec_oprnds0, &vec_oprnds1,
3303 multi_step_cvt, stmt,
3305 gsi, slp_node, code1, code2,
3306 decl1, decl2, op_type,
3310 VEC_free (tree, heap, vec_dsts);
3311 VEC_free (tree, heap, tmp_vec_dsts);
3312 VEC_free (tree, heap, interm_types);
3313 VEC_free (tree, heap, vec_oprnds0);
3314 VEC_free (tree, heap, vec_oprnds1);
3316 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3321 /* Function vectorizable_store.
3323 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
3325 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3326 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3327 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3330 vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
3336 tree vec_oprnd = NULL_TREE;
3337 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3338 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
3339 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3340 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3341 struct loop *loop = NULL;
3342 enum machine_mode vec_mode;
3344 enum dr_alignment_support alignment_support_scheme;
3347 enum vect_def_type dt;
3348 stmt_vec_info prev_stmt_info = NULL;
3349 tree dataref_ptr = NULL_TREE;
3350 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3353 gimple next_stmt, first_stmt = NULL;
3354 bool strided_store = false;
3355 unsigned int group_size, i;
3356 VEC(tree,heap) *dr_chain = NULL, *oprnds = NULL, *result_chain = NULL;
3358 VEC(tree,heap) *vec_oprnds = NULL;
3359 bool slp = (slp_node != NULL);
3360 unsigned int vec_num;
3361 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3364 loop = LOOP_VINFO_LOOP (loop_vinfo);
3366 /* Multiple types in SLP are handled by creating the appropriate number of
3367 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3372 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3374 gcc_assert (ncopies >= 1);
3376 /* FORNOW. This restriction should be relaxed. */
3377 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
3379 if (vect_print_dump_info (REPORT_DETAILS))
3380 fprintf (vect_dump, "multiple types in nested loop.");
3384 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3387 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3390 /* Is vectorizable store? */
3392 if (!is_gimple_assign (stmt))
3395 scalar_dest = gimple_assign_lhs (stmt);
3396 if (TREE_CODE (scalar_dest) != ARRAY_REF
3397 && TREE_CODE (scalar_dest) != INDIRECT_REF
3398 && TREE_CODE (scalar_dest) != COMPONENT_REF
3399 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
3400 && TREE_CODE (scalar_dest) != REALPART_EXPR
3401 && TREE_CODE (scalar_dest) != MEM_REF)
3404 gcc_assert (gimple_assign_single_p (stmt));
3405 op = gimple_assign_rhs1 (stmt);
3406 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt))
3408 if (vect_print_dump_info (REPORT_DETAILS))
3409 fprintf (vect_dump, "use not simple.");
3413 /* The scalar rhs type needs to be trivially convertible to the vector
3414 component type. This should always be the case. */
3415 if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (op)))
3417 if (vect_print_dump_info (REPORT_DETAILS))
3418 fprintf (vect_dump, "??? operands of different types");
3422 vec_mode = TYPE_MODE (vectype);
3423 /* FORNOW. In some cases can vectorize even if data-type not supported
3424 (e.g. - array initialization with 0). */
3425 if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing)
3428 if (!STMT_VINFO_DATA_REF (stmt_info))
3431 if (tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0)
3433 if (vect_print_dump_info (REPORT_DETAILS))
3434 fprintf (vect_dump, "negative step for store.");
3438 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
3440 strided_store = true;
3441 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
3442 if (!vect_strided_store_supported (vectype)
3443 && !PURE_SLP_STMT (stmt_info) && !slp)
3446 if (first_stmt == stmt)
3448 /* STMT is the leader of the group. Check the operands of all the
3449 stmts of the group. */
3450 next_stmt = DR_GROUP_NEXT_DR (stmt_info);
3453 gcc_assert (gimple_assign_single_p (next_stmt));
3454 op = gimple_assign_rhs1 (next_stmt);
3455 if (!vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt,
3458 if (vect_print_dump_info (REPORT_DETAILS))
3459 fprintf (vect_dump, "use not simple.");
3462 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3467 if (!vec_stmt) /* transformation not required. */
3469 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
3470 vect_model_store_cost (stmt_info, ncopies, dt, NULL);
3478 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3479 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3481 DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
3484 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
3486 /* We vectorize all the stmts of the interleaving group when we
3487 reach the last stmt in the group. */
3488 if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
3489 < DR_GROUP_SIZE (vinfo_for_stmt (first_stmt))
3498 strided_store = false;
3499 /* VEC_NUM is the number of vect stmts to be created for this
3501 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3502 first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
3503 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3506 /* VEC_NUM is the number of vect stmts to be created for this
3508 vec_num = group_size;
3514 group_size = vec_num = 1;
3517 if (vect_print_dump_info (REPORT_DETAILS))
3518 fprintf (vect_dump, "transform store. ncopies = %d",ncopies);
3520 dr_chain = VEC_alloc (tree, heap, group_size);
3521 oprnds = VEC_alloc (tree, heap, group_size);
3523 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
3524 gcc_assert (alignment_support_scheme);
3526 /* In case the vectorization factor (VF) is bigger than the number
3527 of elements that we can fit in a vectype (nunits), we have to generate
3528 more than one vector stmt - i.e - we need to "unroll" the
3529 vector stmt by a factor VF/nunits. For more details see documentation in
3530 vect_get_vec_def_for_copy_stmt. */
3532 /* In case of interleaving (non-unit strided access):
3539 We create vectorized stores starting from base address (the access of the
3540 first stmt in the chain (S2 in the above example), when the last store stmt
3541 of the chain (S4) is reached:
3544 VS2: &base + vec_size*1 = vx0
3545 VS3: &base + vec_size*2 = vx1
3546 VS4: &base + vec_size*3 = vx3
3548 Then permutation statements are generated:
3550 VS5: vx5 = VEC_INTERLEAVE_HIGH_EXPR < vx0, vx3 >
3551 VS6: vx6 = VEC_INTERLEAVE_LOW_EXPR < vx0, vx3 >
3554 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3555 (the order of the data-refs in the output of vect_permute_store_chain
3556 corresponds to the order of scalar stmts in the interleaving chain - see
3557 the documentation of vect_permute_store_chain()).
3559 In case of both multiple types and interleaving, above vector stores and
3560 permutation stmts are created for every copy. The result vector stmts are
3561 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3562 STMT_VINFO_RELATED_STMT for the next copies.
3565 prev_stmt_info = NULL;
3566 for (j = 0; j < ncopies; j++)
3575 /* Get vectorized arguments for SLP_NODE. */
3576 vect_get_slp_defs (NULL_TREE, NULL_TREE, slp_node, &vec_oprnds,
3579 vec_oprnd = VEC_index (tree, vec_oprnds, 0);
3583 /* For interleaved stores we collect vectorized defs for all the
3584 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
3585 used as an input to vect_permute_store_chain(), and OPRNDS as
3586 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
3588 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3589 OPRNDS are of size 1. */
3590 next_stmt = first_stmt;
3591 for (i = 0; i < group_size; i++)
3593 /* Since gaps are not supported for interleaved stores,
3594 GROUP_SIZE is the exact number of stmts in the chain.
3595 Therefore, NEXT_STMT can't be NULL_TREE. In case that
3596 there is no interleaving, GROUP_SIZE is 1, and only one
3597 iteration of the loop will be executed. */
3598 gcc_assert (next_stmt
3599 && gimple_assign_single_p (next_stmt));
3600 op = gimple_assign_rhs1 (next_stmt);
3602 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
3604 VEC_quick_push(tree, dr_chain, vec_oprnd);
3605 VEC_quick_push(tree, oprnds, vec_oprnd);
3606 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3610 /* We should have catched mismatched types earlier. */
3611 gcc_assert (useless_type_conversion_p (vectype,
3612 TREE_TYPE (vec_oprnd)));
3613 dataref_ptr = vect_create_data_ref_ptr (first_stmt, NULL, NULL_TREE,
3614 &dummy, &ptr_incr, false,
3616 gcc_assert (bb_vinfo || !inv_p);
3620 /* For interleaved stores we created vectorized defs for all the
3621 defs stored in OPRNDS in the previous iteration (previous copy).
3622 DR_CHAIN is then used as an input to vect_permute_store_chain(),
3623 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
3625 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3626 OPRNDS are of size 1. */
3627 for (i = 0; i < group_size; i++)
3629 op = VEC_index (tree, oprnds, i);
3630 vect_is_simple_use (op, loop_vinfo, bb_vinfo, &def_stmt, &def,
3632 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
3633 VEC_replace(tree, dr_chain, i, vec_oprnd);
3634 VEC_replace(tree, oprnds, i, vec_oprnd);
3637 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
3642 result_chain = VEC_alloc (tree, heap, group_size);
3644 if (!vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
3649 next_stmt = first_stmt;
3650 for (i = 0; i < vec_num; i++)
3652 struct ptr_info_def *pi;
3655 /* Bump the vector pointer. */
3656 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
3660 vec_oprnd = VEC_index (tree, vec_oprnds, i);
3661 else if (strided_store)
3662 /* For strided stores vectorized defs are interleaved in
3663 vect_permute_store_chain(). */
3664 vec_oprnd = VEC_index (tree, result_chain, i);
3666 data_ref = build2 (MEM_REF, TREE_TYPE (vec_oprnd), dataref_ptr,
3667 build_int_cst (reference_alias_ptr_type
3668 (DR_REF (first_dr)), 0));
3669 pi = get_ptr_info (dataref_ptr);
3670 pi->align = TYPE_ALIGN_UNIT (vectype);
3671 if (aligned_access_p (first_dr))
3673 else if (DR_MISALIGNMENT (first_dr) == -1)
3675 TREE_TYPE (data_ref)
3676 = build_aligned_type (TREE_TYPE (data_ref),
3677 TYPE_ALIGN (TREE_TYPE (vectype)));
3678 pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
3683 TREE_TYPE (data_ref)
3684 = build_aligned_type (TREE_TYPE (data_ref),
3685 TYPE_ALIGN (TREE_TYPE (vectype)));
3686 pi->misalign = DR_MISALIGNMENT (first_dr);
3689 /* Arguments are ready. Create the new vector stmt. */
3690 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
3691 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3692 mark_symbols_for_renaming (new_stmt);
3698 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3700 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3702 prev_stmt_info = vinfo_for_stmt (new_stmt);
3703 next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
3709 VEC_free (tree, heap, dr_chain);
3710 VEC_free (tree, heap, oprnds);
3712 VEC_free (tree, heap, result_chain);
3714 VEC_free (tree, heap, vec_oprnds);
3719 /* Given a vector type VECTYPE returns a builtin DECL to be used
3720 for vector permutation and stores a mask into *MASK that implements
3721 reversal of the vector elements. If that is impossible to do
3722 returns NULL (and *MASK is unchanged). */
3725 perm_mask_for_reverse (tree vectype, tree *mask)
3728 tree mask_element_type, mask_type;
3729 tree mask_vec = NULL;
3732 if (!targetm.vectorize.builtin_vec_perm)
3735 builtin_decl = targetm.vectorize.builtin_vec_perm (vectype,
3736 &mask_element_type);
3737 if (!builtin_decl || !mask_element_type)
3740 mask_type = get_vectype_for_scalar_type (mask_element_type);
3741 nunits = TYPE_VECTOR_SUBPARTS (vectype);
3743 || TYPE_VECTOR_SUBPARTS (vectype) != TYPE_VECTOR_SUBPARTS (mask_type))
3746 for (i = 0; i < nunits; i++)
3747 mask_vec = tree_cons (NULL, build_int_cst (mask_element_type, i), mask_vec);
3748 mask_vec = build_vector (mask_type, mask_vec);
3750 if (!targetm.vectorize.builtin_vec_perm_ok (vectype, mask_vec))
3754 return builtin_decl;
3757 /* Given a vector variable X, that was generated for the scalar LHS of
3758 STMT, generate instructions to reverse the vector elements of X,
3759 insert them a *GSI and return the permuted vector variable. */
3762 reverse_vec_elements (tree x, gimple stmt, gimple_stmt_iterator *gsi)
3764 tree vectype = TREE_TYPE (x);
3765 tree mask_vec, builtin_decl;
3766 tree perm_dest, data_ref;
3769 builtin_decl = perm_mask_for_reverse (vectype, &mask_vec);
3771 perm_dest = vect_create_destination_var (gimple_assign_lhs (stmt), vectype);
3773 /* Generate the permute statement. */
3774 perm_stmt = gimple_build_call (builtin_decl, 3, x, x, mask_vec);
3775 if (!useless_type_conversion_p (vectype,
3776 TREE_TYPE (TREE_TYPE (builtin_decl))))
3778 tree tem = create_tmp_reg (TREE_TYPE (TREE_TYPE (builtin_decl)), NULL);
3779 tem = make_ssa_name (tem, perm_stmt);
3780 gimple_call_set_lhs (perm_stmt, tem);
3781 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
3782 perm_stmt = gimple_build_assign (NULL_TREE,
3783 build1 (VIEW_CONVERT_EXPR,
3786 data_ref = make_ssa_name (perm_dest, perm_stmt);
3787 gimple_set_lhs (perm_stmt, data_ref);
3788 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
3793 /* vectorizable_load.
3795 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
3797 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3798 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3799 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3802 vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
3803 slp_tree slp_node, slp_instance slp_node_instance)
3806 tree vec_dest = NULL;
3807 tree data_ref = NULL;
3808 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3809 stmt_vec_info prev_stmt_info;
3810 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3811 struct loop *loop = NULL;
3812 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
3813 bool nested_in_vect_loop = false;
3814 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
3815 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3817 enum machine_mode mode;
3818 gimple new_stmt = NULL;
3820 enum dr_alignment_support alignment_support_scheme;
3821 tree dataref_ptr = NULL_TREE;
3823 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
3825 int i, j, group_size;
3826 tree msq = NULL_TREE, lsq;
3827 tree offset = NULL_TREE;
3828 tree realignment_token = NULL_TREE;
3830 VEC(tree,heap) *dr_chain = NULL;
3831 bool strided_load = false;
3836 bool compute_in_loop = false;
3837 struct loop *at_loop;
3839 bool slp = (slp_node != NULL);
3840 bool slp_perm = false;
3841 enum tree_code code;
3842 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3847 loop = LOOP_VINFO_LOOP (loop_vinfo);
3848 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
3849 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
3854 /* Multiple types in SLP are handled by creating the appropriate number of
3855 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3860 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3862 gcc_assert (ncopies >= 1);
3864 /* FORNOW. This restriction should be relaxed. */
3865 if (nested_in_vect_loop && ncopies > 1)
3867 if (vect_print_dump_info (REPORT_DETAILS))
3868 fprintf (vect_dump, "multiple types in nested loop.");
3872 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3875 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3878 /* Is vectorizable load? */
3879 if (!is_gimple_assign (stmt))
3882 scalar_dest = gimple_assign_lhs (stmt);
3883 if (TREE_CODE (scalar_dest) != SSA_NAME)
3886 code = gimple_assign_rhs_code (stmt);
3887 if (code != ARRAY_REF
3888 && code != INDIRECT_REF
3889 && code != COMPONENT_REF
3890 && code != IMAGPART_EXPR
3891 && code != REALPART_EXPR
3895 if (!STMT_VINFO_DATA_REF (stmt_info))
3898 negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
3899 if (negative && ncopies > 1)
3901 if (vect_print_dump_info (REPORT_DETAILS))
3902 fprintf (vect_dump, "multiple types with negative step.");
3906 scalar_type = TREE_TYPE (DR_REF (dr));
3907 mode = TYPE_MODE (vectype);
3909 /* FORNOW. In some cases can vectorize even if data-type not supported
3910 (e.g. - data copies). */
3911 if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
3913 if (vect_print_dump_info (REPORT_DETAILS))
3914 fprintf (vect_dump, "Aligned load, but unsupported type.");
3918 /* The vector component type needs to be trivially convertible to the
3919 scalar lhs. This should always be the case. */
3920 if (!useless_type_conversion_p (TREE_TYPE (scalar_dest), TREE_TYPE (vectype)))
3922 if (vect_print_dump_info (REPORT_DETAILS))
3923 fprintf (vect_dump, "??? operands of different types");
3927 /* Check if the load is a part of an interleaving chain. */
3928 if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
3930 strided_load = true;
3932 gcc_assert (! nested_in_vect_loop);
3934 /* Check if interleaving is supported. */
3935 if (!vect_strided_load_supported (vectype)
3936 && !PURE_SLP_STMT (stmt_info) && !slp)
3942 gcc_assert (!strided_load);
3943 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
3944 if (alignment_support_scheme != dr_aligned
3945 && alignment_support_scheme != dr_unaligned_supported)
3947 if (vect_print_dump_info (REPORT_DETAILS))
3948 fprintf (vect_dump, "negative step but alignment required.");
3951 if (!perm_mask_for_reverse (vectype, NULL))
3953 if (vect_print_dump_info (REPORT_DETAILS))
3954 fprintf (vect_dump, "negative step and reversing not supported.");
3959 if (!vec_stmt) /* transformation not required. */
3961 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
3962 vect_model_load_cost (stmt_info, ncopies, NULL);
3966 if (vect_print_dump_info (REPORT_DETAILS))
3967 fprintf (vect_dump, "transform load.");
3973 first_stmt = DR_GROUP_FIRST_DR (stmt_info);
3974 /* Check if the chain of loads is already vectorized. */
3975 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt)))
3977 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3980 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
3981 group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt));
3983 /* VEC_NUM is the number of vect stmts to be created for this group. */
3986 strided_load = false;
3987 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
3988 if (SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance))
3992 vec_num = group_size;
3994 dr_chain = VEC_alloc (tree, heap, vec_num);
4000 group_size = vec_num = 1;
4003 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
4004 gcc_assert (alignment_support_scheme);
4006 /* In case the vectorization factor (VF) is bigger than the number
4007 of elements that we can fit in a vectype (nunits), we have to generate
4008 more than one vector stmt - i.e - we need to "unroll" the
4009 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4010 from one copy of the vector stmt to the next, in the field
4011 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4012 stages to find the correct vector defs to be used when vectorizing
4013 stmts that use the defs of the current stmt. The example below
4014 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
4015 need to create 4 vectorized stmts):
4017 before vectorization:
4018 RELATED_STMT VEC_STMT
4022 step 1: vectorize stmt S1:
4023 We first create the vector stmt VS1_0, and, as usual, record a
4024 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
4025 Next, we create the vector stmt VS1_1, and record a pointer to
4026 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
4027 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
4029 RELATED_STMT VEC_STMT
4030 VS1_0: vx0 = memref0 VS1_1 -
4031 VS1_1: vx1 = memref1 VS1_2 -
4032 VS1_2: vx2 = memref2 VS1_3 -
4033 VS1_3: vx3 = memref3 - -
4034 S1: x = load - VS1_0
4037 See in documentation in vect_get_vec_def_for_stmt_copy for how the
4038 information we recorded in RELATED_STMT field is used to vectorize
4041 /* In case of interleaving (non-unit strided access):
4048 Vectorized loads are created in the order of memory accesses
4049 starting from the access of the first stmt of the chain:
4052 VS2: vx1 = &base + vec_size*1
4053 VS3: vx3 = &base + vec_size*2
4054 VS4: vx4 = &base + vec_size*3
4056 Then permutation statements are generated:
4058 VS5: vx5 = VEC_EXTRACT_EVEN_EXPR < vx0, vx1 >
4059 VS6: vx6 = VEC_EXTRACT_ODD_EXPR < vx0, vx1 >
4062 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
4063 (the order of the data-refs in the output of vect_permute_load_chain
4064 corresponds to the order of scalar stmts in the interleaving chain - see
4065 the documentation of vect_permute_load_chain()).
4066 The generation of permutation stmts and recording them in
4067 STMT_VINFO_VEC_STMT is done in vect_transform_strided_load().
4069 In case of both multiple types and interleaving, the vector loads and
4070 permutation stmts above are created for every copy. The result vector
4071 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
4072 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
4074 /* If the data reference is aligned (dr_aligned) or potentially unaligned
4075 on a target that supports unaligned accesses (dr_unaligned_supported)
4076 we generate the following code:
4080 p = p + indx * vectype_size;
4085 Otherwise, the data reference is potentially unaligned on a target that
4086 does not support unaligned accesses (dr_explicit_realign_optimized) -
4087 then generate the following code, in which the data in each iteration is
4088 obtained by two vector loads, one from the previous iteration, and one
4089 from the current iteration:
4091 msq_init = *(floor(p1))
4092 p2 = initial_addr + VS - 1;
4093 realignment_token = call target_builtin;
4096 p2 = p2 + indx * vectype_size
4098 vec_dest = realign_load (msq, lsq, realignment_token)
4103 /* If the misalignment remains the same throughout the execution of the
4104 loop, we can create the init_addr and permutation mask at the loop
4105 preheader. Otherwise, it needs to be created inside the loop.
4106 This can only occur when vectorizing memory accesses in the inner-loop
4107 nested within an outer-loop that is being vectorized. */
4109 if (loop && nested_in_vect_loop_p (loop, stmt)
4110 && (TREE_INT_CST_LOW (DR_STEP (dr))
4111 % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
4113 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
4114 compute_in_loop = true;
4117 if ((alignment_support_scheme == dr_explicit_realign_optimized
4118 || alignment_support_scheme == dr_explicit_realign)
4119 && !compute_in_loop)
4121 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
4122 alignment_support_scheme, NULL_TREE,
4124 if (alignment_support_scheme == dr_explicit_realign_optimized)
4126 phi = SSA_NAME_DEF_STMT (msq);
4127 offset = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
4134 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
4136 prev_stmt_info = NULL;
4137 for (j = 0; j < ncopies; j++)
4139 /* 1. Create the vector pointer update chain. */
4141 dataref_ptr = vect_create_data_ref_ptr (first_stmt,
4143 &dummy, &ptr_incr, false,
4147 bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, NULL_TREE);
4149 for (i = 0; i < vec_num; i++)
4152 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
4155 /* 2. Create the vector-load in the loop. */
4156 switch (alignment_support_scheme)
4159 case dr_unaligned_supported:
4161 struct ptr_info_def *pi;
4163 = build2 (MEM_REF, vectype, dataref_ptr,
4164 build_int_cst (reference_alias_ptr_type
4165 (DR_REF (first_dr)), 0));
4166 pi = get_ptr_info (dataref_ptr);
4167 pi->align = TYPE_ALIGN_UNIT (vectype);
4168 if (alignment_support_scheme == dr_aligned)
4170 gcc_assert (aligned_access_p (first_dr));
4173 else if (DR_MISALIGNMENT (first_dr) == -1)
4175 TREE_TYPE (data_ref)
4176 = build_aligned_type (TREE_TYPE (data_ref),
4177 TYPE_ALIGN (TREE_TYPE (vectype)));
4178 pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
4183 TREE_TYPE (data_ref)
4184 = build_aligned_type (TREE_TYPE (data_ref),
4185 TYPE_ALIGN (TREE_TYPE (vectype)));
4186 pi->misalign = DR_MISALIGNMENT (first_dr);
4190 case dr_explicit_realign:
4193 tree vs_minus_1 = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
4195 if (compute_in_loop)
4196 msq = vect_setup_realignment (first_stmt, gsi,
4198 dr_explicit_realign,
4201 new_stmt = gimple_build_assign_with_ops
4202 (BIT_AND_EXPR, NULL_TREE, dataref_ptr,
4204 (TREE_TYPE (dataref_ptr),
4205 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
4206 ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
4207 gimple_assign_set_lhs (new_stmt, ptr);
4208 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4210 = build2 (MEM_REF, vectype, ptr,
4211 build_int_cst (reference_alias_ptr_type
4212 (DR_REF (first_dr)), 0));
4213 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4214 new_stmt = gimple_build_assign (vec_dest, data_ref);
4215 new_temp = make_ssa_name (vec_dest, new_stmt);
4216 gimple_assign_set_lhs (new_stmt, new_temp);
4217 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
4218 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
4219 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4222 bump = size_binop (MULT_EXPR, vs_minus_1,
4223 TYPE_SIZE_UNIT (scalar_type));
4224 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
4225 new_stmt = gimple_build_assign_with_ops
4226 (BIT_AND_EXPR, NULL_TREE, ptr,
4229 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
4230 ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
4231 gimple_assign_set_lhs (new_stmt, ptr);
4232 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4234 = build2 (MEM_REF, vectype, ptr,
4235 build_int_cst (reference_alias_ptr_type
4236 (DR_REF (first_dr)), 0));
4239 case dr_explicit_realign_optimized:
4240 new_stmt = gimple_build_assign_with_ops
4241 (BIT_AND_EXPR, NULL_TREE, dataref_ptr,
4243 (TREE_TYPE (dataref_ptr),
4244 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
4245 new_temp = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt);
4246 gimple_assign_set_lhs (new_stmt, new_temp);
4247 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4249 = build2 (MEM_REF, vectype, new_temp,
4250 build_int_cst (reference_alias_ptr_type
4251 (DR_REF (first_dr)), 0));
4256 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4257 new_stmt = gimple_build_assign (vec_dest, data_ref);
4258 new_temp = make_ssa_name (vec_dest, new_stmt);
4259 gimple_assign_set_lhs (new_stmt, new_temp);
4260 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4261 mark_symbols_for_renaming (new_stmt);
4263 /* 3. Handle explicit realignment if necessary/supported. Create in
4264 loop: vec_dest = realign_load (msq, lsq, realignment_token) */
4265 if (alignment_support_scheme == dr_explicit_realign_optimized
4266 || alignment_support_scheme == dr_explicit_realign)
4270 lsq = gimple_assign_lhs (new_stmt);
4271 if (!realignment_token)
4272 realignment_token = dataref_ptr;
4273 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4274 tmp = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq,
4276 new_stmt = gimple_build_assign (vec_dest, tmp);
4277 new_temp = make_ssa_name (vec_dest, new_stmt);
4278 gimple_assign_set_lhs (new_stmt, new_temp);
4279 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4281 if (alignment_support_scheme == dr_explicit_realign_optimized)
4284 if (i == vec_num - 1 && j == ncopies - 1)
4285 add_phi_arg (phi, lsq, loop_latch_edge (containing_loop),
4291 /* 4. Handle invariant-load. */
4292 if (inv_p && !bb_vinfo)
4294 gcc_assert (!strided_load);
4295 gcc_assert (nested_in_vect_loop_p (loop, stmt));
4300 tree vec_inv, bitpos, bitsize = TYPE_SIZE (scalar_type);
4302 /* CHECKME: bitpos depends on endianess? */
4303 bitpos = bitsize_zero_node;
4304 vec_inv = build3 (BIT_FIELD_REF, scalar_type, new_temp,
4307 vect_create_destination_var (scalar_dest, NULL_TREE);
4308 new_stmt = gimple_build_assign (vec_dest, vec_inv);
4309 new_temp = make_ssa_name (vec_dest, new_stmt);
4310 gimple_assign_set_lhs (new_stmt, new_temp);
4311 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4313 for (k = nunits - 1; k >= 0; --k)
4314 t = tree_cons (NULL_TREE, new_temp, t);
4315 /* FIXME: use build_constructor directly. */
4316 vec_inv = build_constructor_from_list (vectype, t);
4317 new_temp = vect_init_vector (stmt, vec_inv, vectype, gsi);
4318 new_stmt = SSA_NAME_DEF_STMT (new_temp);
4321 gcc_unreachable (); /* FORNOW. */
4326 new_temp = reverse_vec_elements (new_temp, stmt, gsi);
4327 new_stmt = SSA_NAME_DEF_STMT (new_temp);
4330 /* Collect vector loads and later create their permutation in
4331 vect_transform_strided_load (). */
4332 if (strided_load || slp_perm)
4333 VEC_quick_push (tree, dr_chain, new_temp);
4335 /* Store vector loads in the corresponding SLP_NODE. */
4336 if (slp && !slp_perm)
4337 VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (slp_node), new_stmt);
4340 if (slp && !slp_perm)
4345 if (!vect_transform_slp_perm_load (stmt, dr_chain, gsi, vf,
4346 slp_node_instance, false))
4348 VEC_free (tree, heap, dr_chain);
4356 if (!vect_transform_strided_load (stmt, dr_chain, group_size, gsi))
4359 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
4360 VEC_free (tree, heap, dr_chain);
4361 dr_chain = VEC_alloc (tree, heap, group_size);
4366 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4368 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4369 prev_stmt_info = vinfo_for_stmt (new_stmt);
4375 VEC_free (tree, heap, dr_chain);
4380 /* Function vect_is_simple_cond.
4383 LOOP - the loop that is being vectorized.
4384 COND - Condition that is checked for simple use.
4386 Returns whether a COND can be vectorized. Checks whether
4387 condition operands are supportable using vec_is_simple_use. */
4390 vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
4394 enum vect_def_type dt;
4396 if (!COMPARISON_CLASS_P (cond))
4399 lhs = TREE_OPERAND (cond, 0);
4400 rhs = TREE_OPERAND (cond, 1);
4402 if (TREE_CODE (lhs) == SSA_NAME)
4404 gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
4405 if (!vect_is_simple_use (lhs, loop_vinfo, NULL, &lhs_def_stmt, &def,
4409 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
4410 && TREE_CODE (lhs) != FIXED_CST)
4413 if (TREE_CODE (rhs) == SSA_NAME)
4415 gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
4416 if (!vect_is_simple_use (rhs, loop_vinfo, NULL, &rhs_def_stmt, &def,
4420 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
4421 && TREE_CODE (rhs) != FIXED_CST)
4427 /* vectorizable_condition.
4429 Check if STMT is conditional modify expression that can be vectorized.
4430 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4431 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
4434 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
4435 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
4436 else caluse if it is 2).
4438 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4441 vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
4442 gimple *vec_stmt, tree reduc_def, int reduc_index)
4444 tree scalar_dest = NULL_TREE;
4445 tree vec_dest = NULL_TREE;
4446 tree op = NULL_TREE;
4447 tree cond_expr, then_clause, else_clause;
4448 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4449 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4450 tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
4451 tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
4452 tree vec_compare, vec_cond_expr;
4454 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4455 enum machine_mode vec_mode;
4457 enum vect_def_type dt, dts[4];
4458 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
4459 int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
4460 enum tree_code code;
4461 stmt_vec_info prev_stmt_info = NULL;
4464 /* FORNOW: unsupported in basic block SLP. */
4465 gcc_assert (loop_vinfo);
4467 gcc_assert (ncopies >= 1);
4468 if (reduc_index && ncopies > 1)
4469 return false; /* FORNOW */
4471 if (!STMT_VINFO_RELEVANT_P (stmt_info))
4474 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4475 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
4479 /* FORNOW: SLP not supported. */
4480 if (STMT_SLP_TYPE (stmt_info))
4483 /* FORNOW: not yet supported. */
4484 if (STMT_VINFO_LIVE_P (stmt_info))
4486 if (vect_print_dump_info (REPORT_DETAILS))
4487 fprintf (vect_dump, "value used after loop.");
4491 /* Is vectorizable conditional operation? */
4492 if (!is_gimple_assign (stmt))
4495 code = gimple_assign_rhs_code (stmt);
4497 if (code != COND_EXPR)
4500 gcc_assert (gimple_assign_single_p (stmt));
4501 op = gimple_assign_rhs1 (stmt);
4502 cond_expr = TREE_OPERAND (op, 0);
4503 then_clause = TREE_OPERAND (op, 1);
4504 else_clause = TREE_OPERAND (op, 2);
4506 if (!vect_is_simple_cond (cond_expr, loop_vinfo))
4509 /* We do not handle two different vector types for the condition
4511 if (!types_compatible_p (TREE_TYPE (TREE_OPERAND (cond_expr, 0)),
4512 TREE_TYPE (vectype)))
4515 if (TREE_CODE (then_clause) == SSA_NAME)
4517 gimple then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
4518 if (!vect_is_simple_use (then_clause, loop_vinfo, NULL,
4519 &then_def_stmt, &def, &dt))
4522 else if (TREE_CODE (then_clause) != INTEGER_CST
4523 && TREE_CODE (then_clause) != REAL_CST
4524 && TREE_CODE (then_clause) != FIXED_CST)
4527 if (TREE_CODE (else_clause) == SSA_NAME)
4529 gimple else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
4530 if (!vect_is_simple_use (else_clause, loop_vinfo, NULL,
4531 &else_def_stmt, &def, &dt))
4534 else if (TREE_CODE (else_clause) != INTEGER_CST
4535 && TREE_CODE (else_clause) != REAL_CST
4536 && TREE_CODE (else_clause) != FIXED_CST)
4540 vec_mode = TYPE_MODE (vectype);
4544 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
4545 return expand_vec_cond_expr_p (TREE_TYPE (op), vec_mode);
4551 scalar_dest = gimple_assign_lhs (stmt);
4552 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4554 /* Handle cond expr. */
4555 for (j = 0; j < ncopies; j++)
4562 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
4564 vect_is_simple_use (TREE_OPERAND (cond_expr, 0), loop_vinfo,
4565 NULL, >emp, &def, &dts[0]);
4567 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
4569 vect_is_simple_use (TREE_OPERAND (cond_expr, 1), loop_vinfo,
4570 NULL, >emp, &def, &dts[1]);
4571 if (reduc_index == 1)
4572 vec_then_clause = reduc_def;
4575 vec_then_clause = vect_get_vec_def_for_operand (then_clause,
4577 vect_is_simple_use (then_clause, loop_vinfo,
4578 NULL, >emp, &def, &dts[2]);
4580 if (reduc_index == 2)
4581 vec_else_clause = reduc_def;
4584 vec_else_clause = vect_get_vec_def_for_operand (else_clause,
4586 vect_is_simple_use (else_clause, loop_vinfo,
4587 NULL, >emp, &def, &dts[3]);
4592 vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], vec_cond_lhs);
4593 vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], vec_cond_rhs);
4594 vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
4596 vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
4600 /* Arguments are ready. Create the new vector stmt. */
4601 vec_compare = build2 (TREE_CODE (cond_expr), vectype,
4602 vec_cond_lhs, vec_cond_rhs);
4603 vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
4604 vec_compare, vec_then_clause, vec_else_clause);
4606 new_stmt = gimple_build_assign (vec_dest, vec_cond_expr);
4607 new_temp = make_ssa_name (vec_dest, new_stmt);
4608 gimple_assign_set_lhs (new_stmt, new_temp);
4609 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4611 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4613 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4615 prev_stmt_info = vinfo_for_stmt (new_stmt);
4622 /* Make sure the statement is vectorizable. */
4625 vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
4627 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4628 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4629 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
4631 tree scalar_type, vectype;
4633 if (vect_print_dump_info (REPORT_DETAILS))
4635 fprintf (vect_dump, "==> examining statement: ");
4636 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4639 if (gimple_has_volatile_ops (stmt))
4641 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4642 fprintf (vect_dump, "not vectorized: stmt has volatile operands");
4647 /* Skip stmts that do not need to be vectorized. In loops this is expected
4649 - the COND_EXPR which is the loop exit condition
4650 - any LABEL_EXPRs in the loop
4651 - computations that are used only for array indexing or loop control.
4652 In basic blocks we only analyze statements that are a part of some SLP
4653 instance, therefore, all the statements are relevant. */
4655 if (!STMT_VINFO_RELEVANT_P (stmt_info)
4656 && !STMT_VINFO_LIVE_P (stmt_info))
4658 if (vect_print_dump_info (REPORT_DETAILS))
4659 fprintf (vect_dump, "irrelevant.");
4664 switch (STMT_VINFO_DEF_TYPE (stmt_info))
4666 case vect_internal_def:
4669 case vect_reduction_def:
4670 case vect_nested_cycle:
4671 gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
4672 || relevance == vect_used_in_outer_by_reduction
4673 || relevance == vect_unused_in_scope));
4676 case vect_induction_def:
4677 case vect_constant_def:
4678 case vect_external_def:
4679 case vect_unknown_def_type:
4686 gcc_assert (PURE_SLP_STMT (stmt_info));
4688 scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
4689 if (vect_print_dump_info (REPORT_DETAILS))
4691 fprintf (vect_dump, "get vectype for scalar type: ");
4692 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4695 vectype = get_vectype_for_scalar_type (scalar_type);
4698 if (vect_print_dump_info (REPORT_DETAILS))
4700 fprintf (vect_dump, "not SLPed: unsupported data-type ");
4701 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
4706 if (vect_print_dump_info (REPORT_DETAILS))
4708 fprintf (vect_dump, "vectype: ");
4709 print_generic_expr (vect_dump, vectype, TDF_SLIM);
4712 STMT_VINFO_VECTYPE (stmt_info) = vectype;
4715 if (STMT_VINFO_RELEVANT_P (stmt_info))
4717 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
4718 gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
4719 *need_to_vectorize = true;
4724 && (STMT_VINFO_RELEVANT_P (stmt_info)
4725 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
4726 ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL)
4727 || vectorizable_type_demotion (stmt, NULL, NULL, NULL)
4728 || vectorizable_conversion (stmt, NULL, NULL, NULL)
4729 || vectorizable_shift (stmt, NULL, NULL, NULL)
4730 || vectorizable_operation (stmt, NULL, NULL, NULL)
4731 || vectorizable_assignment (stmt, NULL, NULL, NULL)
4732 || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
4733 || vectorizable_call (stmt, NULL, NULL)
4734 || vectorizable_store (stmt, NULL, NULL, NULL)
4735 || vectorizable_reduction (stmt, NULL, NULL, NULL)
4736 || vectorizable_condition (stmt, NULL, NULL, NULL, 0));
4740 ok = (vectorizable_shift (stmt, NULL, NULL, node)
4741 || vectorizable_operation (stmt, NULL, NULL, node)
4742 || vectorizable_assignment (stmt, NULL, NULL, node)
4743 || vectorizable_load (stmt, NULL, NULL, node, NULL)
4744 || vectorizable_store (stmt, NULL, NULL, node));
4749 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4751 fprintf (vect_dump, "not vectorized: relevant stmt not ");
4752 fprintf (vect_dump, "supported: ");
4753 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4762 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
4763 need extra handling, except for vectorizable reductions. */
4764 if (STMT_VINFO_LIVE_P (stmt_info)
4765 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4766 ok = vectorizable_live_operation (stmt, NULL, NULL);
4770 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
4772 fprintf (vect_dump, "not vectorized: live stmt not ");
4773 fprintf (vect_dump, "supported: ");
4774 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4780 if (!PURE_SLP_STMT (stmt_info))
4782 /* Groups of strided accesses whose size is not a power of 2 are not
4783 vectorizable yet using loop-vectorization. Therefore, if this stmt
4784 feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
4785 loop-based vectorized), the loop cannot be vectorized. */
4786 if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
4787 && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
4788 DR_GROUP_FIRST_DR (stmt_info)))) == -1)
4790 if (vect_print_dump_info (REPORT_DETAILS))
4792 fprintf (vect_dump, "not vectorized: the size of group "
4793 "of strided accesses is not a power of 2");
4794 print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
4805 /* Function vect_transform_stmt.
4807 Create a vectorized stmt to replace STMT, and insert it at BSI. */
4810 vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi,
4811 bool *strided_store, slp_tree slp_node,
4812 slp_instance slp_node_instance)
4814 bool is_store = false;
4815 gimple vec_stmt = NULL;
4816 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4817 gimple orig_stmt_in_pattern;
4820 switch (STMT_VINFO_TYPE (stmt_info))
4822 case type_demotion_vec_info_type:
4823 done = vectorizable_type_demotion (stmt, gsi, &vec_stmt, slp_node);
4827 case type_promotion_vec_info_type:
4828 done = vectorizable_type_promotion (stmt, gsi, &vec_stmt, slp_node);
4832 case type_conversion_vec_info_type:
4833 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
4837 case induc_vec_info_type:
4838 gcc_assert (!slp_node);
4839 done = vectorizable_induction (stmt, gsi, &vec_stmt);
4843 case shift_vec_info_type:
4844 done = vectorizable_shift (stmt, gsi, &vec_stmt, slp_node);
4848 case op_vec_info_type:
4849 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
4853 case assignment_vec_info_type:
4854 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
4858 case load_vec_info_type:
4859 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
4864 case store_vec_info_type:
4865 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
4867 if (STMT_VINFO_STRIDED_ACCESS (stmt_info) && !slp_node)
4869 /* In case of interleaving, the whole chain is vectorized when the
4870 last store in the chain is reached. Store stmts before the last
4871 one are skipped, and there vec_stmt_info shouldn't be freed
4873 *strided_store = true;
4874 if (STMT_VINFO_VEC_STMT (stmt_info))
4881 case condition_vec_info_type:
4882 gcc_assert (!slp_node);
4883 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0);
4887 case call_vec_info_type:
4888 gcc_assert (!slp_node);
4889 done = vectorizable_call (stmt, gsi, &vec_stmt);
4890 stmt = gsi_stmt (*gsi);
4893 case reduc_vec_info_type:
4894 done = vectorizable_reduction (stmt, gsi, &vec_stmt, slp_node);
4899 if (!STMT_VINFO_LIVE_P (stmt_info))
4901 if (vect_print_dump_info (REPORT_DETAILS))
4902 fprintf (vect_dump, "stmt not supported.");
4907 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
4908 is being vectorized, but outside the immediately enclosing loop. */
4910 && STMT_VINFO_LOOP_VINFO (stmt_info)
4911 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
4912 STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
4913 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
4914 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
4915 || STMT_VINFO_RELEVANT (stmt_info) ==
4916 vect_used_in_outer_by_reduction))
4918 struct loop *innerloop = LOOP_VINFO_LOOP (
4919 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
4920 imm_use_iterator imm_iter;
4921 use_operand_p use_p;
4925 if (vect_print_dump_info (REPORT_DETAILS))
4926 fprintf (vect_dump, "Record the vdef for outer-loop vectorization.");
4928 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
4929 (to be used when vectorizing outer-loop stmts that use the DEF of
4931 if (gimple_code (stmt) == GIMPLE_PHI)
4932 scalar_dest = PHI_RESULT (stmt);
4934 scalar_dest = gimple_assign_lhs (stmt);
4936 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
4938 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
4940 exit_phi = USE_STMT (use_p);
4941 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
4946 /* Handle stmts whose DEF is used outside the loop-nest that is
4947 being vectorized. */
4948 if (STMT_VINFO_LIVE_P (stmt_info)
4949 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
4951 done = vectorizable_live_operation (stmt, gsi, &vec_stmt);
4957 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
4958 orig_stmt_in_pattern = STMT_VINFO_RELATED_STMT (stmt_info);
4959 if (orig_stmt_in_pattern)
4961 stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt_in_pattern);
4962 /* STMT was inserted by the vectorizer to replace a computation idiom.
4963 ORIG_STMT_IN_PATTERN is a stmt in the original sequence that
4964 computed this idiom. We need to record a pointer to VEC_STMT in
4965 the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the
4966 documentation of vect_pattern_recog. */
4967 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
4968 STMT_VINFO_VEC_STMT (stmt_vinfo) = vec_stmt;
4976 /* Remove a group of stores (for SLP or interleaving), free their
4980 vect_remove_stores (gimple first_stmt)
4982 gimple next = first_stmt;
4984 gimple_stmt_iterator next_si;
4988 /* Free the attached stmt_vec_info and remove the stmt. */
4989 next_si = gsi_for_stmt (next);
4990 gsi_remove (&next_si, true);
4991 tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
4992 free_stmt_vec_info (next);
4998 /* Function new_stmt_vec_info.
5000 Create and initialize a new stmt_vec_info struct for STMT. */
5003 new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
5004 bb_vec_info bb_vinfo)
5007 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
5009 STMT_VINFO_TYPE (res) = undef_vec_info_type;
5010 STMT_VINFO_STMT (res) = stmt;
5011 STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
5012 STMT_VINFO_BB_VINFO (res) = bb_vinfo;
5013 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
5014 STMT_VINFO_LIVE_P (res) = false;
5015 STMT_VINFO_VECTYPE (res) = NULL;
5016 STMT_VINFO_VEC_STMT (res) = NULL;
5017 STMT_VINFO_VECTORIZABLE (res) = true;
5018 STMT_VINFO_IN_PATTERN_P (res) = false;
5019 STMT_VINFO_RELATED_STMT (res) = NULL;
5020 STMT_VINFO_DATA_REF (res) = NULL;
5022 STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
5023 STMT_VINFO_DR_OFFSET (res) = NULL;
5024 STMT_VINFO_DR_INIT (res) = NULL;
5025 STMT_VINFO_DR_STEP (res) = NULL;
5026 STMT_VINFO_DR_ALIGNED_TO (res) = NULL;
5028 if (gimple_code (stmt) == GIMPLE_PHI
5029 && is_loop_header_bb_p (gimple_bb (stmt)))
5030 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
5032 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
5034 STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5);
5035 STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0;
5036 STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0;
5037 STMT_SLP_TYPE (res) = loop_vect;
5038 DR_GROUP_FIRST_DR (res) = NULL;
5039 DR_GROUP_NEXT_DR (res) = NULL;
5040 DR_GROUP_SIZE (res) = 0;
5041 DR_GROUP_STORE_COUNT (res) = 0;
5042 DR_GROUP_GAP (res) = 0;
5043 DR_GROUP_SAME_DR_STMT (res) = NULL;
5044 DR_GROUP_READ_WRITE_DEPENDENCE (res) = false;
5050 /* Create a hash table for stmt_vec_info. */
5053 init_stmt_vec_info_vec (void)
5055 gcc_assert (!stmt_vec_info_vec);
5056 stmt_vec_info_vec = VEC_alloc (vec_void_p, heap, 50);
5060 /* Free hash table for stmt_vec_info. */
5063 free_stmt_vec_info_vec (void)
5065 gcc_assert (stmt_vec_info_vec);
5066 VEC_free (vec_void_p, heap, stmt_vec_info_vec);
5070 /* Free stmt vectorization related info. */
5073 free_stmt_vec_info (gimple stmt)
5075 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5080 VEC_free (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmt_info));
5081 set_vinfo_for_stmt (stmt, NULL);
5086 /* Function get_vectype_for_scalar_type_and_size.
5088 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
5092 get_vectype_for_scalar_type_and_size (tree scalar_type, unsigned size)
5094 enum machine_mode inner_mode = TYPE_MODE (scalar_type);
5095 enum machine_mode simd_mode;
5096 unsigned int nbytes = GET_MODE_SIZE (inner_mode);
5103 /* We can't build a vector type of elements with alignment bigger than
5105 if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
5108 /* If we'd build a vector type of elements whose mode precision doesn't
5109 match their types precision we'll get mismatched types on vector
5110 extracts via BIT_FIELD_REFs. This effectively means we disable
5111 vectorization of bool and/or enum types in some languages. */
5112 if (INTEGRAL_TYPE_P (scalar_type)
5113 && GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type))
5116 if (GET_MODE_CLASS (inner_mode) != MODE_INT
5117 && GET_MODE_CLASS (inner_mode) != MODE_FLOAT)
5120 /* If no size was supplied use the mode the target prefers. Otherwise
5121 lookup a vector mode of the specified size. */
5123 simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
5125 simd_mode = mode_for_vector (inner_mode, size / nbytes);
5126 nunits = GET_MODE_SIZE (simd_mode) / nbytes;
5130 vectype = build_vector_type (scalar_type, nunits);
5131 if (vect_print_dump_info (REPORT_DETAILS))
5133 fprintf (vect_dump, "get vectype with %d units of type ", nunits);
5134 print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
5140 if (vect_print_dump_info (REPORT_DETAILS))
5142 fprintf (vect_dump, "vectype: ");
5143 print_generic_expr (vect_dump, vectype, TDF_SLIM);
5146 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
5147 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
5149 if (vect_print_dump_info (REPORT_DETAILS))
5150 fprintf (vect_dump, "mode not supported by target.");
5157 unsigned int current_vector_size;
5159 /* Function get_vectype_for_scalar_type.
5161 Returns the vector type corresponding to SCALAR_TYPE as supported
5165 get_vectype_for_scalar_type (tree scalar_type)
5168 vectype = get_vectype_for_scalar_type_and_size (scalar_type,
5169 current_vector_size);
5171 && current_vector_size == 0)
5172 current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
5176 /* Function get_same_sized_vectype
5178 Returns a vector type corresponding to SCALAR_TYPE of size
5179 VECTOR_TYPE if supported by the target. */
5182 get_same_sized_vectype (tree scalar_type, tree vector_type)
5184 return get_vectype_for_scalar_type_and_size
5185 (scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
5188 /* Function vect_is_simple_use.
5191 LOOP_VINFO - the vect info of the loop that is being vectorized.
5192 BB_VINFO - the vect info of the basic block that is being vectorized.
5193 OPERAND - operand of a stmt in the loop or bb.
5194 DEF - the defining stmt in case OPERAND is an SSA_NAME.
5196 Returns whether a stmt with OPERAND can be vectorized.
5197 For loops, supportable operands are constants, loop invariants, and operands
5198 that are defined by the current iteration of the loop. Unsupportable
5199 operands are those that are defined by a previous iteration of the loop (as
5200 is the case in reduction/induction computations).
5201 For basic blocks, supportable operands are constants and bb invariants.
5202 For now, operands defined outside the basic block are not supported. */
5205 vect_is_simple_use (tree operand, loop_vec_info loop_vinfo,
5206 bb_vec_info bb_vinfo, gimple *def_stmt,
5207 tree *def, enum vect_def_type *dt)
5210 stmt_vec_info stmt_vinfo;
5211 struct loop *loop = NULL;
5214 loop = LOOP_VINFO_LOOP (loop_vinfo);
5219 if (vect_print_dump_info (REPORT_DETAILS))
5221 fprintf (vect_dump, "vect_is_simple_use: operand ");
5222 print_generic_expr (vect_dump, operand, TDF_SLIM);
5225 if (TREE_CODE (operand) == INTEGER_CST || TREE_CODE (operand) == REAL_CST)
5227 *dt = vect_constant_def;
5231 if (is_gimple_min_invariant (operand))
5234 *dt = vect_external_def;
5238 if (TREE_CODE (operand) == PAREN_EXPR)
5240 if (vect_print_dump_info (REPORT_DETAILS))
5241 fprintf (vect_dump, "non-associatable copy.");
5242 operand = TREE_OPERAND (operand, 0);
5245 if (TREE_CODE (operand) != SSA_NAME)
5247 if (vect_print_dump_info (REPORT_DETAILS))
5248 fprintf (vect_dump, "not ssa-name.");
5252 *def_stmt = SSA_NAME_DEF_STMT (operand);
5253 if (*def_stmt == NULL)
5255 if (vect_print_dump_info (REPORT_DETAILS))
5256 fprintf (vect_dump, "no def_stmt.");
5260 if (vect_print_dump_info (REPORT_DETAILS))
5262 fprintf (vect_dump, "def_stmt: ");
5263 print_gimple_stmt (vect_dump, *def_stmt, 0, TDF_SLIM);
5266 /* Empty stmt is expected only in case of a function argument.
5267 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
5268 if (gimple_nop_p (*def_stmt))
5271 *dt = vect_external_def;
5275 bb = gimple_bb (*def_stmt);
5277 if ((loop && !flow_bb_inside_loop_p (loop, bb))
5278 || (!loop && bb != BB_VINFO_BB (bb_vinfo))
5279 || (!loop && gimple_code (*def_stmt) == GIMPLE_PHI))
5280 *dt = vect_external_def;
5283 stmt_vinfo = vinfo_for_stmt (*def_stmt);
5284 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
5287 if (*dt == vect_unknown_def_type)
5289 if (vect_print_dump_info (REPORT_DETAILS))
5290 fprintf (vect_dump, "Unsupported pattern.");
5294 if (vect_print_dump_info (REPORT_DETAILS))
5295 fprintf (vect_dump, "type of def: %d.",*dt);
5297 switch (gimple_code (*def_stmt))
5300 *def = gimple_phi_result (*def_stmt);
5304 *def = gimple_assign_lhs (*def_stmt);
5308 *def = gimple_call_lhs (*def_stmt);
5313 if (vect_print_dump_info (REPORT_DETAILS))
5314 fprintf (vect_dump, "unsupported defining stmt: ");
5321 /* Function vect_is_simple_use_1.
5323 Same as vect_is_simple_use_1 but also determines the vector operand
5324 type of OPERAND and stores it to *VECTYPE. If the definition of
5325 OPERAND is vect_uninitialized_def, vect_constant_def or
5326 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
5327 is responsible to compute the best suited vector type for the
5331 vect_is_simple_use_1 (tree operand, loop_vec_info loop_vinfo,
5332 bb_vec_info bb_vinfo, gimple *def_stmt,
5333 tree *def, enum vect_def_type *dt, tree *vectype)
5335 if (!vect_is_simple_use (operand, loop_vinfo, bb_vinfo, def_stmt, def, dt))
5338 /* Now get a vector type if the def is internal, otherwise supply
5339 NULL_TREE and leave it up to the caller to figure out a proper
5340 type for the use stmt. */
5341 if (*dt == vect_internal_def
5342 || *dt == vect_induction_def
5343 || *dt == vect_reduction_def
5344 || *dt == vect_double_reduction_def
5345 || *dt == vect_nested_cycle)
5347 stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
5348 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
5349 stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
5350 *vectype = STMT_VINFO_VECTYPE (stmt_info);
5351 gcc_assert (*vectype != NULL_TREE);
5353 else if (*dt == vect_uninitialized_def
5354 || *dt == vect_constant_def
5355 || *dt == vect_external_def)
5356 *vectype = NULL_TREE;
5364 /* Function supportable_widening_operation
5366 Check whether an operation represented by the code CODE is a
5367 widening operation that is supported by the target platform in
5368 vector form (i.e., when operating on arguments of type VECTYPE_IN
5369 producing a result of type VECTYPE_OUT).
5371 Widening operations we currently support are NOP (CONVERT), FLOAT
5372 and WIDEN_MULT. This function checks if these operations are supported
5373 by the target platform either directly (via vector tree-codes), or via
5377 - CODE1 and CODE2 are codes of vector operations to be used when
5378 vectorizing the operation, if available.
5379 - DECL1 and DECL2 are decls of target builtin functions to be used
5380 when vectorizing the operation, if available. In this case,
5381 CODE1 and CODE2 are CALL_EXPR.
5382 - MULTI_STEP_CVT determines the number of required intermediate steps in
5383 case of multi-step conversion (like char->short->int - in that case
5384 MULTI_STEP_CVT will be 1).
5385 - INTERM_TYPES contains the intermediate type required to perform the
5386 widening operation (short in the above example). */
5389 supportable_widening_operation (enum tree_code code, gimple stmt,
5390 tree vectype_out, tree vectype_in,
5391 tree *decl1, tree *decl2,
5392 enum tree_code *code1, enum tree_code *code2,
5393 int *multi_step_cvt,
5394 VEC (tree, heap) **interm_types)
5396 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5397 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
5398 struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
5400 enum machine_mode vec_mode;
5401 enum insn_code icode1, icode2;
5402 optab optab1, optab2;
5403 tree vectype = vectype_in;
5404 tree wide_vectype = vectype_out;
5405 enum tree_code c1, c2;
5407 /* The result of a vectorized widening operation usually requires two vectors
5408 (because the widened results do not fit int one vector). The generated
5409 vector results would normally be expected to be generated in the same
5410 order as in the original scalar computation, i.e. if 8 results are
5411 generated in each vector iteration, they are to be organized as follows:
5412 vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
5414 However, in the special case that the result of the widening operation is
5415 used in a reduction computation only, the order doesn't matter (because
5416 when vectorizing a reduction we change the order of the computation).
5417 Some targets can take advantage of this and generate more efficient code.
5418 For example, targets like Altivec, that support widen_mult using a sequence
5419 of {mult_even,mult_odd} generate the following vectors:
5420 vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8].
5422 When vectorizing outer-loops, we execute the inner-loop sequentially
5423 (each vectorized inner-loop iteration contributes to VF outer-loop
5424 iterations in parallel). We therefore don't allow to change the order
5425 of the computation in the inner-loop during outer-loop vectorization. */
5427 if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
5428 && !nested_in_vect_loop_p (vect_loop, stmt))
5434 && code == WIDEN_MULT_EXPR
5435 && targetm.vectorize.builtin_mul_widen_even
5436 && targetm.vectorize.builtin_mul_widen_even (vectype)
5437 && targetm.vectorize.builtin_mul_widen_odd
5438 && targetm.vectorize.builtin_mul_widen_odd (vectype))
5440 if (vect_print_dump_info (REPORT_DETAILS))
5441 fprintf (vect_dump, "Unordered widening operation detected.");
5443 *code1 = *code2 = CALL_EXPR;
5444 *decl1 = targetm.vectorize.builtin_mul_widen_even (vectype);
5445 *decl2 = targetm.vectorize.builtin_mul_widen_odd (vectype);
5451 case WIDEN_MULT_EXPR:
5452 if (BYTES_BIG_ENDIAN)
5454 c1 = VEC_WIDEN_MULT_HI_EXPR;
5455 c2 = VEC_WIDEN_MULT_LO_EXPR;
5459 c2 = VEC_WIDEN_MULT_HI_EXPR;
5460 c1 = VEC_WIDEN_MULT_LO_EXPR;
5465 if (BYTES_BIG_ENDIAN)
5467 c1 = VEC_UNPACK_HI_EXPR;
5468 c2 = VEC_UNPACK_LO_EXPR;
5472 c2 = VEC_UNPACK_HI_EXPR;
5473 c1 = VEC_UNPACK_LO_EXPR;
5478 if (BYTES_BIG_ENDIAN)
5480 c1 = VEC_UNPACK_FLOAT_HI_EXPR;
5481 c2 = VEC_UNPACK_FLOAT_LO_EXPR;
5485 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
5486 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
5490 case FIX_TRUNC_EXPR:
5491 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
5492 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
5493 computing the operation. */
5500 if (code == FIX_TRUNC_EXPR)
5502 /* The signedness is determined from output operand. */
5503 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
5504 optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
5508 optab1 = optab_for_tree_code (c1, vectype, optab_default);
5509 optab2 = optab_for_tree_code (c2, vectype, optab_default);
5512 if (!optab1 || !optab2)
5515 vec_mode = TYPE_MODE (vectype);
5516 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
5517 || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
5520 /* Check if it's a multi-step conversion that can be done using intermediate
5522 if (insn_data[icode1].operand[0].mode != TYPE_MODE (wide_vectype)
5523 || insn_data[icode2].operand[0].mode != TYPE_MODE (wide_vectype))
5526 tree prev_type = vectype, intermediate_type;
5527 enum machine_mode intermediate_mode, prev_mode = vec_mode;
5528 optab optab3, optab4;
5530 if (!CONVERT_EXPR_CODE_P (code))
5536 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
5537 intermediate steps in promotion sequence. We try
5538 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
5540 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
5541 for (i = 0; i < 3; i++)
5543 intermediate_mode = insn_data[icode1].operand[0].mode;
5544 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
5545 TYPE_UNSIGNED (prev_type));
5546 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
5547 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
5549 if (!optab3 || !optab4
5550 || ((icode1 = optab_handler (optab1, prev_mode))
5551 == CODE_FOR_nothing)
5552 || insn_data[icode1].operand[0].mode != intermediate_mode
5553 || ((icode2 = optab_handler (optab2, prev_mode))
5554 == CODE_FOR_nothing)
5555 || insn_data[icode2].operand[0].mode != intermediate_mode
5556 || ((icode1 = optab_handler (optab3, intermediate_mode))
5557 == CODE_FOR_nothing)
5558 || ((icode2 = optab_handler (optab4, intermediate_mode))
5559 == CODE_FOR_nothing))
5562 VEC_quick_push (tree, *interm_types, intermediate_type);
5563 (*multi_step_cvt)++;
5565 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
5566 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
5569 prev_type = intermediate_type;
5570 prev_mode = intermediate_mode;
5582 /* Function supportable_narrowing_operation
5584 Check whether an operation represented by the code CODE is a
5585 narrowing operation that is supported by the target platform in
5586 vector form (i.e., when operating on arguments of type VECTYPE_IN
5587 and producing a result of type VECTYPE_OUT).
5589 Narrowing operations we currently support are NOP (CONVERT) and
5590 FIX_TRUNC. This function checks if these operations are supported by
5591 the target platform directly via vector tree-codes.
5594 - CODE1 is the code of a vector operation to be used when
5595 vectorizing the operation, if available.
5596 - MULTI_STEP_CVT determines the number of required intermediate steps in
5597 case of multi-step conversion (like int->short->char - in that case
5598 MULTI_STEP_CVT will be 1).
5599 - INTERM_TYPES contains the intermediate type required to perform the
5600 narrowing operation (short in the above example). */
5603 supportable_narrowing_operation (enum tree_code code,
5604 tree vectype_out, tree vectype_in,
5605 enum tree_code *code1, int *multi_step_cvt,
5606 VEC (tree, heap) **interm_types)
5608 enum machine_mode vec_mode;
5609 enum insn_code icode1;
5610 optab optab1, interm_optab;
5611 tree vectype = vectype_in;
5612 tree narrow_vectype = vectype_out;
5614 tree intermediate_type, prev_type;
5620 c1 = VEC_PACK_TRUNC_EXPR;
5623 case FIX_TRUNC_EXPR:
5624 c1 = VEC_PACK_FIX_TRUNC_EXPR;
5628 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
5629 tree code and optabs used for computing the operation. */
5636 if (code == FIX_TRUNC_EXPR)
5637 /* The signedness is determined from output operand. */
5638 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
5640 optab1 = optab_for_tree_code (c1, vectype, optab_default);
5645 vec_mode = TYPE_MODE (vectype);
5646 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
5649 /* Check if it's a multi-step conversion that can be done using intermediate
5651 if (insn_data[icode1].operand[0].mode != TYPE_MODE (narrow_vectype))
5653 enum machine_mode intermediate_mode, prev_mode = vec_mode;
5656 prev_type = vectype;
5657 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
5658 intermediate steps in promotion sequence. We try
5659 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
5661 *interm_types = VEC_alloc (tree, heap, MAX_INTERM_CVT_STEPS);
5662 for (i = 0; i < 3; i++)
5664 intermediate_mode = insn_data[icode1].operand[0].mode;
5665 intermediate_type = lang_hooks.types.type_for_mode (intermediate_mode,
5666 TYPE_UNSIGNED (prev_type));
5667 interm_optab = optab_for_tree_code (c1, intermediate_type,
5670 || ((icode1 = optab_handler (optab1, prev_mode))
5671 == CODE_FOR_nothing)
5672 || insn_data[icode1].operand[0].mode != intermediate_mode
5673 || ((icode1 = optab_handler (interm_optab, intermediate_mode))
5674 == CODE_FOR_nothing))
5677 VEC_quick_push (tree, *interm_types, intermediate_type);
5678 (*multi_step_cvt)++;
5680 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
5683 prev_type = intermediate_type;
5684 prev_mode = intermediate_mode;