1 /* Transformation Utilities for Loop Vectorization.
2 Copyright (C) 2003,2004,2005 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
24 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "diagnostic.h"
32 #include "tree-flow.h"
33 #include "tree-dump.h"
38 #include "tree-data-ref.h"
39 #include "tree-chrec.h"
40 #include "tree-scalar-evolution.h"
41 #include "tree-vectorizer.h"
42 #include "langhooks.h"
43 #include "tree-pass.h"
46 /* Utility functions for the code transformation. */
47 static bool vect_transform_stmt (tree, block_stmt_iterator *);
48 static void vect_align_data_ref (tree);
49 static tree vect_create_destination_var (tree, tree);
50 static tree vect_create_data_ref_ptr
51 (tree, block_stmt_iterator *, tree, tree *, bool);
52 static tree vect_create_index_for_vector_ref (loop_vec_info);
53 static tree vect_create_addr_base_for_vector_ref (tree, tree *, tree);
54 static tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
55 static tree vect_get_vec_def_for_operand (tree, tree);
56 static tree vect_init_vector (tree, tree);
57 static void vect_finish_stmt_generation
58 (tree stmt, tree vec_stmt, block_stmt_iterator *bsi);
59 static bool vect_is_simple_cond (tree, loop_vec_info);
60 static void update_vuses_to_preheader (tree, struct loop*);
62 /* Utility function dealing with loop peeling (not peeling itself). */
63 static void vect_generate_tmps_on_preheader
64 (loop_vec_info, tree *, tree *, tree *);
65 static tree vect_build_loop_niters (loop_vec_info);
66 static void vect_update_ivs_after_vectorizer (loop_vec_info, tree, edge);
67 static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);
68 static void vect_update_init_of_dr (struct data_reference *, tree niters);
69 static void vect_update_inits_of_drs (loop_vec_info, tree);
70 static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);
71 static void vect_do_peeling_for_loop_bound
72 (loop_vec_info, tree *, struct loops *);
75 /* Function vect_get_new_vect_var.
77 Returns a name for a new variable. The current naming scheme appends the
78 prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to
79 the name of vectorizer generated variables, and appends that to NAME if
83 vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)
88 if (var_kind == vect_simple_var)
94 new_vect_var = create_tmp_var (type, concat (prefix, name, NULL));
96 new_vect_var = create_tmp_var (type, prefix);
102 /* Function vect_create_index_for_vector_ref.
104 Create (and return) an index variable, along with it's update chain in the
105 loop. This variable will be used to access a memory location in a vector
109 LOOP: The loop being vectorized.
110 BSI: The block_stmt_iterator where STMT is. Any new stmts created by this
111 function can be added here, or in the loop pre-header.
114 Return an index that will be used to index a vector array. It is expected
115 that a pointer to the first vector will be used as the base address for the
118 FORNOW: we are not trying to be efficient, just creating a new index each
119 time from scratch. At this time all vector references could use the same
122 TODO: create only one index to be used by all vector references. Record
123 the index in the LOOP_VINFO the first time this procedure is called and
124 return it on subsequent calls. The increment of this index must be placed
125 just before the conditional expression that ends the single block loop. */
128 vect_create_index_for_vector_ref (loop_vec_info loop_vinfo)
131 block_stmt_iterator incr_bsi;
133 tree indx_before_incr, indx_after_incr;
134 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
137 /* It is assumed that the base pointer used for vectorized access contains
138 the address of the first vector. Therefore the index used for vectorized
139 access must be initialized to zero and incremented by 1. */
141 init = integer_zero_node;
142 step = integer_one_node;
144 standard_iv_increment_position (loop, &incr_bsi, &insert_after);
145 create_iv (init, step, NULL_TREE, loop, &incr_bsi, insert_after,
146 &indx_before_incr, &indx_after_incr);
147 incr = bsi_stmt (incr_bsi);
148 set_stmt_info (stmt_ann (incr), new_stmt_vec_info (incr, loop_vinfo));
150 return indx_before_incr;
154 /* Function vect_create_addr_base_for_vector_ref.
156 Create an expression that computes the address of the first memory location
157 that will be accessed for a data reference.
160 STMT: The statement containing the data reference.
161 NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
162 OFFSET: Optional. If supplied, it is be added to the initial address.
165 1. Return an SSA_NAME whose value is the address of the memory location of
166 the first vector of the data reference.
167 2. If new_stmt_list is not NULL_TREE after return then the caller must insert
168 these statement(s) which define the returned SSA_NAME.
170 FORNOW: We are only handling array accesses with step 1. */
173 vect_create_addr_base_for_vector_ref (tree stmt,
177 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
178 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
180 unshare_expr (STMT_VINFO_VECT_DR_BASE_ADDRESS (stmt_info));
181 tree base_name = build_fold_indirect_ref (data_ref_base);
182 tree ref = DR_REF (dr);
183 tree scalar_type = TREE_TYPE (ref);
184 tree scalar_ptr_type = build_pointer_type (scalar_type);
187 tree addr_base, addr_expr;
189 tree base_offset = unshare_expr (STMT_VINFO_VECT_INIT_OFFSET (stmt_info));
191 /* Create base_offset */
192 dest = create_tmp_var (TREE_TYPE (base_offset), "base_off");
193 add_referenced_tmp_var (dest);
194 base_offset = force_gimple_operand (base_offset, &new_stmt, false, dest);
195 append_to_statement_list_force (new_stmt, new_stmt_list);
199 tree tmp = create_tmp_var (TREE_TYPE (base_offset), "offset");
200 add_referenced_tmp_var (tmp);
201 offset = fold (build2 (MULT_EXPR, TREE_TYPE (offset), offset,
202 STMT_VINFO_VECT_STEP (stmt_info)));
203 base_offset = fold (build2 (PLUS_EXPR, TREE_TYPE (base_offset),
204 base_offset, offset));
205 base_offset = force_gimple_operand (base_offset, &new_stmt, false, tmp);
206 append_to_statement_list_force (new_stmt, new_stmt_list);
209 /* base + base_offset */
210 addr_base = fold (build2 (PLUS_EXPR, TREE_TYPE (data_ref_base), data_ref_base,
213 /* addr_expr = addr_base */
214 addr_expr = vect_get_new_vect_var (scalar_ptr_type, vect_pointer_var,
215 get_name (base_name));
216 add_referenced_tmp_var (addr_expr);
217 vec_stmt = build2 (MODIFY_EXPR, void_type_node, addr_expr, addr_base);
218 new_temp = make_ssa_name (addr_expr, vec_stmt);
219 TREE_OPERAND (vec_stmt, 0) = new_temp;
220 append_to_statement_list_force (vec_stmt, new_stmt_list);
222 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
224 fprintf (vect_dump, "created ");
225 print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
231 /* Function vect_align_data_ref.
233 Handle misalignment of a memory accesses.
235 FORNOW: Can't handle misaligned accesses.
236 Make sure that the dataref is aligned. */
239 vect_align_data_ref (tree stmt)
241 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
242 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
244 /* FORNOW: can't handle misaligned accesses;
245 all accesses expected to be aligned. */
246 gcc_assert (aligned_access_p (dr));
250 /* Function vect_create_data_ref_ptr.
252 Create a memory reference expression for vector access, to be used in a
253 vector load/store stmt. The reference is based on a new pointer to vector
257 1. STMT: a stmt that references memory. Expected to be of the form
258 MODIFY_EXPR <name, data-ref> or MODIFY_EXPR <data-ref, name>.
259 2. BSI: block_stmt_iterator where new stmts can be added.
260 3. OFFSET (optional): an offset to be added to the initial address accessed
261 by the data-ref in STMT.
262 4. ONLY_INIT: indicate if vp is to be updated in the loop, or remain
263 pointing to the initial address.
266 1. Declare a new ptr to vector_type, and have it point to the base of the
267 data reference (initial addressed accessed by the data reference).
268 For example, for vector of type V8HI, the following code is generated:
271 vp = (v8hi *)initial_address;
273 if OFFSET is not supplied:
274 initial_address = &a[init];
275 if OFFSET is supplied:
276 initial_address = &a[init + OFFSET];
278 Return the initial_address in INITIAL_ADDRESS.
280 2. Create a data-reference in the loop based on the new vector pointer vp,
281 and using a new index variable 'idx' as follows:
285 where if ONLY_INIT is true:
288 update = idx + vector_type_size
290 Return the pointer vp'.
293 FORNOW: handle only aligned and consecutive accesses. */
296 vect_create_data_ref_ptr (tree stmt, block_stmt_iterator *bsi, tree offset,
297 tree *initial_address, bool only_init)
300 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
301 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
302 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
303 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
309 tree new_stmt_list = NULL_TREE;
311 edge pe = loop_preheader_edge (loop);
317 tree type, tmp, size;
319 base_name = build_fold_indirect_ref (unshare_expr (
320 STMT_VINFO_VECT_DR_BASE_ADDRESS (stmt_info)));
322 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
324 tree data_ref_base = base_name;
325 fprintf (vect_dump, "create array_ref of type: ");
326 print_generic_expr (vect_dump, vectype, TDF_SLIM);
327 if (TREE_CODE (data_ref_base) == VAR_DECL)
328 fprintf (vect_dump, " vectorizing a one dimensional array ref: ");
329 else if (TREE_CODE (data_ref_base) == ARRAY_REF)
330 fprintf (vect_dump, " vectorizing a multidimensional array ref: ");
331 else if (TREE_CODE (data_ref_base) == COMPONENT_REF)
332 fprintf (vect_dump, " vectorizing a record based array ref: ");
333 else if (TREE_CODE (data_ref_base) == SSA_NAME)
334 fprintf (vect_dump, " vectorizing a pointer ref: ");
335 print_generic_expr (vect_dump, base_name, TDF_SLIM);
338 /** (1) Create the new vector-pointer variable: **/
340 vect_ptr_type = build_pointer_type (vectype);
341 vect_ptr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
342 get_name (base_name));
343 add_referenced_tmp_var (vect_ptr);
346 /** (2) Add aliasing information to the new vector-pointer:
347 (The points-to info (SSA_NAME_PTR_INFO) may be defined later.) **/
349 tag = STMT_VINFO_MEMTAG (stmt_info);
352 /* If tag is a variable (and NOT_A_TAG) than a new type alias
353 tag must be created with tag added to its may alias list. */
354 if (var_ann (tag)->mem_tag_kind == NOT_A_TAG)
355 new_type_alias (vect_ptr, tag);
357 var_ann (vect_ptr)->type_mem_tag = tag;
359 var_ann (vect_ptr)->subvars = STMT_VINFO_SUBVARS (stmt_info);
361 /** (3) Calculate the initial address the vector-pointer, and set
362 the vector-pointer to point to it before the loop: **/
364 /* Create: (&(base[init_val+offset]) in the loop preheader. */
365 new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list,
367 pe = loop_preheader_edge (loop);
368 new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);
369 gcc_assert (!new_bb);
370 *initial_address = new_temp;
372 /* Create: p = (vectype *) initial_base */
373 vec_stmt = fold_convert (vect_ptr_type, new_temp);
374 vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
375 new_temp = make_ssa_name (vect_ptr, vec_stmt);
376 TREE_OPERAND (vec_stmt, 0) = new_temp;
377 new_bb = bsi_insert_on_edge_immediate (pe, vec_stmt);
378 gcc_assert (!new_bb);
379 vect_ptr_init = TREE_OPERAND (vec_stmt, 0);
382 /** (4) Handle the updating of the vector-pointer inside the loop: **/
384 if (only_init) /* No update in loop is required. */
386 /* Copy the points-to information if it exists. */
387 if (STMT_VINFO_PTR_INFO (stmt_info))
388 duplicate_ssa_name_ptr_info (vect_ptr_init,
389 STMT_VINFO_PTR_INFO (stmt_info));
390 return vect_ptr_init;
393 idx = vect_create_index_for_vector_ref (loop_vinfo);
395 /* Create: update = idx * vectype_size */
396 tmp = create_tmp_var (integer_type_node, "update");
397 add_referenced_tmp_var (tmp);
398 size = TYPE_SIZE (vect_ptr_type);
399 type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
400 ptr_update = create_tmp_var (type, "update");
401 add_referenced_tmp_var (ptr_update);
402 vectype_size = TYPE_SIZE_UNIT (vectype);
403 vec_stmt = build2 (MULT_EXPR, integer_type_node, idx, vectype_size);
404 vec_stmt = build2 (MODIFY_EXPR, void_type_node, tmp, vec_stmt);
405 new_temp = make_ssa_name (tmp, vec_stmt);
406 TREE_OPERAND (vec_stmt, 0) = new_temp;
407 bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
408 vec_stmt = fold_convert (type, new_temp);
409 vec_stmt = build2 (MODIFY_EXPR, void_type_node, ptr_update, vec_stmt);
410 new_temp = make_ssa_name (ptr_update, vec_stmt);
411 TREE_OPERAND (vec_stmt, 0) = new_temp;
412 bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
414 /* Create: data_ref_ptr = vect_ptr_init + update */
415 vec_stmt = build2 (PLUS_EXPR, vect_ptr_type, vect_ptr_init, new_temp);
416 vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
417 new_temp = make_ssa_name (vect_ptr, vec_stmt);
418 TREE_OPERAND (vec_stmt, 0) = new_temp;
419 bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
420 data_ref_ptr = TREE_OPERAND (vec_stmt, 0);
422 /* Copy the points-to information if it exists. */
423 if (STMT_VINFO_PTR_INFO (stmt_info))
424 duplicate_ssa_name_ptr_info (data_ref_ptr, STMT_VINFO_PTR_INFO (stmt_info));
429 /* Function vect_create_destination_var.
431 Create a new temporary of type VECTYPE. */
434 vect_create_destination_var (tree scalar_dest, tree vectype)
437 const char *new_name;
439 gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);
441 new_name = get_name (scalar_dest);
444 vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, new_name);
445 add_referenced_tmp_var (vec_dest);
451 /* Function vect_init_vector.
453 Insert a new stmt (INIT_STMT) that initializes a new vector variable with
454 the vector elements of VECTOR_VAR. Return the DEF of INIT_STMT. It will be
455 used in the vectorization of STMT. */
458 vect_init_vector (tree stmt, tree vector_var)
460 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
461 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
462 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
465 tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
471 new_var = vect_get_new_vect_var (vectype, vect_simple_var, "cst_");
472 add_referenced_tmp_var (new_var);
474 init_stmt = build2 (MODIFY_EXPR, vectype, new_var, vector_var);
475 new_temp = make_ssa_name (new_var, init_stmt);
476 TREE_OPERAND (init_stmt, 0) = new_temp;
478 pe = loop_preheader_edge (loop);
479 new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
480 gcc_assert (!new_bb);
482 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
484 fprintf (vect_dump, "created new init_stmt: ");
485 print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
488 vec_oprnd = TREE_OPERAND (init_stmt, 0);
493 /* Function vect_get_vec_def_for_operand.
495 OP is an operand in STMT. This function returns a (vector) def that will be
496 used in the vectorized stmt for STMT.
498 In the case that OP is an SSA_NAME which is defined in the loop, then
499 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
501 In case OP is an invariant or constant, a new stmt that creates a vector def
502 needs to be introduced. */
505 vect_get_vec_def_for_operand (tree op, tree stmt)
510 stmt_vec_info def_stmt_info = NULL;
511 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
512 tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
513 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
514 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
515 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
522 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
524 fprintf (vect_dump, "vect_get_vec_def_for_operand: ");
525 print_generic_expr (vect_dump, op, TDF_SLIM);
528 /** ===> Case 1: operand is a constant. **/
530 if (TREE_CODE (op) == INTEGER_CST || TREE_CODE (op) == REAL_CST)
532 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
536 /* Build a tree with vector elements. */
537 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
538 fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
540 for (i = nunits - 1; i >= 0; --i)
542 t = tree_cons (NULL_TREE, op, t);
544 vec_cst = build_vector (vectype, t);
545 return vect_init_vector (stmt, vec_cst);
548 gcc_assert (TREE_CODE (op) == SSA_NAME);
550 /** ===> Case 2: operand is an SSA_NAME - find the stmt that defines it. **/
552 def_stmt = SSA_NAME_DEF_STMT (op);
553 def_stmt_info = vinfo_for_stmt (def_stmt);
555 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
557 fprintf (vect_dump, "vect_get_vec_def_for_operand: def_stmt: ");
558 print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
562 /** ==> Case 2.1: operand is defined inside the loop. **/
566 /* Get the def from the vectorized stmt. */
568 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
569 gcc_assert (vec_stmt);
570 vec_oprnd = TREE_OPERAND (vec_stmt, 0);
575 /** ==> Case 2.2: operand is defined by the loop-header phi-node -
576 it is a reduction/induction. **/
578 bb = bb_for_stmt (def_stmt);
579 if (TREE_CODE (def_stmt) == PHI_NODE && flow_bb_inside_loop_p (loop, bb))
581 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
582 fprintf (vect_dump, "reduction/induction - unsupported.");
583 internal_error ("no support for reduction/induction"); /* FORNOW */
587 /** ==> Case 2.3: operand is defined outside the loop -
588 it is a loop invariant. */
590 switch (TREE_CODE (def_stmt))
593 def = PHI_RESULT (def_stmt);
596 def = TREE_OPERAND (def_stmt, 0);
599 def = TREE_OPERAND (def_stmt, 0);
600 gcc_assert (IS_EMPTY_STMT (def_stmt));
604 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
606 fprintf (vect_dump, "unsupported defining stmt: ");
607 print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
609 internal_error ("unsupported defining stmt");
612 /* Build a tree with vector elements.
613 Create 'vec_inv = {inv,inv,..,inv}' */
615 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
616 fprintf (vect_dump, "Create vector_inv.");
618 for (i = nunits - 1; i >= 0; --i)
620 t = tree_cons (NULL_TREE, def, t);
623 vec_inv = build_constructor (vectype, t);
624 return vect_init_vector (stmt, vec_inv);
628 /* Function vect_finish_stmt_generation.
630 Insert a new stmt. */
633 vect_finish_stmt_generation (tree stmt, tree vec_stmt, block_stmt_iterator *bsi)
635 bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
637 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
639 fprintf (vect_dump, "add new stmt: ");
640 print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
643 #ifdef ENABLE_CHECKING
644 /* Make sure bsi points to the stmt that is being vectorized. */
645 gcc_assert (stmt == bsi_stmt (*bsi));
648 #ifdef USE_MAPPED_LOCATION
649 SET_EXPR_LOCATION (vec_stmt, EXPR_LOCATION (stmt));
651 SET_EXPR_LOCUS (vec_stmt, EXPR_LOCUS (stmt));
656 /* Function vectorizable_assignment.
658 Check if STMT performs an assignment (copy) that can be vectorized.
659 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
660 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
661 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
664 vectorizable_assignment (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
670 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
671 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
672 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
675 /* Is vectorizable assignment? */
677 if (TREE_CODE (stmt) != MODIFY_EXPR)
680 scalar_dest = TREE_OPERAND (stmt, 0);
681 if (TREE_CODE (scalar_dest) != SSA_NAME)
684 op = TREE_OPERAND (stmt, 1);
685 if (!vect_is_simple_use (op, loop_vinfo, NULL))
687 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
688 fprintf (vect_dump, "use not simple.");
692 if (!vec_stmt) /* transformation not required. */
694 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
699 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
700 fprintf (vect_dump, "transform assignment.");
703 vec_dest = vect_create_destination_var (scalar_dest, vectype);
706 op = TREE_OPERAND (stmt, 1);
707 vec_oprnd = vect_get_vec_def_for_operand (op, stmt);
709 /* Arguments are ready. create the new vector stmt. */
710 *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_oprnd);
711 new_temp = make_ssa_name (vec_dest, *vec_stmt);
712 TREE_OPERAND (*vec_stmt, 0) = new_temp;
713 vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
719 /* Function vect_min_worthwhile_factor.
721 For a loop where we could vectorize the operation indicated by CODE,
722 return the minimum vectorization factor that makes it worthwhile
723 to use generic vectors. */
725 vect_min_worthwhile_factor (enum tree_code code)
745 /* Function vectorizable_operation.
747 Check if STMT performs a binary or unary operation that can be vectorized.
748 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
749 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
750 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
753 vectorizable_operation (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
758 tree op0, op1 = NULL;
759 tree vec_oprnd0, vec_oprnd1=NULL;
760 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
761 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
762 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
765 enum machine_mode vec_mode;
771 /* Is STMT a vectorizable binary/unary operation? */
772 if (TREE_CODE (stmt) != MODIFY_EXPR)
775 if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
778 operation = TREE_OPERAND (stmt, 1);
779 code = TREE_CODE (operation);
780 optab = optab_for_tree_code (code, vectype);
782 /* Support only unary or binary operations. */
783 op_type = TREE_CODE_LENGTH (code);
784 if (op_type != unary_op && op_type != binary_op)
786 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
787 fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type);
791 for (i = 0; i < op_type; i++)
793 op = TREE_OPERAND (operation, i);
794 if (!vect_is_simple_use (op, loop_vinfo, NULL))
796 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
797 fprintf (vect_dump, "use not simple.");
802 /* Supportable by target? */
805 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
806 fprintf (vect_dump, "no optab.");
809 vec_mode = TYPE_MODE (vectype);
810 if (optab->handlers[(int) vec_mode].insn_code == CODE_FOR_nothing)
812 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
813 fprintf (vect_dump, "op not supported by target.");
814 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
815 || LOOP_VINFO_VECT_FACTOR (loop_vinfo)
816 < vect_min_worthwhile_factor (code))
818 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
819 fprintf (vect_dump, "proceeding using word mode.");
822 /* Worthwhile without SIMD support? */
823 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
824 && LOOP_VINFO_VECT_FACTOR (loop_vinfo)
825 < vect_min_worthwhile_factor (code))
827 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
828 fprintf (vect_dump, "not worthwhile without SIMD support.");
832 if (!vec_stmt) /* transformation not required. */
834 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
840 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
841 fprintf (vect_dump, "transform binary/unary operation.");
844 scalar_dest = TREE_OPERAND (stmt, 0);
845 vec_dest = vect_create_destination_var (scalar_dest, vectype);
848 op0 = TREE_OPERAND (operation, 0);
849 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt);
851 if (op_type == binary_op)
853 op1 = TREE_OPERAND (operation, 1);
854 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt);
857 /* Arguments are ready. create the new vector stmt. */
859 if (op_type == binary_op)
860 *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
861 build2 (code, vectype, vec_oprnd0, vec_oprnd1));
863 *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
864 build1 (code, vectype, vec_oprnd0));
865 new_temp = make_ssa_name (vec_dest, *vec_stmt);
866 TREE_OPERAND (*vec_stmt, 0) = new_temp;
867 vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
873 /* Function vectorizable_store.
875 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
877 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
878 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
879 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
882 vectorizable_store (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
888 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
889 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
890 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
891 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
892 enum machine_mode vec_mode;
894 enum dr_alignment_support alignment_support_cheme;
898 /* Is vectorizable store? */
900 if (TREE_CODE (stmt) != MODIFY_EXPR)
903 scalar_dest = TREE_OPERAND (stmt, 0);
904 if (TREE_CODE (scalar_dest) != ARRAY_REF
905 && TREE_CODE (scalar_dest) != INDIRECT_REF)
908 op = TREE_OPERAND (stmt, 1);
909 if (!vect_is_simple_use (op, loop_vinfo, NULL))
911 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
912 fprintf (vect_dump, "use not simple.");
916 vec_mode = TYPE_MODE (vectype);
917 /* FORNOW. In some cases can vectorize even if data-type not supported
918 (e.g. - array initialization with 0). */
919 if (mov_optab->handlers[(int)vec_mode].insn_code == CODE_FOR_nothing)
922 if (!STMT_VINFO_DATA_REF (stmt_info))
926 if (!vec_stmt) /* transformation not required. */
928 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
934 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
935 fprintf (vect_dump, "transform store");
937 alignment_support_cheme = vect_supportable_dr_alignment (dr);
938 gcc_assert (alignment_support_cheme);
939 gcc_assert (alignment_support_cheme == dr_aligned); /* FORNOW */
941 /* Handle use - get the vectorized def from the defining stmt. */
942 vec_oprnd1 = vect_get_vec_def_for_operand (op, stmt);
945 /* FORNOW: make sure the data reference is aligned. */
946 vect_align_data_ref (stmt);
947 data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &dummy, false);
948 data_ref = build_fold_indirect_ref (data_ref);
950 /* Arguments are ready. create the new vector stmt. */
951 *vec_stmt = build2 (MODIFY_EXPR, vectype, data_ref, vec_oprnd1);
952 vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
954 /* Copy the V_MAY_DEFS representing the aliasing of the original array
955 element's definition to the vector's definition then update the
956 defining statement. The original is being deleted so the same
957 SSA_NAMEs can be used. */
958 copy_virtual_operands (*vec_stmt, stmt);
960 FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_VMAYDEF)
962 SSA_NAME_DEF_STMT (def) = *vec_stmt;
964 /* If this virtual def has a use outside the loop and a loop peel is performed
965 then the def may be renamed by the peel. Mark it for renaming so the
966 later use will also be renamed. */
967 mark_sym_for_renaming (SSA_NAME_VAR (def));
974 /* vectorizable_load.
976 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
978 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
979 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
980 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
983 vectorizable_load (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
986 tree vec_dest = NULL;
987 tree data_ref = NULL;
989 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
990 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
991 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
998 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
999 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1000 edge pe = loop_preheader_edge (loop);
1001 enum dr_alignment_support alignment_support_cheme;
1003 /* Is vectorizable load? */
1005 if (TREE_CODE (stmt) != MODIFY_EXPR)
1008 scalar_dest = TREE_OPERAND (stmt, 0);
1009 if (TREE_CODE (scalar_dest) != SSA_NAME)
1012 op = TREE_OPERAND (stmt, 1);
1013 if (TREE_CODE (op) != ARRAY_REF && TREE_CODE (op) != INDIRECT_REF)
1016 if (!STMT_VINFO_DATA_REF (stmt_info))
1019 mode = (int) TYPE_MODE (vectype);
1021 /* FORNOW. In some cases can vectorize even if data-type not supported
1022 (e.g. - data copies). */
1023 if (mov_optab->handlers[mode].insn_code == CODE_FOR_nothing)
1025 if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo)))
1026 fprintf (vect_dump, "Aligned load, but unsupported type.");
1030 if (!vec_stmt) /* transformation not required. */
1032 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
1038 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1039 fprintf (vect_dump, "transform load.");
1041 alignment_support_cheme = vect_supportable_dr_alignment (dr);
1042 gcc_assert (alignment_support_cheme);
1044 if (alignment_support_cheme == dr_aligned
1045 || alignment_support_cheme == dr_unaligned_supported)
1056 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1057 data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &dummy, false);
1058 if (aligned_access_p (dr))
1059 data_ref = build_fold_indirect_ref (data_ref);
1062 int mis = DR_MISALIGNMENT (dr);
1063 tree tmis = (mis == -1 ? size_zero_node : size_int (mis));
1064 tmis = size_binop (MULT_EXPR, tmis, size_int(BITS_PER_UNIT));
1065 data_ref = build2 (MISALIGNED_INDIRECT_REF, vectype, data_ref, tmis);
1067 new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
1068 new_temp = make_ssa_name (vec_dest, new_stmt);
1069 TREE_OPERAND (new_stmt, 0) = new_temp;
1070 vect_finish_stmt_generation (stmt, new_stmt, bsi);
1071 copy_virtual_operands (new_stmt, stmt);
1073 else if (alignment_support_cheme == dr_unaligned_software_pipeline)
1077 msq_init = *(floor(p1))
1078 p2 = initial_addr + VS - 1;
1079 magic = have_builtin ? builtin_result : initial_address;
1082 p2' = p2 + indx * vectype_size
1084 vec_dest = realign_load (msq, lsq, magic)
1098 /* <1> Create msq_init = *(floor(p1)) in the loop preheader */
1099 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1100 data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE,
1102 data_ref = build1 (ALIGN_INDIRECT_REF, vectype, data_ref);
1103 new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
1104 new_temp = make_ssa_name (vec_dest, new_stmt);
1105 TREE_OPERAND (new_stmt, 0) = new_temp;
1106 new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
1107 gcc_assert (!new_bb);
1108 msq_init = TREE_OPERAND (new_stmt, 0);
1109 copy_virtual_operands (new_stmt, stmt);
1110 update_vuses_to_preheader (new_stmt, loop);
1113 /* <2> Create lsq = *(floor(p2')) in the loop */
1114 offset = build_int_cst (integer_type_node,
1115 TYPE_VECTOR_SUBPARTS (vectype));
1116 offset = int_const_binop (MINUS_EXPR, offset, integer_one_node, 1);
1117 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1118 dataref_ptr = vect_create_data_ref_ptr (stmt, bsi, offset, &dummy, false);
1119 data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr);
1120 new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
1121 new_temp = make_ssa_name (vec_dest, new_stmt);
1122 TREE_OPERAND (new_stmt, 0) = new_temp;
1123 vect_finish_stmt_generation (stmt, new_stmt, bsi);
1124 lsq = TREE_OPERAND (new_stmt, 0);
1125 copy_virtual_operands (new_stmt, stmt);
1129 if (targetm.vectorize.builtin_mask_for_load)
1131 /* Create permutation mask, if required, in loop preheader. */
1133 params = build_tree_list (NULL_TREE, init_addr);
1134 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1135 builtin_decl = targetm.vectorize.builtin_mask_for_load ();
1136 new_stmt = build_function_call_expr (builtin_decl, params);
1137 new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
1138 new_temp = make_ssa_name (vec_dest, new_stmt);
1139 TREE_OPERAND (new_stmt, 0) = new_temp;
1140 new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
1141 gcc_assert (!new_bb);
1142 magic = TREE_OPERAND (new_stmt, 0);
1144 /* The result of the CALL_EXPR to this builtin is determined from
1145 the value of the parameter and no global variables are touched
1146 which makes the builtin a "const" function. Requiring the
1147 builtin to have the "const" attribute makes it unnecessary
1148 to call mark_call_clobbered_vars_to_rename. */
1149 gcc_assert (TREE_READONLY (builtin_decl));
1153 /* Use current address instead of init_addr for reduced reg pressure.
1155 magic = dataref_ptr;
1159 /* <4> Create msq = phi <msq_init, lsq> in loop */
1160 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1161 msq = make_ssa_name (vec_dest, NULL_TREE);
1162 phi_stmt = create_phi_node (msq, loop->header); /* CHECKME */
1163 SSA_NAME_DEF_STMT (msq) = phi_stmt;
1164 add_phi_arg (phi_stmt, msq_init, loop_preheader_edge (loop));
1165 add_phi_arg (phi_stmt, lsq, loop_latch_edge (loop));
1168 /* <5> Create <vec_dest = realign_load (msq, lsq, magic)> in loop */
1169 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1170 new_stmt = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, magic);
1171 new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
1172 new_temp = make_ssa_name (vec_dest, new_stmt);
1173 TREE_OPERAND (new_stmt, 0) = new_temp;
1174 vect_finish_stmt_generation (stmt, new_stmt, bsi);
1179 *vec_stmt = new_stmt;
1183 /* Function vect_is_simple_cond.
1186 LOOP - the loop that is being vectorized.
1187 COND - Condition that is checked for simple use.
1189 Returns whether a COND can be vectorized. Checks whether
1190 condition operands are supportable using vec_is_simple_use. */
1193 vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
1197 if (!COMPARISON_CLASS_P (cond))
1200 lhs = TREE_OPERAND (cond, 0);
1201 rhs = TREE_OPERAND (cond, 1);
1203 if (TREE_CODE (lhs) == SSA_NAME)
1205 tree lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
1206 if (!vect_is_simple_use (lhs, loop_vinfo, &lhs_def_stmt))
1209 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST)
1212 if (TREE_CODE (rhs) == SSA_NAME)
1214 tree rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
1215 if (!vect_is_simple_use (rhs, loop_vinfo, &rhs_def_stmt))
1218 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST)
1224 /* vectorizable_condition.
1226 Check if STMT is conditional modify expression that can be vectorized.
1227 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1228 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
1231 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1234 vectorizable_condition (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
1236 tree scalar_dest = NULL_TREE;
1237 tree vec_dest = NULL_TREE;
1238 tree op = NULL_TREE;
1239 tree cond_expr, then_clause, else_clause;
1240 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1241 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1242 tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause;
1243 tree vec_compare, vec_cond_expr;
1245 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1246 enum machine_mode vec_mode;
1248 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1251 if (TREE_CODE (stmt) != MODIFY_EXPR)
1254 op = TREE_OPERAND (stmt, 1);
1256 if (TREE_CODE (op) != COND_EXPR)
1259 cond_expr = TREE_OPERAND (op, 0);
1260 then_clause = TREE_OPERAND (op, 1);
1261 else_clause = TREE_OPERAND (op, 2);
1263 if (!vect_is_simple_cond (cond_expr, loop_vinfo))
1266 if (TREE_CODE (then_clause) == SSA_NAME)
1268 tree then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
1269 if (!vect_is_simple_use (then_clause, loop_vinfo, &then_def_stmt))
1272 else if (TREE_CODE (then_clause) != INTEGER_CST
1273 && TREE_CODE (then_clause) != REAL_CST)
1276 if (TREE_CODE (else_clause) == SSA_NAME)
1278 tree else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
1279 if (!vect_is_simple_use (else_clause, loop_vinfo, &else_def_stmt))
1282 else if (TREE_CODE (else_clause) != INTEGER_CST
1283 && TREE_CODE (else_clause) != REAL_CST)
1287 vec_mode = TYPE_MODE (vectype);
1291 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
1292 return expand_vec_cond_expr_p (op, vec_mode);
1298 scalar_dest = TREE_OPERAND (stmt, 0);
1299 vec_dest = vect_create_destination_var (scalar_dest, vectype);
1301 /* Handle cond expr. */
1303 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt);
1305 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt);
1306 vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt);
1307 vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt);
1309 /* Arguments are ready. create the new vector stmt. */
1310 vec_compare = build2 (TREE_CODE (cond_expr), vectype,
1311 vec_cond_lhs, vec_cond_rhs);
1312 vec_cond_expr = build (VEC_COND_EXPR, vectype,
1313 vec_compare, vec_then_clause, vec_else_clause);
1315 *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_cond_expr);
1316 new_temp = make_ssa_name (vec_dest, *vec_stmt);
1317 TREE_OPERAND (*vec_stmt, 0) = new_temp;
1318 vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
1323 /* Function vect_transform_stmt.
1325 Create a vectorized stmt to replace STMT, and insert it at BSI. */
1328 vect_transform_stmt (tree stmt, block_stmt_iterator *bsi)
1330 bool is_store = false;
1331 tree vec_stmt = NULL_TREE;
1332 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1335 switch (STMT_VINFO_TYPE (stmt_info))
1337 case op_vec_info_type:
1338 done = vectorizable_operation (stmt, bsi, &vec_stmt);
1342 case assignment_vec_info_type:
1343 done = vectorizable_assignment (stmt, bsi, &vec_stmt);
1347 case load_vec_info_type:
1348 done = vectorizable_load (stmt, bsi, &vec_stmt);
1352 case store_vec_info_type:
1353 done = vectorizable_store (stmt, bsi, &vec_stmt);
1358 case condition_vec_info_type:
1359 done = vectorizable_condition (stmt, bsi, &vec_stmt);
1364 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1365 fprintf (vect_dump, "stmt not supported.");
1369 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
1375 /* This function builds ni_name = number of iterations loop executes
1376 on the loop preheader. */
1379 vect_build_loop_niters (loop_vec_info loop_vinfo)
1381 tree ni_name, stmt, var;
1383 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1384 tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
1386 var = create_tmp_var (TREE_TYPE (ni), "niters");
1387 add_referenced_tmp_var (var);
1388 ni_name = force_gimple_operand (ni, &stmt, false, var);
1390 pe = loop_preheader_edge (loop);
1393 basic_block new_bb = bsi_insert_on_edge_immediate (pe, stmt);
1394 gcc_assert (!new_bb);
1401 /* This function generates the following statements:
1403 ni_name = number of iterations loop executes
1404 ratio = ni_name / vf
1405 ratio_mult_vf_name = ratio * vf
1407 and places them at the loop preheader edge. */
1410 vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
1412 tree *ratio_mult_vf_name_ptr,
1413 tree *ratio_name_ptr)
1421 tree ratio_mult_vf_name;
1422 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1423 tree ni = LOOP_VINFO_NITERS (loop_vinfo);
1424 int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1425 tree log_vf = build_int_cst (unsigned_type_node, exact_log2 (vf));
1427 pe = loop_preheader_edge (loop);
1429 /* Generate temporary variable that contains
1430 number of iterations loop executes. */
1432 ni_name = vect_build_loop_niters (loop_vinfo);
1434 /* Create: ratio = ni >> log2(vf) */
1436 var = create_tmp_var (TREE_TYPE (ni), "bnd");
1437 add_referenced_tmp_var (var);
1438 ratio_name = make_ssa_name (var, NULL_TREE);
1439 stmt = build2 (MODIFY_EXPR, void_type_node, ratio_name,
1440 build2 (RSHIFT_EXPR, TREE_TYPE (ni_name), ni_name, log_vf));
1441 SSA_NAME_DEF_STMT (ratio_name) = stmt;
1443 pe = loop_preheader_edge (loop);
1444 new_bb = bsi_insert_on_edge_immediate (pe, stmt);
1445 gcc_assert (!new_bb);
1447 /* Create: ratio_mult_vf = ratio << log2 (vf). */
1449 var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
1450 add_referenced_tmp_var (var);
1451 ratio_mult_vf_name = make_ssa_name (var, NULL_TREE);
1452 stmt = build2 (MODIFY_EXPR, void_type_node, ratio_mult_vf_name,
1453 build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name), ratio_name, log_vf));
1454 SSA_NAME_DEF_STMT (ratio_mult_vf_name) = stmt;
1456 pe = loop_preheader_edge (loop);
1457 new_bb = bsi_insert_on_edge_immediate (pe, stmt);
1458 gcc_assert (!new_bb);
1460 *ni_name_ptr = ni_name;
1461 *ratio_mult_vf_name_ptr = ratio_mult_vf_name;
1462 *ratio_name_ptr = ratio_name;
1468 /* Function update_vuses_to_preheader.
1471 STMT - a statement with potential VUSEs.
1472 LOOP - the loop whose preheader will contain STMT.
1474 It's possible to vectorize a loop even though an SSA_NAME from a VUSE
1475 appears to be defined in a V_MAY_DEF in another statement in a loop.
1476 One such case is when the VUSE is at the dereference of a __restricted__
1477 pointer in a load and the V_MAY_DEF is at the dereference of a different
1478 __restricted__ pointer in a store. Vectorization may result in
1479 copy_virtual_uses being called to copy the problematic VUSE to a new
1480 statement that is being inserted in the loop preheader. This procedure
1481 is called to change the SSA_NAME in the new statement's VUSE from the
1482 SSA_NAME updated in the loop to the related SSA_NAME available on the
1483 path entering the loop.
1485 When this function is called, we have the following situation:
1490 # name1 = phi < name0 , name2>
1495 # name2 = vdef <name1>
1500 Stmt S1 was created in the loop preheader block as part of misaligned-load
1501 handling. This function fixes the name of the vuse of S1 from 'name1' to
1505 update_vuses_to_preheader (tree stmt, struct loop *loop)
1507 basic_block header_bb = loop->header;
1508 edge preheader_e = loop_preheader_edge (loop);
1510 use_operand_p use_p;
1512 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_VUSE)
1514 tree ssa_name = USE_FROM_PTR (use_p);
1515 tree def_stmt = SSA_NAME_DEF_STMT (ssa_name);
1516 tree name_var = SSA_NAME_VAR (ssa_name);
1517 basic_block bb = bb_for_stmt (def_stmt);
1519 /* For a use before any definitions, def_stmt is a NOP_EXPR. */
1520 if (!IS_EMPTY_STMT (def_stmt)
1521 && flow_bb_inside_loop_p (loop, bb))
1523 /* If the block containing the statement defining the SSA_NAME
1524 is in the loop then it's necessary to find the definition
1525 outside the loop using the PHI nodes of the header. */
1527 bool updated = false;
1529 for (phi = phi_nodes (header_bb); phi; phi = TREE_CHAIN (phi))
1531 if (SSA_NAME_VAR (PHI_RESULT (phi)) == name_var)
1533 SET_USE (use_p, PHI_ARG_DEF (phi, preheader_e->dest_idx));
1538 gcc_assert (updated);
1544 /* Function vect_update_ivs_after_vectorizer.
1546 "Advance" the induction variables of LOOP to the value they should take
1547 after the execution of LOOP. This is currently necessary because the
1548 vectorizer does not handle induction variables that are used after the
1549 loop. Such a situation occurs when the last iterations of LOOP are
1551 1. We introduced new uses after LOOP for IVs that were not originally used
1552 after LOOP: the IVs of LOOP are now used by an epilog loop.
1553 2. LOOP is going to be vectorized; this means that it will iterate N/VF
1554 times, whereas the loop IVs should be bumped N times.
1557 - LOOP - a loop that is going to be vectorized. The last few iterations
1558 of LOOP were peeled.
1559 - NITERS - the number of iterations that LOOP executes (before it is
1560 vectorized). i.e, the number of times the ivs should be bumped.
1561 - UPDATE_E - a successor edge of LOOP->exit that is on the (only) path
1562 coming out from LOOP on which there are uses of the LOOP ivs
1563 (this is the path from LOOP->exit to epilog_loop->preheader).
1565 The new definitions of the ivs are placed in LOOP->exit.
1566 The phi args associated with the edge UPDATE_E in the bb
1567 UPDATE_E->dest are updated accordingly.
1569 Assumption 1: Like the rest of the vectorizer, this function assumes
1570 a single loop exit that has a single predecessor.
1572 Assumption 2: The phi nodes in the LOOP header and in update_bb are
1573 organized in the same order.
1575 Assumption 3: The access function of the ivs is simple enough (see
1576 vect_can_advance_ivs_p). This assumption will be relaxed in the future.
1578 Assumption 4: Exactly one of the successors of LOOP exit-bb is on a path
1579 coming out of LOOP on which the ivs of LOOP are used (this is the path
1580 that leads to the epilog loop; other paths skip the epilog loop). This
1581 path starts with the edge UPDATE_E, and its destination (denoted update_bb)
1582 needs to have its phis updated.
1586 vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, tree niters,
1589 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1590 basic_block exit_bb = loop->single_exit->dest;
1592 basic_block update_bb = update_e->dest;
1594 /* gcc_assert (vect_can_advance_ivs_p (loop_vinfo)); */
1596 /* Make sure there exists a single-predecessor exit bb: */
1597 gcc_assert (single_pred_p (exit_bb));
1599 for (phi = phi_nodes (loop->header), phi1 = phi_nodes (update_bb);
1601 phi = PHI_CHAIN (phi), phi1 = PHI_CHAIN (phi1))
1603 tree access_fn = NULL;
1604 tree evolution_part;
1607 tree var, stmt, ni, ni_name;
1608 block_stmt_iterator last_bsi;
1610 /* Skip virtual phi's. */
1611 if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi))))
1613 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1614 fprintf (vect_dump, "virtual phi. skip.");
1618 access_fn = analyze_scalar_evolution (loop, PHI_RESULT (phi));
1619 gcc_assert (access_fn);
1621 unshare_expr (evolution_part_in_loop_num (access_fn, loop->num));
1622 gcc_assert (evolution_part != NULL_TREE);
1624 /* FORNOW: We do not support IVs whose evolution function is a polynomial
1625 of degree >= 2 or exponential. */
1626 gcc_assert (!tree_is_chrec (evolution_part));
1628 step_expr = evolution_part;
1629 init_expr = unshare_expr (initial_condition_in_loop_num (access_fn,
1632 ni = build2 (PLUS_EXPR, TREE_TYPE (init_expr),
1633 build2 (MULT_EXPR, TREE_TYPE (niters),
1634 niters, step_expr), init_expr);
1636 var = create_tmp_var (TREE_TYPE (init_expr), "tmp");
1637 add_referenced_tmp_var (var);
1639 ni_name = force_gimple_operand (ni, &stmt, false, var);
1641 /* Insert stmt into exit_bb. */
1642 last_bsi = bsi_last (exit_bb);
1644 bsi_insert_before (&last_bsi, stmt, BSI_SAME_STMT);
1646 /* Fix phi expressions in the successor bb. */
1647 SET_PHI_ARG_DEF (phi1, update_e->dest_idx, ni_name);
1652 /* Function vect_do_peeling_for_loop_bound
1654 Peel the last iterations of the loop represented by LOOP_VINFO.
1655 The peeled iterations form a new epilog loop. Given that the loop now
1656 iterates NITERS times, the new epilog loop iterates
1657 NITERS % VECTORIZATION_FACTOR times.
1659 The original loop will later be made to iterate
1660 NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO). */
1663 vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
1664 struct loops *loops)
1667 tree ni_name, ratio_mult_vf_name;
1668 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1669 struct loop *new_loop;
1671 basic_block preheader;
1672 #ifdef ENABLE_CHECKING
1676 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1677 fprintf (vect_dump, "=== vect_transtorm_for_unknown_loop_bound ===");
1679 /* Generate the following variables on the preheader of original loop:
1681 ni_name = number of iteration the original loop executes
1682 ratio = ni_name / vf
1683 ratio_mult_vf_name = ratio * vf */
1684 vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
1685 &ratio_mult_vf_name, ratio);
1687 #ifdef ENABLE_CHECKING
1688 loop_num = loop->num;
1690 new_loop = slpeel_tree_peel_loop_to_edge (loop, loops, loop->single_exit,
1691 ratio_mult_vf_name, ni_name, false);
1692 #ifdef ENABLE_CHECKING
1693 gcc_assert (new_loop);
1694 gcc_assert (loop_num == loop->num);
1695 slpeel_verify_cfg_after_peeling (loop, new_loop);
1698 /* A guard that controls whether the new_loop is to be executed or skipped
1699 is placed in LOOP->exit. LOOP->exit therefore has two successors - one
1700 is the preheader of NEW_LOOP, where the IVs from LOOP are used. The other
1701 is a bb after NEW_LOOP, where these IVs are not used. Find the edge that
1702 is on the path where the LOOP IVs are used and need to be updated. */
1704 preheader = loop_preheader_edge (new_loop)->src;
1705 if (EDGE_PRED (preheader, 0)->src == loop->single_exit->dest)
1706 update_e = EDGE_PRED (preheader, 0);
1708 update_e = EDGE_PRED (preheader, 1);
1710 /* Update IVs of original loop as if they were advanced
1711 by ratio_mult_vf_name steps. */
1712 vect_update_ivs_after_vectorizer (loop_vinfo, ratio_mult_vf_name, update_e);
1714 /* After peeling we have to reset scalar evolution analyzer. */
1721 /* Function vect_gen_niters_for_prolog_loop
1723 Set the number of iterations for the loop represented by LOOP_VINFO
1724 to the minimum between LOOP_NITERS (the original iteration count of the loop)
1725 and the misalignment of DR - the data reference recorded in
1726 LOOP_VINFO_UNALIGNED_DR (LOOP_VINFO). As a result, after the execution of
1727 this loop, the data reference DR will refer to an aligned location.
1729 The following computation is generated:
1731 If the misalignment of DR is known at compile time:
1732 addr_mis = int mis = DR_MISALIGNMENT (dr);
1733 Else, compute address misalignment in bytes:
1734 addr_mis = addr & (vectype_size - 1)
1736 prolog_niters = min ( LOOP_NITERS , (VF - addr_mis/elem_size)&(VF-1) )
1738 (elem_size = element type size; an element is the scalar element
1739 whose type is the inner type of the vectype) */
1742 vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo, tree loop_niters)
1744 struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
1745 int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1746 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1748 tree iters, iters_name;
1751 tree dr_stmt = DR_STMT (dr);
1752 stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
1753 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1754 int vectype_align = TYPE_ALIGN (vectype) / BITS_PER_UNIT;
1755 tree vf_minus_1 = build_int_cst (unsigned_type_node, vf - 1);
1756 tree niters_type = TREE_TYPE (loop_niters);
1758 pe = loop_preheader_edge (loop);
1760 if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
1762 int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
1763 int element_size = vectype_align/vf;
1764 int elem_misalign = byte_misalign / element_size;
1766 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1767 fprintf (vect_dump, "known alignment = %d.", byte_misalign);
1768 iters = build_int_cst (niters_type, (vf - elem_misalign)&(vf-1));
1772 tree new_stmts = NULL_TREE;
1774 vect_create_addr_base_for_vector_ref (dr_stmt, &new_stmts, NULL_TREE);
1775 tree ptr_type = TREE_TYPE (start_addr);
1776 tree size = TYPE_SIZE (ptr_type);
1777 tree type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
1778 tree vectype_size_minus_1 = build_int_cst (type, vectype_align - 1);
1779 tree elem_size_log =
1780 build_int_cst (unsigned_type_node, exact_log2 (vectype_align/vf));
1781 tree vf_tree = build_int_cst (unsigned_type_node, vf);
1785 new_bb = bsi_insert_on_edge_immediate (pe, new_stmts);
1786 gcc_assert (!new_bb);
1788 /* Create: byte_misalign = addr & (vectype_size - 1) */
1790 build2 (BIT_AND_EXPR, type, start_addr, vectype_size_minus_1);
1792 /* Create: elem_misalign = byte_misalign / element_size */
1794 build2 (RSHIFT_EXPR, unsigned_type_node, byte_misalign, elem_size_log);
1796 /* Create: (niters_type) (VF - elem_misalign)&(VF - 1) */
1797 iters = build2 (MINUS_EXPR, unsigned_type_node, vf_tree, elem_misalign);
1798 iters = build2 (BIT_AND_EXPR, unsigned_type_node, iters, vf_minus_1);
1799 iters = fold_convert (niters_type, iters);
1802 /* Create: prolog_loop_niters = min (iters, loop_niters) */
1803 /* If the loop bound is known at compile time we already verified that it is
1804 greater than vf; since the misalignment ('iters') is at most vf, there's
1805 no need to generate the MIN_EXPR in this case. */
1806 if (TREE_CODE (loop_niters) != INTEGER_CST)
1807 iters = build2 (MIN_EXPR, niters_type, iters, loop_niters);
1809 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1811 fprintf (vect_dump, "niters for prolog loop: ");
1812 print_generic_expr (vect_dump, iters, TDF_SLIM);
1815 var = create_tmp_var (niters_type, "prolog_loop_niters");
1816 add_referenced_tmp_var (var);
1817 iters_name = force_gimple_operand (iters, &stmt, false, var);
1819 /* Insert stmt on loop preheader edge. */
1822 basic_block new_bb = bsi_insert_on_edge_immediate (pe, stmt);
1823 gcc_assert (!new_bb);
1830 /* Function vect_update_init_of_dr
1832 NITERS iterations were peeled from LOOP. DR represents a data reference
1833 in LOOP. This function updates the information recorded in DR to
1834 account for the fact that the first NITERS iterations had already been
1835 executed. Specifically, it updates the OFFSET field of stmt_info. */
1838 vect_update_init_of_dr (struct data_reference *dr, tree niters)
1840 stmt_vec_info stmt_info = vinfo_for_stmt (DR_STMT (dr));
1841 tree offset = STMT_VINFO_VECT_INIT_OFFSET (stmt_info);
1843 niters = fold (build2 (MULT_EXPR, TREE_TYPE (niters), niters,
1844 STMT_VINFO_VECT_STEP (stmt_info)));
1845 offset = fold (build2 (PLUS_EXPR, TREE_TYPE (offset), offset, niters));
1846 STMT_VINFO_VECT_INIT_OFFSET (stmt_info) = offset;
1850 /* Function vect_update_inits_of_drs
1852 NITERS iterations were peeled from the loop represented by LOOP_VINFO.
1853 This function updates the information recorded for the data references in
1854 the loop to account for the fact that the first NITERS iterations had
1855 already been executed. Specifically, it updates the initial_condition of the
1856 access_function of all the data_references in the loop. */
1859 vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters)
1862 varray_type loop_write_datarefs = LOOP_VINFO_DATAREF_WRITES (loop_vinfo);
1863 varray_type loop_read_datarefs = LOOP_VINFO_DATAREF_READS (loop_vinfo);
1865 if (vect_dump && (dump_flags & TDF_DETAILS))
1866 fprintf (vect_dump, "=== vect_update_inits_of_dr ===");
1868 for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_write_datarefs); i++)
1870 struct data_reference *dr = VARRAY_GENERIC_PTR (loop_write_datarefs, i);
1871 vect_update_init_of_dr (dr, niters);
1874 for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_read_datarefs); i++)
1876 struct data_reference *dr = VARRAY_GENERIC_PTR (loop_read_datarefs, i);
1877 vect_update_init_of_dr (dr, niters);
1882 /* Function vect_do_peeling_for_alignment
1884 Peel the first 'niters' iterations of the loop represented by LOOP_VINFO.
1885 'niters' is set to the misalignment of one of the data references in the
1886 loop, thereby forcing it to refer to an aligned location at the beginning
1887 of the execution of this loop. The data reference for which we are
1888 peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
1891 vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, struct loops *loops)
1893 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1894 tree niters_of_prolog_loop, ni_name;
1896 struct loop *new_loop;
1898 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1899 fprintf (vect_dump, "=== vect_do_peeling_for_alignment ===");
1901 ni_name = vect_build_loop_niters (loop_vinfo);
1902 niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
1904 /* Peel the prolog loop and iterate it niters_of_prolog_loop. */
1906 slpeel_tree_peel_loop_to_edge (loop, loops, loop_preheader_edge (loop),
1907 niters_of_prolog_loop, ni_name, true);
1908 #ifdef ENABLE_CHECKING
1909 gcc_assert (new_loop);
1910 slpeel_verify_cfg_after_peeling (new_loop, loop);
1913 /* Update number of times loop executes. */
1914 n_iters = LOOP_VINFO_NITERS (loop_vinfo);
1915 LOOP_VINFO_NITERS (loop_vinfo) = fold (build2 (MINUS_EXPR,
1916 TREE_TYPE (n_iters), n_iters, niters_of_prolog_loop));
1918 /* Update the init conditions of the access functions of all data refs. */
1919 vect_update_inits_of_drs (loop_vinfo, niters_of_prolog_loop);
1921 /* After peeling we have to reset scalar evolution analyzer. */
1928 /* Function vect_transform_loop.
1930 The analysis phase has determined that the loop is vectorizable.
1931 Vectorize the loop - created vectorized stmts to replace the scalar
1932 stmts in the loop, and update the loop exit condition. */
1935 vect_transform_loop (loop_vec_info loop_vinfo,
1936 struct loops *loops ATTRIBUTE_UNUSED)
1938 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1939 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
1940 int nbbs = loop->num_nodes;
1941 block_stmt_iterator si;
1944 int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1946 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1947 fprintf (vect_dump, "=== vec_transform_loop ===");
1950 /* Peel the loop if there are data refs with unknown alignment.
1951 Only one data ref with unknown store is allowed. */
1953 if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
1954 vect_do_peeling_for_alignment (loop_vinfo, loops);
1956 /* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
1957 compile time constant), or it is a constant that doesn't divide by the
1958 vectorization factor, then an epilog loop needs to be created.
1959 We therefore duplicate the loop: the original loop will be vectorized,
1960 and will compute the first (n/VF) iterations. The second copy of the loop
1961 will remain scalar and will compute the remaining (n%VF) iterations.
1962 (VF is the vectorization factor). */
1964 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1965 || (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
1966 && LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0))
1967 vect_do_peeling_for_loop_bound (loop_vinfo, &ratio, loops);
1969 ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
1970 LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
1972 /* 1) Make sure the loop header has exactly two entries
1973 2) Make sure we have a preheader basic block. */
1975 gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
1977 loop_split_edge_with (loop_preheader_edge (loop), NULL);
1980 /* FORNOW: the vectorizer supports only loops which body consist
1981 of one basic block (header + empty latch). When the vectorizer will
1982 support more involved loop forms, the order by which the BBs are
1983 traversed need to be reconsidered. */
1985 for (i = 0; i < nbbs; i++)
1987 basic_block bb = bbs[i];
1989 for (si = bsi_start (bb); !bsi_end_p (si);)
1991 tree stmt = bsi_stmt (si);
1992 stmt_vec_info stmt_info;
1995 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
1997 fprintf (vect_dump, "------>vectorizing statement: ");
1998 print_generic_expr (vect_dump, stmt, TDF_SLIM);
2000 stmt_info = vinfo_for_stmt (stmt);
2001 gcc_assert (stmt_info);
2002 if (!STMT_VINFO_RELEVANT_P (stmt_info))
2007 #ifdef ENABLE_CHECKING
2008 /* FORNOW: Verify that all stmts operate on the same number of
2009 units and no inner unrolling is necessary. */
2011 (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info))
2012 == vectorization_factor);
2014 /* -------- vectorize statement ------------ */
2015 if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
2016 fprintf (vect_dump, "transform statement.");
2018 is_store = vect_transform_stmt (stmt, &si);
2021 /* Free the attached stmt_vec_info and remove the stmt. */
2022 stmt_ann_t ann = stmt_ann (stmt);
2024 set_stmt_info (ann, NULL);
2033 slpeel_make_loop_iterate_ntimes (loop, ratio);
2035 /* The memory tags and pointers in vectorized statements need to
2036 have their SSA forms updated. FIXME, why can't this be delayed
2037 until all the loops have been transformed? */
2038 update_ssa (TODO_update_ssa);
2040 if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS, LOOP_LOC (loop_vinfo)))
2041 fprintf (vect_dump, "LOOP VECTORIZED.");