/* Loop Vectorization
- Copyright (C) 2003, 2004, 2005 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
#include "cfgloop.h"
#include "cfglayout.h"
#include "expr.h"
+#include "recog.h"
#include "optabs.h"
+#include "params.h"
#include "toplev.h"
#include "tree-chrec.h"
#include "tree-data-ref.h"
bsi_insert_before (&loop_cond_bsi, cond_stmt, BSI_SAME_STMT);
/* Remove old loop exit test: */
- bsi_remove (&loop_cond_bsi);
+ bsi_remove (&loop_cond_bsi, true);
loop_loc = find_loop_location (loop);
if (dump_file && (dump_flags & TDF_DETAILS))
exit_dest) == loop->header ?
true : false);
- new_bbs = xmalloc (sizeof (basic_block) * loop->num_nodes);
+ new_bbs = XNEWVEC (basic_block, loop->num_nodes);
copy_bbs (bbs, loop->num_nodes, new_bbs,
&loop->single_exit, 1, &new_loop->single_exit, NULL,
if (vl > vect_verbosity_level)
return false;
+ if (!current_function_decl || !vect_dump)
+ return false;
+
if (vect_loop_location == UNKNOWN_LOC)
fprintf (vect_dump, "\n%s:%d: note: ",
- DECL_SOURCE_FILE (current_function_decl),
- DECL_SOURCE_LINE (current_function_decl));
+ DECL_SOURCE_FILE (current_function_decl),
+ DECL_SOURCE_LINE (current_function_decl));
else
fprintf (vect_dump, "\n%s:%d: note: ",
LOC_FILE (vect_loop_location), LOC_LINE (vect_loop_location));
-
return true;
}
STMT_VINFO_TYPE (res) = undef_vec_info_type;
STMT_VINFO_STMT (res) = stmt;
STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
- STMT_VINFO_RELEVANT_P (res) = 0;
- STMT_VINFO_LIVE_P (res) = 0;
+ STMT_VINFO_RELEVANT (res) = 0;
+ STMT_VINFO_LIVE_P (res) = false;
STMT_VINFO_VECTYPE (res) = NULL;
STMT_VINFO_VEC_STMT (res) = NULL;
+ STMT_VINFO_IN_PATTERN_P (res) = false;
+ STMT_VINFO_RELATED_STMT (res) = NULL;
STMT_VINFO_DATA_REF (res) = NULL;
if (TREE_CODE (stmt) == PHI_NODE)
STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
{
- tree_ann_t ann = get_tree_ann (phi);
+ stmt_ann_t ann = get_stmt_ann (phi);
set_stmt_info (ann, new_stmt_vec_info (phi, res));
}
stmt_ann_t ann;
ann = stmt_ann (stmt);
- set_stmt_info ((tree_ann_t)ann, new_stmt_vec_info (stmt, res));
+ set_stmt_info (ann, new_stmt_vec_info (stmt, res));
}
}
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
LOOP_PEELING_FOR_ALIGNMENT (res) = 0;
LOOP_VINFO_VECT_FACTOR (res) = 0;
- VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DATAREFS (res), 20, "loop_datarefs");
- VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DDRS (res), 20, "loop_ddrs");
+ LOOP_VINFO_DATAREFS (res) = VEC_alloc (data_reference_p, heap, 10);
+ LOOP_VINFO_DDRS (res) = VEC_alloc (ddr_p, heap, 10 * 10);
LOOP_VINFO_UNALIGNED_DR (res) = NULL;
+ LOOP_VINFO_MAY_MISALIGN_STMTS (res)
+ = VEC_alloc (tree, heap, PARAM_VALUE (PARAM_VECT_MAX_VERSION_CHECKS));
return res;
}
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
{
- tree_ann_t ann = get_tree_ann (phi);
+ stmt_ann_t ann = stmt_ann (phi);
stmt_info = vinfo_for_stmt (phi);
free (stmt_info);
set_stmt_info (ann, NULL);
}
- for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
+ for (si = bsi_start (bb); !bsi_end_p (si); )
{
tree stmt = bsi_stmt (si);
stmt_ann_t ann = stmt_ann (stmt);
if (stmt_info)
{
+ /* Check if this is a "pattern stmt" (introduced by the
+ vectorizer during the pattern recognition pass). */
+ bool remove_stmt_p = false;
+ tree orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+ if (orig_stmt)
+ {
+ stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
+ if (orig_stmt_info
+ && STMT_VINFO_IN_PATTERN_P (orig_stmt_info))
+ remove_stmt_p = true;
+ }
+
+ /* Free stmt_vec_info. */
VEC_free (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmt_info));
free (stmt_info);
- set_stmt_info ((tree_ann_t)ann, NULL);
+ set_stmt_info (ann, NULL);
+
+ /* Remove dead "pattern stmts". */
+ if (remove_stmt_p)
+ bsi_remove (&si, true);
}
+ bsi_next (&si);
}
}
free (LOOP_VINFO_BBS (loop_vinfo));
- varray_clear (LOOP_VINFO_DATAREFS (loop_vinfo));
- varray_clear (LOOP_VINFO_DDRS (loop_vinfo));
+ free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
+ free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
+ VEC_free (tree, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
free (loop_vinfo);
}
}
+/* Function supportable_widening_operation
+
+ Check whether an operation represented by the code CODE is a
+ widening operation that is supported by the target platform in
+ vector form (i.e., when operating on arguments of type VECTYPE).
+
+ The two kinds of widening operations we currently support are
+ NOP and WIDEN_MULT. This function checks if these oprations
+ are supported by the target platform either directly (via vector
+ tree-codes), or via target builtins.
+
+ Output:
+ - CODE1 and CODE2 are codes of vector operations to be used when
+ vectorizing the operation, if available.
+ - DECL1 and DECL2 are decls of target builtin functions to be used
+ when vectorizing the operation, if available. In this case,
+ CODE1 and CODE2 are CALL_EXPR. */
+
+bool
+supportable_widening_operation (enum tree_code code, tree stmt, tree vectype,
+ tree *decl1, tree *decl2,
+ enum tree_code *code1, enum tree_code *code2)
+{
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ bool ordered_p;
+ enum machine_mode vec_mode;
+ enum insn_code icode1, icode2;
+ optab optab1, optab2;
+ tree expr = TREE_OPERAND (stmt, 1);
+ tree type = TREE_TYPE (expr);
+ tree wide_vectype = get_vectype_for_scalar_type (type);
+ enum tree_code c1, c2;
+
+ /* The result of a vectorized widening operation usually requires two vectors
+ (because the widened results do not fit int one vector). The generated
+ vector results would normally be expected to be generated in the same
+ order as in the original scalar computation. i.e. if 8 results are
+ generated in each vector iteration, they are to be organized as follows:
+ vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
+
+ However, in the special case that the result of the widening operation is
+ used in a reduction copmutation only, the order doesn't matter (because
+ when vectorizing a reduction we change the order of the computation).
+ Some targets can take advatage of this and generate more efficient code.
+ For example, targets like Altivec, that support widen_mult using a sequence
+ of {mult_even,mult_odd} generate the following vectors:
+ vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8]. */
+
+ if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction)
+ ordered_p = false;
+ else
+ ordered_p = true;
+
+ if (!ordered_p
+ && code == WIDEN_MULT_EXPR
+ && targetm.vectorize.builtin_mul_widen_even
+ && targetm.vectorize.builtin_mul_widen_even (vectype)
+ && targetm.vectorize.builtin_mul_widen_odd
+ && targetm.vectorize.builtin_mul_widen_odd (vectype))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Unordered widening operation detected.");
+
+ *code1 = *code2 = CALL_EXPR;
+ *decl1 = targetm.vectorize.builtin_mul_widen_even (vectype);
+ *decl2 = targetm.vectorize.builtin_mul_widen_odd (vectype);
+ return true;
+ }
+
+ switch (code)
+ {
+ case WIDEN_MULT_EXPR:
+ if (BYTES_BIG_ENDIAN)
+ {
+ c1 = VEC_WIDEN_MULT_HI_EXPR;
+ c2 = VEC_WIDEN_MULT_LO_EXPR;
+ }
+ else
+ {
+ c2 = VEC_WIDEN_MULT_HI_EXPR;
+ c1 = VEC_WIDEN_MULT_LO_EXPR;
+ }
+ break;
+
+ case NOP_EXPR:
+ if (BYTES_BIG_ENDIAN)
+ {
+ c1 = VEC_UNPACK_HI_EXPR;
+ c2 = VEC_UNPACK_LO_EXPR;
+ }
+ else
+ {
+ c2 = VEC_UNPACK_HI_EXPR;
+ c1 = VEC_UNPACK_LO_EXPR;
+ }
+ break;
+
+ default:
+ gcc_unreachable ();
+ }
+
+ *code1 = c1;
+ *code2 = c2;
+ optab1 = optab_for_tree_code (c1, vectype);
+ optab2 = optab_for_tree_code (c2, vectype);
+
+ if (!optab1 || !optab2)
+ return false;
+
+ vec_mode = TYPE_MODE (vectype);
+ if ((icode1 = optab1->handlers[(int) vec_mode].insn_code) == CODE_FOR_nothing
+ || insn_data[icode1].operand[0].mode != TYPE_MODE (wide_vectype)
+ || (icode2 = optab2->handlers[(int) vec_mode].insn_code)
+ == CODE_FOR_nothing
+ || insn_data[icode2].operand[0].mode != TYPE_MODE (wide_vectype))
+ return false;
+
+ return true;
+}
+
+
/* Function reduction_code_for_scalar_code
Input:
Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized. */
tree
-vect_is_simple_reduction (struct loop *loop ATTRIBUTE_UNUSED,
- tree phi ATTRIBUTE_UNUSED)
+vect_is_simple_reduction (struct loop *loop, tree phi)
{
edge latch_e = loop_latch_edge (loop);
tree loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
/* CHECKME: check for !flag_finite_math_only too? */
if (SCALAR_FLOAT_TYPE_P (type) && !flag_unsafe_math_optimizations)
{
- /* Changing the order of operations changes the sematics. */
+ /* Changing the order of operations changes the semantics. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "reduction: unsafe fp math optimization: ");
}
else if (INTEGRAL_TYPE_P (type) && !TYPE_UNSIGNED (type) && flag_trapv)
{
- /* Changing the order of operations changes the sematics. */
+ /* Changing the order of operations changes the semantics. */
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "reduction: unsafe int math optimization: ");
&& flow_bb_inside_loop_p (loop, bb_for_stmt (def2))
&& def1 == phi)
{
- use_operand_p use;
- ssa_op_iter iter;
-
/* Swap operands (just for simplicity - so that the rest of the code
can assume that the reduction variable is always the last (second)
argument). */
fprintf (vect_dump, "detected reduction: need to swap operands:");
print_generic_expr (vect_dump, operation, TDF_SLIM);
}
-
- /* CHECKME */
- FOR_EACH_SSA_USE_OPERAND (use, def_stmt, iter, SSA_OP_USE)
- {
- tree tuse = USE_FROM_PTR (use);
- if (tuse == op1)
- SET_USE (use, op2);
- else if (tuse == op2)
- SET_USE (use, op1);
- }
+ swap_tree_operands (def_stmt, &TREE_OPERAND (operation, 0),
+ &TREE_OPERAND (operation, 1));
return def_stmt;
}
else
if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
continue;
- vect_transform_loop (loop_vinfo, loops);
+ vect_transform_loop (loop_vinfo, loops);
num_vectorized_loops++;
}
+ vect_loop_location = UNKNOWN_LOC;
if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS))
fprintf (vect_dump, "vectorized %u loops in function.\n",
OSDN Git Service
