/* Loop Vectorization
- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com> and
Ira Rosen <irar@il.ibm.com>
stmt_vec_info stmt_info;
int i;
HOST_WIDE_INT dummy;
- gimple stmt, pattern_stmt = NULL, pattern_def_stmt = NULL;
- bool analyze_pattern_stmt = false, pattern_def = false;
+ gimple stmt, pattern_stmt = NULL;
+ gimple_seq pattern_def_seq = NULL;
+ gimple_stmt_iterator pattern_def_si = gsi_start (NULL);
+ bool analyze_pattern_stmt = false;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
tree vf_vectype;
if (analyze_pattern_stmt)
- {
- stmt = pattern_stmt;
- analyze_pattern_stmt = false;
- }
+ stmt = pattern_stmt;
else
stmt = gsi_stmt (si);
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
analyze_pattern_stmt = true;
- /* If a pattern statement has a def stmt, analyze it too. */
- if (is_pattern_stmt_p (stmt_info)
- && (pattern_def_stmt = STMT_VINFO_PATTERN_DEF_STMT (stmt_info))
- && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
- || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))))
- {
- if (pattern_def)
- pattern_def = false;
- else
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "==> examining pattern def stmt: ");
- print_gimple_stmt (vect_dump, pattern_def_stmt, 0,
- TDF_SLIM);
- }
+ /* If a pattern statement has def stmts, analyze them too. */
+ if (is_pattern_stmt_p (stmt_info))
+ {
+ if (pattern_def_seq == NULL)
+ {
+ pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
+ pattern_def_si = gsi_start (pattern_def_seq);
+ }
+ else if (!gsi_end_p (pattern_def_si))
+ gsi_next (&pattern_def_si);
+ if (pattern_def_seq != NULL)
+ {
+ gimple pattern_def_stmt = NULL;
+ stmt_vec_info pattern_def_stmt_info = NULL;
- pattern_def = true;
- stmt = pattern_def_stmt;
- stmt_info = vinfo_for_stmt (stmt);
- }
- }
+ while (!gsi_end_p (pattern_def_si))
+ {
+ pattern_def_stmt = gsi_stmt (pattern_def_si);
+ pattern_def_stmt_info
+ = vinfo_for_stmt (pattern_def_stmt);
+ if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
+ || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
+ break;
+ gsi_next (&pattern_def_si);
+ }
+
+ if (!gsi_end_p (pattern_def_si))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump,
+ "==> examining pattern def stmt: ");
+ print_gimple_stmt (vect_dump, pattern_def_stmt, 0,
+ TDF_SLIM);
+ }
+
+ stmt = pattern_def_stmt;
+ stmt_info = pattern_def_stmt_info;
+ }
+ else
+ {
+ pattern_def_si = gsi_start (NULL);
+ analyze_pattern_stmt = false;
+ }
+ }
+ else
+ analyze_pattern_stmt = false;
+ }
if (gimple_get_lhs (stmt) == NULL_TREE)
{
idiom). */
gcc_assert (STMT_VINFO_DATA_REF (stmt_info)
|| is_pattern_stmt_p (stmt_info)
- || pattern_def);
+ || !gsi_end_p (pattern_def_si));
vectype = STMT_VINFO_VECTYPE (stmt_info);
}
else
|| (nunits > vectorization_factor))
vectorization_factor = nunits;
- if (!analyze_pattern_stmt && !pattern_def)
- gsi_next (&si);
+ if (!analyze_pattern_stmt && gsi_end_p (pattern_def_si))
+ {
+ pattern_def_seq = NULL;
+ gsi_next (&si);
+ }
}
}
/* Analyze the evolution function. */
access_fn = analyze_scalar_evolution (loop, def);
if (access_fn)
- STRIP_NOPS (access_fn);
- if (access_fn && vect_print_dump_info (REPORT_DETAILS))
{
- fprintf (vect_dump, "Access function of PHI: ");
- print_generic_expr (vect_dump, access_fn, TDF_SLIM);
+ STRIP_NOPS (access_fn);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Access function of PHI: ");
+ print_generic_expr (vect_dump, access_fn, TDF_SLIM);
+ }
+ STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo)
+ = evolution_part_in_loop_num (access_fn, loop->num);
}
if (!access_fn
continue;
}
+ gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo) != NULL_TREE);
+
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Detected induction.");
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
return false;
op_def_stmt = SSA_NAME_DEF_STMT (phi_op);
- if (!op_def_stmt || !vinfo_for_stmt (op_def_stmt))
+ if (!op_def_stmt
+ || !flow_bb_inside_loop_p (loop, gimple_bb (op_def_stmt))
+ || !vinfo_for_stmt (op_def_stmt))
return false;
if (STMT_VINFO_RELEVANT (vinfo_for_stmt (op_def_stmt))
if (orig_code == MINUS_EXPR)
{
tree rhs = gimple_assign_rhs2 (def_stmt);
- tree negrhs = make_ssa_name (SSA_NAME_VAR (rhs), NULL);
+ tree var = TREE_CODE (rhs) == SSA_NAME
+ ? SSA_NAME_VAR (rhs)
+ : create_tmp_reg (TREE_TYPE (rhs), NULL);
+ tree negrhs = make_ssa_name (var, NULL);
gimple negate_stmt = gimple_build_assign_with_ops (NEGATE_EXPR, negrhs,
rhs, NULL);
gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
/* Calculate the cost of one scalar iteration of the loop. */
int
-vect_get_single_scalar_iteraion_cost (loop_vec_info loop_vinfo)
+vect_get_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
if (stmt_info
&& !STMT_VINFO_RELEVANT_P (stmt_info)
&& (!STMT_VINFO_LIVE_P (stmt_info)
- || STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def))
+ || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
+ && !STMT_VINFO_IN_PATTERN_P (stmt_info))
continue;
if (STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt)))
{
gimple stmt = gsi_stmt (si);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+
+ if (STMT_VINFO_IN_PATTERN_P (stmt_info))
+ {
+ stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+ stmt_info = vinfo_for_stmt (stmt);
+ }
+
/* Skip stmts that are not vectorized inside the loop. */
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& (!STMT_VINFO_LIVE_P (stmt_info)
- || STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def))
+ || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))))
continue;
+
vec_inside_cost += STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) * factor;
/* FIXME: for stmts in the inner-loop in outer-loop vectorization,
some of the "outside" costs are generated inside the outer-loop. */
vec_outside_cost += STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info);
+ if (is_pattern_stmt_p (stmt_info)
+ && STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))
+ {
+ gimple_stmt_iterator gsi;
+
+ for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple pattern_def_stmt = gsi_stmt (gsi);
+ stmt_vec_info pattern_def_stmt_info
+ = vinfo_for_stmt (pattern_def_stmt);
+ if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
+ || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
+ {
+ vec_inside_cost
+ += STMT_VINFO_INSIDE_OF_LOOP_COST
+ (pattern_def_stmt_info) * factor;
+ vec_outside_cost
+ += STMT_VINFO_OUTSIDE_OF_LOOP_COST
+ (pattern_def_stmt_info);
+ }
+ }
+ }
}
}
- scalar_single_iter_cost = vect_get_single_scalar_iteraion_cost (loop_vinfo);
+ scalar_single_iter_cost = vect_get_single_scalar_iteration_cost (loop_vinfo);
/* Add additional cost for the peeled instructions in prologue and epilogue
loop.
if (orig_stmt)
{
orig_stmt_info = vinfo_for_stmt (orig_stmt);
- gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt);
gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
}
The last use is the reduction variable. In case of nested cycle this
assumption is not true: we use reduc_index to record the index of the
reduction variable. */
- for (i = 0; i < op_type-1; i++)
+ for (i = 0; i < op_type - 1; i++)
{
/* The condition of COND_EXPR is checked in vectorizable_condition(). */
if (i == 0 && code == COND_EXPR)
continue;
- is_simple_use = vect_is_simple_use_1 (ops[i], loop_vinfo, NULL,
+ is_simple_use = vect_is_simple_use_1 (ops[i], stmt, loop_vinfo, NULL,
&def_stmt, &def, &dt, &tem);
if (!vectype_in)
vectype_in = tem;
}
}
- is_simple_use = vect_is_simple_use_1 (ops[i], loop_vinfo, NULL, &def_stmt,
- &def, &dt, &tem);
+ is_simple_use = vect_is_simple_use_1 (ops[i], stmt, loop_vinfo, NULL,
+ &def_stmt, &def, &dt, &tem);
if (!vectype_in)
vectype_in = tem;
gcc_assert (is_simple_use);
- gcc_assert (dt == vect_reduction_def
- || dt == vect_nested_cycle
- || ((dt == vect_internal_def || dt == vect_external_def
- || dt == vect_constant_def || dt == vect_induction_def)
- && nested_cycle && found_nested_cycle_def));
+ if (!(dt == vect_reduction_def
+ || dt == vect_nested_cycle
+ || ((dt == vect_internal_def || dt == vect_external_def
+ || dt == vect_constant_def || dt == vect_induction_def)
+ && nested_cycle && found_nested_cycle_def)))
+ {
+ /* For pattern recognized stmts, orig_stmt might be a reduction,
+ but some helper statements for the pattern might not, or
+ might be COND_EXPRs with reduction uses in the condition. */
+ gcc_assert (orig_stmt);
+ return false;
+ }
if (!found_nested_cycle_def)
reduc_def_stmt = def_stmt;
gimple dummy_stmt;
tree dummy;
- vect_is_simple_use (ops[!reduc_index], loop_vinfo, NULL,
+ vect_is_simple_use (ops[!reduc_index], stmt, loop_vinfo, NULL,
&dummy_stmt, &dummy, &dt);
loop_vec_def0 = vect_get_vec_def_for_stmt_copy (dt,
loop_vec_def0);
VEC_replace (tree, vec_oprnds0, 0, loop_vec_def0);
if (op_type == ternary_op)
{
- vect_is_simple_use (op1, loop_vinfo, NULL, &dummy_stmt,
+ vect_is_simple_use (op1, stmt, loop_vinfo, NULL, &dummy_stmt,
&dummy, &dt);
loop_vec_def1 = vect_get_vec_def_for_stmt_copy (dt,
loop_vec_def1);
tree vec_def;
gcc_assert (ncopies >= 1);
- /* FORNOW. This restriction should be relaxed. */
- if (nested_in_vect_loop_p (loop, phi) && ncopies > 1)
+ /* FORNOW. These restrictions should be relaxed. */
+ if (nested_in_vect_loop_p (loop, phi))
{
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "multiple types in nested loop.");
- return false;
+ imm_use_iterator imm_iter;
+ use_operand_p use_p;
+ gimple exit_phi;
+ edge latch_e;
+ tree loop_arg;
+
+ if (ncopies > 1)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "multiple types in nested loop.");
+ return false;
+ }
+
+ exit_phi = NULL;
+ latch_e = loop_latch_edge (loop->inner);
+ loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
+ {
+ if (!flow_bb_inside_loop_p (loop->inner,
+ gimple_bb (USE_STMT (use_p))))
+ {
+ exit_phi = USE_STMT (use_p);
+ break;
+ }
+ }
+ if (exit_phi)
+ {
+ stmt_vec_info exit_phi_vinfo = vinfo_for_stmt (exit_phi);
+ if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
+ && !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "inner-loop induction only used outside "
+ "of the outer vectorized loop.");
+ return false;
+ }
+ }
}
if (!STMT_VINFO_RELEVANT_P (stmt_info))
else
op = gimple_op (stmt, i + 1);
if (op
- && !vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt))
+ && !vect_is_simple_use (op, stmt, loop_vinfo, NULL, &def_stmt, &def,
+ &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "use not simple.");
tree cond_expr = NULL_TREE;
gimple_seq cond_expr_stmt_list = NULL;
bool do_peeling_for_loop_bound;
- gimple stmt, pattern_stmt, pattern_def_stmt;
- bool transform_pattern_stmt = false, pattern_def = false;
+ gimple stmt, pattern_stmt;
+ gimple_seq pattern_def_seq = NULL;
+ gimple_stmt_iterator pattern_def_si = gsi_start (NULL);
+ bool transform_pattern_stmt = false;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vec_transform_loop ===");
bool is_store;
if (transform_pattern_stmt)
- {
- stmt = pattern_stmt;
- transform_pattern_stmt = false;
- }
+ stmt = pattern_stmt;
else
stmt = gsi_stmt (si);
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
transform_pattern_stmt = true;
- /* If pattern statement has a def stmt, vectorize it too. */
- if (is_pattern_stmt_p (stmt_info)
- && (pattern_def_stmt = STMT_VINFO_PATTERN_DEF_STMT (stmt_info))
- && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
- || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))))
- {
- if (pattern_def)
- pattern_def = false;
- else
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "==> vectorizing pattern def"
- " stmt: ");
- print_gimple_stmt (vect_dump, pattern_def_stmt, 0,
- TDF_SLIM);
- }
+ /* If pattern statement has def stmts, vectorize them too. */
+ if (is_pattern_stmt_p (stmt_info))
+ {
+ if (pattern_def_seq == NULL)
+ {
+ pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
+ pattern_def_si = gsi_start (pattern_def_seq);
+ }
+ else if (!gsi_end_p (pattern_def_si))
+ gsi_next (&pattern_def_si);
+ if (pattern_def_seq != NULL)
+ {
+ gimple pattern_def_stmt = NULL;
+ stmt_vec_info pattern_def_stmt_info = NULL;
- pattern_def = true;
- stmt = pattern_def_stmt;
- stmt_info = vinfo_for_stmt (stmt);
- }
+ while (!gsi_end_p (pattern_def_si))
+ {
+ pattern_def_stmt = gsi_stmt (pattern_def_si);
+ pattern_def_stmt_info
+ = vinfo_for_stmt (pattern_def_stmt);
+ if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
+ || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
+ break;
+ gsi_next (&pattern_def_si);
+ }
+
+ if (!gsi_end_p (pattern_def_si))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "==> vectorizing pattern def"
+ " stmt: ");
+ print_gimple_stmt (vect_dump, pattern_def_stmt, 0,
+ TDF_SLIM);
+ }
+
+ stmt = pattern_def_stmt;
+ stmt_info = pattern_def_stmt_info;
+ }
+ else
+ {
+ pattern_def_si = gsi_start (NULL);
+ transform_pattern_stmt = false;
+ }
+ }
+ else
+ transform_pattern_stmt = false;
}
gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
{
- if (!transform_pattern_stmt && !pattern_def)
- gsi_next (&si);
- continue;
+ if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
+ {
+ pattern_def_seq = NULL;
+ gsi_next (&si);
+ }
+ continue;
}
}
}
}
- if (!transform_pattern_stmt && !pattern_def)
- gsi_next (&si);
+ if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
+ {
+ pattern_def_seq = NULL;
+ gsi_next (&si);
+ }
} /* stmts in BB */
} /* BBs in loop */
OSDN Git Service
