stmt_vec_info stmt_info;
int i;
HOST_WIDE_INT dummy;
+ gimple stmt, pattern_stmt = NULL;
+ bool analyze_pattern_stmt = false;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
}
}
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+ for (si = gsi_start_bb (bb); !gsi_end_p (si) || analyze_pattern_stmt;)
{
- tree vf_vectype;
- gimple stmt = gsi_stmt (si), pattern_stmt;
- stmt_info = vinfo_for_stmt (stmt);
+ tree vf_vectype;
+
+ if (analyze_pattern_stmt)
+ {
+ stmt = pattern_stmt;
+ analyze_pattern_stmt = false;
+ }
+ else
+ stmt = gsi_stmt (si);
+
+ stmt_info = vinfo_for_stmt (stmt);
if (vect_print_dump_info (REPORT_DETAILS))
{
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "skip.");
+ gsi_next (&si);
continue;
}
}
+ else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
+ && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
+ || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
+ analyze_pattern_stmt = true;
if (gimple_get_lhs (stmt) == NULL_TREE)
{
if (!vectorization_factor
|| (nunits > vectorization_factor))
vectorization_factor = nunits;
+
+ if (!analyze_pattern_stmt)
+ gsi_next (&si);
}
}
tree cond_expr = NULL_TREE;
gimple_seq cond_expr_stmt_list = NULL;
bool do_peeling_for_loop_bound;
+ gimple stmt, pattern_stmt;
+ bool transform_pattern_stmt = false;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vec_transform_loop ===");
}
}
- for (si = gsi_start_bb (bb); !gsi_end_p (si);)
+ pattern_stmt = NULL;
+ for (si = gsi_start_bb (bb); !gsi_end_p (si) || transform_pattern_stmt;)
{
- gimple stmt = gsi_stmt (si), pattern_stmt;
bool is_store;
+ if (transform_pattern_stmt)
+ {
+ stmt = pattern_stmt;
+ transform_pattern_stmt = false;
+ }
+ else
+ stmt = gsi_stmt (si);
+
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "------>vectorizing statement: ");
continue;
}
}
+ else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
+ && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
+ || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
+ transform_pattern_stmt = true;
gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
nunits = (unsigned int) TYPE_VECTOR_SUBPARTS (
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
{
- gsi_next (&si);
- continue;
+ if (!transform_pattern_stmt)
+ gsi_next (&si);
+ continue;
}
}
the chain. */
vect_remove_stores (GROUP_FIRST_ELEMENT (stmt_info));
gsi_remove (&si, true);
- continue;
+ continue;
}
else
{
continue;
}
}
- gsi_next (&si);
+
+ if (!transform_pattern_stmt)
+ gsi_next (&si);
} /* stmts in BB */
} /* BBs in loop */
static void
vect_mark_relevant (VEC(gimple,heap) **worklist, gimple stmt,
- enum vect_relevant relevant, bool live_p)
+ enum vect_relevant relevant, bool live_p,
+ bool used_in_pattern)
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
+ gimple pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "mark relevant %d, live %d.", relevant, live_p);
+ /* If this stmt is an original stmt in a pattern, we might need to mark its
+ related pattern stmt instead of the original stmt. However, such stmts
+ may have their own uses that are not in any pattern, in such cases the
+ stmt itself should be marked. */
if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
- gimple pattern_stmt;
+ bool found = false;
+ if (!used_in_pattern)
+ {
+ imm_use_iterator imm_iter;
+ use_operand_p use_p;
+ gimple use_stmt;
+ tree lhs;
- /* This is the last stmt in a sequence that was detected as a
- pattern that can potentially be vectorized. Don't mark the stmt
- as relevant/live because it's not going to be vectorized.
- Instead mark the pattern-stmt that replaces it. */
+ if (is_gimple_assign (stmt))
+ lhs = gimple_assign_lhs (stmt);
+ else
+ lhs = gimple_call_lhs (stmt);
- pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+ /* This use is out of pattern use, if LHS has other uses that are
+ pattern uses, we should mark the stmt itself, and not the pattern
+ stmt. */
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
+ {
+ if (is_gimple_debug (USE_STMT (use_p)))
+ continue;
+ use_stmt = USE_STMT (use_p);
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "last stmt in pattern. don't mark relevant/live.");
- stmt_info = vinfo_for_stmt (pattern_stmt);
- gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
- save_relevant = STMT_VINFO_RELEVANT (stmt_info);
- save_live_p = STMT_VINFO_LIVE_P (stmt_info);
- stmt = pattern_stmt;
+ if (vinfo_for_stmt (use_stmt)
+ && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt)))
+ {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ {
+ /* This is the last stmt in a sequence that was detected as a
+ pattern that can potentially be vectorized. Don't mark the stmt
+ as relevant/live because it's not going to be vectorized.
+ Instead mark the pattern-stmt that replaces it. */
+
+ pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "last stmt in pattern. don't mark"
+ " relevant/live.");
+ stmt_info = vinfo_for_stmt (pattern_stmt);
+ gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
+ save_relevant = STMT_VINFO_RELEVANT (stmt_info);
+ save_live_p = STMT_VINFO_LIVE_P (stmt_info);
+ stmt = pattern_stmt;
+ }
}
STMT_VINFO_LIVE_P (stmt_info) |= live_p;
}
}
- vect_mark_relevant (worklist, def_stmt, relevant, live_p);
+ vect_mark_relevant (worklist, def_stmt, relevant, live_p,
+ is_pattern_stmt_p (stmt_vinfo));
return true;
}
}
if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
- vect_mark_relevant (&worklist, phi, relevant, live_p);
+ vect_mark_relevant (&worklist, phi, relevant, live_p, false);
}
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
}
if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
- vect_mark_relevant (&worklist, stmt, relevant, live_p);
+ vect_mark_relevant (&worklist, stmt, relevant, live_p, false);
}
}
/* Get the def from the vectorized stmt. */
def_stmt_info = vinfo_for_stmt (def_stmt);
+
vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
+ /* Get vectorized pattern statement. */
+ if (!vec_stmt
+ && STMT_VINFO_IN_PATTERN_P (def_stmt_info)
+ && !STMT_VINFO_RELEVANT (def_stmt_info))
+ vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (
+ STMT_VINFO_RELATED_STMT (def_stmt_info)));
gcc_assert (vec_stmt);
if (gimple_code (vec_stmt) == GIMPLE_PHI)
vec_oprnd = PHI_RESULT (vec_stmt);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
tree scalar_type, vectype;
+ gimple pattern_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
{
- any LABEL_EXPRs in the loop
- computations that are used only for array indexing or loop control.
In basic blocks we only analyze statements that are a part of some SLP
- instance, therefore, all the statements are relevant. */
+ instance, therefore, all the statements are relevant.
+ Pattern statement need to be analyzed instead of the original statement
+ if the original statement is not relevant. Otherwise, we analyze both
+ statements. */
+
+ pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
{
- gimple pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && pattern_stmt
&& (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
{
+ /* Analyze PATTERN_STMT instead of the original stmt. */
stmt = pattern_stmt;
stmt_info = vinfo_for_stmt (pattern_stmt);
if (vect_print_dump_info (REPORT_DETAILS))
return true;
}
}
+ else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && pattern_stmt
+ && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
+ || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
+ {
+ /* Analyze PATTERN_STMT too. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "==> examining pattern statement: ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node))
+ return false;
+ }
switch (STMT_VINFO_DEF_TYPE (stmt_info))
{
bool is_store = false;
gimple vec_stmt = NULL;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- gimple orig_stmt_in_pattern, orig_scalar_stmt = stmt;
bool done;
switch (STMT_VINFO_TYPE (stmt_info))
}
if (vec_stmt)
- {
- STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
- orig_stmt_in_pattern = STMT_VINFO_RELATED_STMT (stmt_info);
- if (orig_stmt_in_pattern)
- {
- stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt_in_pattern);
- /* STMT was inserted by the vectorizer to replace a computation idiom.
- ORIG_STMT_IN_PATTERN is a stmt in the original sequence that
- computed this idiom. We need to record a pointer to VEC_STMT in
- the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the
- documentation of vect_pattern_recog. */
- if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
- {
- gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo)
- == orig_scalar_stmt);
- STMT_VINFO_VEC_STMT (stmt_vinfo) = vec_stmt;
- }
- }
- }
+ STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
return is_store;
}
|| *dt == vect_nested_cycle)
{
stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
- if (STMT_VINFO_IN_PATTERN_P (stmt_info))
+
+ if (STMT_VINFO_IN_PATTERN_P (stmt_info)
+ && !STMT_VINFO_RELEVANT (stmt_info)
+ && !STMT_VINFO_LIVE_P (stmt_info))
stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
+
*vectype = STMT_VINFO_VECTYPE (stmt_info);
gcc_assert (*vectype != NULL_TREE);
}
OSDN Git Service
