/* SLP - Basic Block Vectorization
- Copyright (C) 2007, 2008, 2009, 2010, 2011
+ Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012
Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (instance));
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
+ free (instance);
}
oprnd = TREE_OPERAND (oprnd, 0);
}
- if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def,
- &dt)
+ if (!vect_is_simple_use (oprnd, NULL, loop_vinfo, bb_vinfo, &def_stmt,
+ &def, &dt)
|| (!def_stmt && dt != vect_constant_def))
{
if (vect_print_dump_info (REPORT_SLP))
vect_model_store_cost (stmt_info, ncopies_for_cost, false,
dt, slp_node);
else
- /* Not memory operation (we don't call this function for
- loads). */
- vect_model_simple_cost (stmt_info, ncopies_for_cost, &dt,
- slp_node);
+ {
+ enum vect_def_type dts[2];
+ dts[0] = dt;
+ dts[1] = vect_uninitialized_def;
+ /* Not memory operation (we don't call this function for
+ loads). */
+ vect_model_simple_cost (stmt_info, ncopies_for_cost, dts,
+ slp_node);
+ }
}
}
else
/* We checked that this case ok, so there is no need to proceed with
permutation tests. */
- if (complex_numbers == 2)
+ if (complex_numbers == 2
+ && VEC_length (slp_tree, SLP_INSTANCE_LOADS (slp_instn)) == 2)
{
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (slp_instn));
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (slp_instn));
if (vect_print_dump_info (REPORT_SLP))
fprintf (vect_dump, "Build SLP failed: unrolling required in basic"
" block SLP");
+ vect_free_slp_tree (node);
+ VEC_free (int, heap, load_permutation);
+ VEC_free (slp_tree, heap, loads);
return false;
}
vect_detect_hybrid_slp_stmts (slp_tree node)
{
int i;
- gimple stmt;
+ VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (node);
+ gimple stmt = VEC_index (gimple, stmts, 0);
imm_use_iterator imm_iter;
gimple use_stmt;
- stmt_vec_info stmt_vinfo;
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
slp_void_p child;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ struct loop *loop = NULL;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
+ basic_block bb = NULL;
if (!node)
return;
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ else
+ bb = BB_VINFO_BB (bb_vinfo);
+
FOR_EACH_VEC_ELT (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt)
if (PURE_SLP_STMT (vinfo_for_stmt (stmt))
&& TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0))
- if ((stmt_vinfo = vinfo_for_stmt (use_stmt))
+ if (gimple_bb (use_stmt)
+ && ((loop && flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ || bb == gimple_bb (use_stmt))
+ && (stmt_vinfo = vinfo_for_stmt (use_stmt))
&& !STMT_SLP_TYPE (stmt_vinfo)
&& (STMT_VINFO_RELEVANT (stmt_vinfo)
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo)))
- && !(gimple_code (use_stmt) == GIMPLE_PHI
- && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (use_stmt))
- == vect_reduction_def))
+ && !(gimple_code (use_stmt) == GIMPLE_PHI
+ && STMT_VINFO_DEF_TYPE (stmt_vinfo)
+ == vect_reduction_def))
vect_mark_slp_stmts (node, hybrid, i);
FOR_EACH_VEC_ELT (slp_void_p, SLP_TREE_CHILDREN (node), i, child)
static void
destroy_bb_vec_info (bb_vec_info bb_vinfo)
{
+ VEC (slp_instance, heap) *slp_instances;
+ slp_instance instance;
basic_block bb;
gimple_stmt_iterator si;
+ unsigned i;
if (!bb_vinfo)
return;
free_data_refs (BB_VINFO_DATAREFS (bb_vinfo));
free_dependence_relations (BB_VINFO_DDRS (bb_vinfo));
VEC_free (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo));
+ slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
+ FOR_EACH_VEC_ELT (slp_instance, slp_instances, i, instance)
+ vect_free_slp_instance (instance);
VEC_free (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo));
free (bb_vinfo);
bb->aux = NULL;
&& REFERENCE_CLASS_P (gimple_get_lhs (stmt)))
{
gimple last_store = vect_find_last_store_in_slp_instance (instance);
+ if (is_pattern_stmt_p (vinfo_for_stmt (last_store)))
+ last_store = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (last_store));
si = gsi_for_stmt (last_store);
}
{
VEC (slp_instance, heap) *slp_instances;
slp_instance instance;
- unsigned int i, vf;
+ slp_tree loads_node;
+ unsigned int i, j, vf;
bool is_store = false;
if (loop_vinfo)
/* Schedule the tree of INSTANCE. */
is_store = vect_schedule_slp_instance (SLP_INSTANCE_TREE (instance),
instance, vf);
+
+ /* Clear STMT_VINFO_VEC_STMT of all loads. With shared loads
+ between SLP instances we fail to properly initialize the
+ vectorized SLP stmts and confuse different load permutations. */
+ FOR_EACH_VEC_ELT (slp_tree, SLP_INSTANCE_LOADS (instance), j, loads_node)
+ STMT_VINFO_VEC_STMT
+ (vinfo_for_stmt
+ (VEC_index (gimple, SLP_TREE_SCALAR_STMTS (loads_node), 0))) = NULL;
+
if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS)
|| vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "vectorizing stmts using SLP.");
unsigned int j;
gimple_stmt_iterator gsi;
- vect_remove_slp_scalar_calls (root);
+ /* Remove scalar call stmts. Do not do this for basic-block
+ vectorization as not all uses may be vectorized.
+ ??? Why should this be necessary? DCE should be able to
+ remove the stmts itself.
+ ??? For BB vectorization we can as well remove scalar
+ stmts starting from the SLP tree root if they have no
+ uses. */
+ if (loop_vinfo)
+ vect_remove_slp_scalar_calls (root);
for (j = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (root), j, store)
&& j < SLP_INSTANCE_GROUP_SIZE (instance); j++)
if (!STMT_VINFO_DATA_REF (vinfo_for_stmt (store)))
break;
+ if (is_pattern_stmt_p (vinfo_for_stmt (store)))
+ store = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (store));
/* Free the attached stmt_vec_info and remove the stmt. */
gsi = gsi_for_stmt (store);
gsi_remove (&gsi, true);