/* Vectorizer Specific Loop Manipulations
- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software
- Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "ggc.h"
#include "tree.h"
#include "basic-block.h"
-#include "diagnostic.h"
+#include "tree-pretty-print.h"
+#include "gimple-pretty-print.h"
#include "tree-flow.h"
#include "tree-dump.h"
#include "cfgloop.h"
#include "cfglayout.h"
-#include "expr.h"
-#include "toplev.h"
+#include "diagnostic-core.h"
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
#include "langhooks.h"
free (bbs);
}
+typedef struct
+{
+ tree from, to;
+ basic_block bb;
+} adjust_info;
+
+DEF_VEC_O(adjust_info);
+DEF_VEC_ALLOC_O_STACK(adjust_info);
+#define VEC_adjust_info_stack_alloc(alloc) VEC_stack_alloc (adjust_info, alloc)
+
+/* A stack of values to be adjusted in debug stmts. We have to
+ process them LIFO, so that the closest substitution applies. If we
+ processed them FIFO, without the stack, we might substitute uses
+ with a PHI DEF that would soon become non-dominant, and when we got
+ to the suitable one, it wouldn't have anything to substitute any
+ more. */
+static VEC(adjust_info, stack) *adjust_vec;
+
+/* Adjust any debug stmts that referenced AI->from values to use the
+ loop-closed AI->to, if the references are dominated by AI->bb and
+ not by the definition of AI->from. */
+
+static void
+adjust_debug_stmts_now (adjust_info *ai)
+{
+ basic_block bbphi = ai->bb;
+ tree orig_def = ai->from;
+ tree new_def = ai->to;
+ imm_use_iterator imm_iter;
+ gimple stmt;
+ basic_block bbdef = gimple_bb (SSA_NAME_DEF_STMT (orig_def));
+
+ gcc_assert (dom_info_available_p (CDI_DOMINATORS));
+
+ /* Adjust any debug stmts that held onto non-loop-closed
+ references. */
+ FOR_EACH_IMM_USE_STMT (stmt, imm_iter, orig_def)
+ {
+ use_operand_p use_p;
+ basic_block bbuse;
+
+ if (!is_gimple_debug (stmt))
+ continue;
+
+ gcc_assert (gimple_debug_bind_p (stmt));
+
+ bbuse = gimple_bb (stmt);
+
+ if ((bbuse == bbphi
+ || dominated_by_p (CDI_DOMINATORS, bbuse, bbphi))
+ && !(bbuse == bbdef
+ || dominated_by_p (CDI_DOMINATORS, bbuse, bbdef)))
+ {
+ if (new_def)
+ FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
+ SET_USE (use_p, new_def);
+ else
+ {
+ gimple_debug_bind_reset_value (stmt);
+ update_stmt (stmt);
+ }
+ }
+ }
+}
+
+/* Adjust debug stmts as scheduled before. */
+
+static void
+adjust_vec_debug_stmts (void)
+{
+ if (!MAY_HAVE_DEBUG_STMTS)
+ return;
+
+ gcc_assert (adjust_vec);
+
+ while (!VEC_empty (adjust_info, adjust_vec))
+ {
+ adjust_debug_stmts_now (VEC_last (adjust_info, adjust_vec));
+ VEC_pop (adjust_info, adjust_vec);
+ }
+
+ VEC_free (adjust_info, stack, adjust_vec);
+}
+
+/* Adjust any debug stmts that referenced FROM values to use the
+ loop-closed TO, if the references are dominated by BB and not by
+ the definition of FROM. If adjust_vec is non-NULL, adjustments
+ will be postponed until adjust_vec_debug_stmts is called. */
+
+static void
+adjust_debug_stmts (tree from, tree to, basic_block bb)
+{
+ adjust_info ai;
+
+ if (MAY_HAVE_DEBUG_STMTS && TREE_CODE (from) == SSA_NAME
+ && SSA_NAME_VAR (from) != gimple_vop (cfun))
+ {
+ ai.from = from;
+ ai.to = to;
+ ai.bb = bb;
+
+ if (adjust_vec)
+ VEC_safe_push (adjust_info, stack, adjust_vec, &ai);
+ else
+ adjust_debug_stmts_now (&ai);
+ }
+}
+
+/* Change E's phi arg in UPDATE_PHI to NEW_DEF, and record information
+ to adjust any debug stmts that referenced the old phi arg,
+ presumably non-loop-closed references left over from other
+ transformations. */
+
+static void
+adjust_phi_and_debug_stmts (gimple update_phi, edge e, tree new_def)
+{
+ tree orig_def = PHI_ARG_DEF_FROM_EDGE (update_phi, e);
+
+ SET_PHI_ARG_DEF (update_phi, e->dest_idx, new_def);
+
+ if (MAY_HAVE_DEBUG_STMTS)
+ adjust_debug_stmts (orig_def, PHI_RESULT (update_phi),
+ gimple_bb (update_phi));
+}
+
/* Update the PHI nodes of NEW_LOOP.
/* An ordinary ssa name defined in the loop. */
add_phi_arg (phi_new, new_ssa_name, loop_latch_edge (new_loop), locus);
+ /* Drop any debug references outside the loop, if they would
+ become ill-formed SSA. */
+ adjust_debug_stmts (def, NULL, single_exit (orig_loop)->dest);
+
/* step 3 (case 1). */
if (!after)
{
gcc_assert (new_loop_exit_e == orig_entry_e);
- SET_PHI_ARG_DEF (phi_orig,
- new_loop_exit_e->dest_idx,
- new_ssa_name);
+ adjust_phi_and_debug_stmts (phi_orig, new_loop_exit_e, new_ssa_name);
}
}
}
/* 1.3. Update phi in successor block. */
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi, e) == loop_arg
|| PHI_ARG_DEF_FROM_EDGE (update_phi, e) == guard_arg);
- SET_PHI_ARG_DEF (update_phi, e->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (update_phi, e, PHI_RESULT (new_phi));
update_phi2 = new_phi;
/* 2.3. Update phi in successor of NEW_EXIT_BB: */
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, new_exit_e) == loop_arg);
- SET_PHI_ARG_DEF (update_phi2, new_exit_e->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (update_phi2, new_exit_e,
+ PHI_RESULT (new_phi));
/* 2.4. Record the newly created name with set_current_def.
We want to find a name such that
/* 1.3. Update phi in successor block. */
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi, e) == orig_def);
- SET_PHI_ARG_DEF (update_phi, e->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (update_phi, e, PHI_RESULT (new_phi));
update_phi2 = new_phi;
/* 2.3. Update phi in successor of NEW_EXIT_BB: */
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, new_exit_e) == loop_arg);
- SET_PHI_ARG_DEF (update_phi2, new_exit_e->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (update_phi2, new_exit_e,
+ PHI_RESULT (new_phi));
/** 3. Handle loop-closed-ssa-form phis for first loop **/
/* 3.4. Update phi in successor of GUARD_BB: */
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, guard_edge)
== guard_arg);
- SET_PHI_ARG_DEF (update_phi2, guard_edge->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (update_phi2, guard_edge,
+ PHI_RESULT (new_phi));
}
}
}
+/* Remove dead assignments from loop NEW_LOOP. */
+
+static void
+remove_dead_stmts_from_loop (struct loop *new_loop)
+{
+ basic_block *bbs = get_loop_body (new_loop);
+ unsigned i;
+ for (i = 0; i < new_loop->num_nodes; ++i)
+ {
+ gimple_stmt_iterator gsi;
+ for (gsi = gsi_start_bb (bbs[i]); !gsi_end_p (gsi);)
+ {
+ gimple stmt = gsi_stmt (gsi);
+ if (is_gimple_assign (stmt)
+ && TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME
+ && has_zero_uses (gimple_assign_lhs (stmt)))
+ {
+ gsi_remove (&gsi, true);
+ release_defs (stmt);
+ }
+ else
+ gsi_next (&gsi);
+ }
+ }
+ free (bbs);
+}
+
+
/* Function slpeel_tree_peel_loop_to_edge.
Peel the first (last) iterations of LOOP into a new prolog (epilog) loop
return NULL;
}
+ if (MAY_HAVE_DEBUG_STMTS)
+ {
+ gcc_assert (!adjust_vec);
+ adjust_vec = VEC_alloc (adjust_info, stack, 32);
+ }
+
if (e == exit_e)
{
/* NEW_LOOP was placed after LOOP. */
BITMAP_FREE (definitions);
delete_update_ssa ();
+ /* Remove all pattern statements from the loop copy. They will confuse
+ the expander if DCE is disabled.
+ ??? The pattern recognizer should be split into an analysis and
+ a transformation phase that is then run only on the loop that is
+ going to be transformed. */
+ remove_dead_stmts_from_loop (new_loop);
+
+ adjust_vec_debug_stmts ();
+
return new_loop;
}
true, GSI_SAME_STMT);
/* Fix phi expressions in the successor bb. */
- SET_PHI_ARG_DEF (phi1, update_e->dest_idx, ni_name);
+ adjust_phi_and_debug_stmts (phi1, update_e, ni_name);
}
}
use TYPE_VECTOR_SUBPARTS. */
static tree
-vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo, tree loop_niters)
+vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo, tree loop_niters,
+ tree *wide_prolog_niters)
{
struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
int vectype_align = TYPE_ALIGN (vectype) / BITS_PER_UNIT;
tree niters_type = TREE_TYPE (loop_niters);
- int step = 1;
- int element_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr))));
int nelements = TYPE_VECTOR_SUBPARTS (vectype);
- if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
- step = DR_GROUP_SIZE (vinfo_for_stmt (DR_GROUP_FIRST_DR (stmt_info)));
-
pe = loop_preheader_edge (loop);
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
{
- int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
- int elem_misalign = byte_misalign / element_size;
+ int npeel = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "known alignment = %d.", byte_misalign);
+ fprintf (vect_dump, "known peeling = %d.", npeel);
- iters = build_int_cst (niters_type,
- (((nelements - elem_misalign) & (nelements - 1)) / step));
+ iters = build_int_cst (niters_type, npeel);
}
else
{
gimple_seq new_stmts = NULL;
+ bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
+ tree offset = negative
+ ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : NULL_TREE;
tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt,
- &new_stmts, NULL_TREE, loop);
+ &new_stmts, offset, loop);
tree ptr_type = TREE_TYPE (start_addr);
tree size = TYPE_SIZE (ptr_type);
tree type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
/* Create: byte_misalign = addr & (vectype_size - 1) */
byte_misalign =
- fold_build2 (BIT_AND_EXPR, type, fold_convert (type, start_addr), vectype_size_minus_1);
+ fold_build2 (BIT_AND_EXPR, type, fold_convert (type, start_addr),
+ vectype_size_minus_1);
/* Create: elem_misalign = byte_misalign / element_size */
elem_misalign =
fold_build2 (RSHIFT_EXPR, type, byte_misalign, elem_size_log);
/* Create: (niters_type) (nelements - elem_misalign)&(nelements - 1) */
- iters = fold_build2 (MINUS_EXPR, type, nelements_tree, elem_misalign);
+ if (negative)
+ iters = fold_build2 (MINUS_EXPR, type, elem_misalign, nelements_tree);
+ else
+ iters = fold_build2 (MINUS_EXPR, type, nelements_tree, elem_misalign);
iters = fold_build2 (BIT_AND_EXPR, type, iters, nelements_minus_1);
iters = fold_convert (niters_type, iters);
}
add_referenced_var (var);
stmts = NULL;
iters_name = force_gimple_operand (iters, &stmts, false, var);
+ if (types_compatible_p (sizetype, niters_type))
+ *wide_prolog_niters = iters_name;
+ else
+ {
+ gimple_seq seq = NULL;
+ tree wide_iters = fold_convert (sizetype, iters);
+ var = create_tmp_var (sizetype, "prolog_loop_niters");
+ add_referenced_var (var);
+ *wide_prolog_niters = force_gimple_operand (wide_iters, &seq, false,
+ var);
+ if (seq)
+ gimple_seq_add_seq (&stmts, seq);
+ }
/* Insert stmt on loop preheader edge. */
if (stmts)
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_update_inits_of_dr ===");
- for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
+ FOR_EACH_VEC_ELT (data_reference_p, datarefs, i, dr)
vect_update_init_of_dr (dr, niters);
}
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree niters_of_prolog_loop, ni_name;
tree n_iters;
+ tree wide_prolog_niters;
struct loop *new_loop;
unsigned int th = 0;
int min_profitable_iters;
initialize_original_copy_tables ();
ni_name = vect_build_loop_niters (loop_vinfo, NULL);
- niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
+ niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name,
+ &wide_prolog_niters);
/* Get profitability threshold for vectorized loop. */
TREE_TYPE (n_iters), n_iters, niters_of_prolog_loop);
/* Update the init conditions of the access functions of all data refs. */
- vect_update_inits_of_drs (loop_vinfo, niters_of_prolog_loop);
+ vect_update_inits_of_drs (loop_vinfo, wide_prolog_niters);
/* After peeling we have to reset scalar evolution analyzer. */
scev_reset ();
/* Create expression (mask & (dr_1 || ... || dr_n)) where dr_i is the address
of the first vector of the i'th data reference. */
- for (i = 0; VEC_iterate (gimple, may_misalign_stmts, i, ref_stmt); i++)
+ FOR_EACH_VEC_ELT (gimple, may_misalign_stmts, i, ref_stmt)
{
gimple_seq new_stmt_list = NULL;
tree addr_base;
tree addr_tmp, addr_tmp_name;
tree or_tmp, new_or_tmp_name;
gimple addr_stmt, or_stmt;
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (ref_stmt);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
+ bool negative = tree_int_cst_compare
+ (DR_STEP (STMT_VINFO_DATA_REF (stmt_vinfo)), size_zero_node) < 0;
+ tree offset = negative
+ ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : NULL_TREE;
/* create: addr_tmp = (int)(address_of_first_vector) */
addr_base =
vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list,
- NULL_TREE, loop);
+ offset, loop);
if (new_stmt_list != NULL)
gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list);
tree segment_length = fold_build2 (MULT_EXPR, integer_type_node,
DR_STEP (dr), vect_factor);
- if (vect_supportable_dr_alignment (dr) == dr_explicit_realign_optimized)
+ if (vect_supportable_dr_alignment (dr, false)
+ == dr_explicit_realign_optimized)
{
tree vector_size = TYPE_SIZE_UNIT
(STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr))));
if (VEC_empty (ddr_p, may_alias_ddrs))
return;
- for (i = 0; VEC_iterate (ddr_p, may_alias_ddrs, i, ddr); i++)
+ FOR_EACH_VEC_ELT (ddr_p, may_alias_ddrs, i, ddr)
{
struct data_reference *dr_a, *dr_b;
gimple dr_group_first_a, dr_group_first_b;
tree addr_base_a, addr_base_b;
tree segment_length_a, segment_length_b;
gimple stmt_a, stmt_b;
+ tree seg_a_min, seg_a_max, seg_b_min, seg_b_max;
dr_a = DDR_A (ddr);
stmt_a = DR_STMT (DDR_A (ddr));
print_generic_expr (vect_dump, DR_REF (dr_b), TDF_SLIM);
}
+ seg_a_min = addr_base_a;
+ seg_a_max = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (addr_base_a),
+ addr_base_a, segment_length_a);
+ if (tree_int_cst_compare (DR_STEP (dr_a), size_zero_node) < 0)
+ seg_a_min = seg_a_max, seg_a_max = addr_base_a;
+
+ seg_b_min = addr_base_b;
+ seg_b_max = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (addr_base_b),
+ addr_base_b, segment_length_b);
+ if (tree_int_cst_compare (DR_STEP (dr_b), size_zero_node) < 0)
+ seg_b_min = seg_b_max, seg_b_max = addr_base_b;
part_cond_expr =
fold_build2 (TRUTH_OR_EXPR, boolean_type_node,
- fold_build2 (LT_EXPR, boolean_type_node,
- fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (addr_base_a),
- addr_base_a,
- segment_length_a),
- addr_base_b),
- fold_build2 (LT_EXPR, boolean_type_node,
- fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (addr_base_b),
- addr_base_b,
- segment_length_b),
- addr_base_a));
+ fold_build2 (LT_EXPR, boolean_type_node, seg_a_max, seg_b_min),
+ fold_build2 (LT_EXPR, boolean_type_node, seg_b_max, seg_a_min));
if (*cond_expr)
*cond_expr = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
tree *cond_expr, gimple_seq *cond_expr_stmt_list)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- struct loop *nloop;
basic_block condition_bb;
gimple_stmt_iterator gsi, cond_exp_gsi;
basic_block merge_bb;
return;
initialize_original_copy_tables ();
- nloop = loop_version (loop, *cond_expr, &condition_bb,
- prob, prob, REG_BR_PROB_BASE - prob, true);
+ loop_version (loop, *cond_expr, &condition_bb,
+ prob, prob, REG_BR_PROB_BASE - prob, true);
free_original_copy_tables();
/* Loop versioning violates an assumption we try to maintain during
arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
add_phi_arg (new_phi, arg, new_exit_e,
gimple_phi_arg_location_from_edge (orig_phi, e));
- SET_PHI_ARG_DEF (orig_phi, e->dest_idx, PHI_RESULT (new_phi));
+ adjust_phi_and_debug_stmts (orig_phi, e, PHI_RESULT (new_phi));
}
/* End loop-exit-fixes after versioning. */
OSDN Git Service
