+
+/* Returns true if we can unroll LOOP FACTOR times. Number
+ of iterations of the loop is returned in NITER. */
+
+bool
+can_unroll_loop_p (struct loop *loop, unsigned factor,
+ struct tree_niter_desc *niter)
+{
+ edge exit;
+
+ /* Check whether unrolling is possible. We only want to unroll loops
+ for that we are able to determine number of iterations. We also
+ want to split the extra iterations of the loop from its end,
+ therefore we require that the loop has precisely one
+ exit. */
+
+ exit = single_dom_exit (loop);
+ if (!exit)
+ return false;
+
+ if (!number_of_iterations_exit (loop, exit, niter, false)
+ || niter->cmp == ERROR_MARK
+ /* Scalar evolutions analysis might have copy propagated
+ the abnormal ssa names into these expressions, hence
+ emitting the computations based on them during loop
+ unrolling might create overlapping life ranges for
+ them, and failures in out-of-ssa. */
+ || contains_abnormal_ssa_name_p (niter->may_be_zero)
+ || contains_abnormal_ssa_name_p (niter->control.base)
+ || contains_abnormal_ssa_name_p (niter->control.step)
+ || contains_abnormal_ssa_name_p (niter->bound))
+ return false;
+
+ /* And of course, we must be able to duplicate the loop. */
+ if (!can_duplicate_loop_p (loop))
+ return false;
+
+ /* The final loop should be small enough. */
+ if (tree_num_loop_insns (loop, &eni_size_weights) * factor
+ > (unsigned) PARAM_VALUE (PARAM_MAX_UNROLLED_INSNS))
+ return false;
+
+ return true;
+}
+
+/* Determines the conditions that control execution of LOOP unrolled FACTOR
+ times. DESC is number of iterations of LOOP. ENTER_COND is set to
+ condition that must be true if the main loop can be entered.
+ EXIT_BASE, EXIT_STEP, EXIT_CMP and EXIT_BOUND are set to values describing
+ how the exit from the unrolled loop should be controlled. */
+
+static void
+determine_exit_conditions (struct loop *loop, struct tree_niter_desc *desc,
+ unsigned factor, tree *enter_cond,
+ tree *exit_base, tree *exit_step,
+ enum tree_code *exit_cmp, tree *exit_bound)
+{
+ gimple_seq stmts;
+ tree base = desc->control.base;
+ tree step = desc->control.step;
+ tree bound = desc->bound;
+ tree type = TREE_TYPE (step);
+ tree bigstep, delta;
+ tree min = lower_bound_in_type (type, type);
+ tree max = upper_bound_in_type (type, type);
+ enum tree_code cmp = desc->cmp;
+ tree cond = boolean_true_node, assum;
+
+ /* For pointers, do the arithmetics in the type of step (sizetype). */
+ base = fold_convert (type, base);
+ bound = fold_convert (type, bound);
+
+ *enter_cond = boolean_false_node;
+ *exit_base = NULL_TREE;
+ *exit_step = NULL_TREE;
+ *exit_cmp = ERROR_MARK;
+ *exit_bound = NULL_TREE;
+ gcc_assert (cmp != ERROR_MARK);
+
+ /* We only need to be correct when we answer question
+ "Do at least FACTOR more iterations remain?" in the unrolled loop.
+ Thus, transforming BASE + STEP * i <> BOUND to
+ BASE + STEP * i < BOUND is ok. */
+ if (cmp == NE_EXPR)
+ {
+ if (tree_int_cst_sign_bit (step))
+ cmp = GT_EXPR;
+ else
+ cmp = LT_EXPR;
+ }
+ else if (cmp == LT_EXPR)
+ {
+ gcc_assert (!tree_int_cst_sign_bit (step));
+ }
+ else if (cmp == GT_EXPR)
+ {
+ gcc_assert (tree_int_cst_sign_bit (step));
+ }
+ else
+ gcc_unreachable ();
+
+ /* The main body of the loop may be entered iff:
+
+ 1) desc->may_be_zero is false.
+ 2) it is possible to check that there are at least FACTOR iterations
+ of the loop, i.e., BOUND - step * FACTOR does not overflow.
+ 3) # of iterations is at least FACTOR */
+
+ if (!integer_zerop (desc->may_be_zero))
+ cond = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
+ invert_truthvalue (desc->may_be_zero),
+ cond);
+
+ bigstep = fold_build2 (MULT_EXPR, type, step,
+ build_int_cst_type (type, factor));
+ delta = fold_build2 (MINUS_EXPR, type, bigstep, step);
+ if (cmp == LT_EXPR)
+ assum = fold_build2 (GE_EXPR, boolean_type_node,
+ bound,
+ fold_build2 (PLUS_EXPR, type, min, delta));
+ else
+ assum = fold_build2 (LE_EXPR, boolean_type_node,
+ bound,
+ fold_build2 (PLUS_EXPR, type, max, delta));
+ cond = fold_build2 (TRUTH_AND_EXPR, boolean_type_node, assum, cond);
+
+ bound = fold_build2 (MINUS_EXPR, type, bound, delta);
+ assum = fold_build2 (cmp, boolean_type_node, base, bound);
+ cond = fold_build2 (TRUTH_AND_EXPR, boolean_type_node, assum, cond);
+
+ cond = force_gimple_operand (unshare_expr (cond), &stmts, false, NULL_TREE);
+ if (stmts)
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ /* cond now may be a gimple comparison, which would be OK, but also any
+ other gimple rhs (say a && b). In this case we need to force it to
+ operand. */
+ if (!is_gimple_condexpr (cond))
+ {
+ cond = force_gimple_operand (cond, &stmts, true, NULL_TREE);
+ if (stmts)
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ }
+ *enter_cond = cond;
+
+ base = force_gimple_operand (unshare_expr (base), &stmts, true, NULL_TREE);
+ if (stmts)
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ bound = force_gimple_operand (unshare_expr (bound), &stmts, true, NULL_TREE);
+ if (stmts)
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+
+ *exit_base = base;
+ *exit_step = bigstep;
+ *exit_cmp = cmp;
+ *exit_bound = bound;
+}
+
+/* Scales the frequencies of all basic blocks in LOOP that are strictly
+ dominated by BB by NUM/DEN. */
+
+static void
+scale_dominated_blocks_in_loop (struct loop *loop, basic_block bb,
+ int num, int den)
+{
+ basic_block son;
+
+ if (den == 0)
+ return;
+
+ for (son = first_dom_son (CDI_DOMINATORS, bb);
+ son;
+ son = next_dom_son (CDI_DOMINATORS, son))
+ {
+ if (!flow_bb_inside_loop_p (loop, son))
+ continue;
+ scale_bbs_frequencies_int (&son, 1, num, den);
+ scale_dominated_blocks_in_loop (loop, son, num, den);
+ }
+}
+
+/* Unroll LOOP FACTOR times. DESC describes number of iterations of LOOP.
+ EXIT is the exit of the loop to that DESC corresponds.
+
+ If N is number of iterations of the loop and MAY_BE_ZERO is the condition
+ under that loop exits in the first iteration even if N != 0,
+
+ while (1)
+ {
+ x = phi (init, next);
+
+ pre;
+ if (st)
+ break;
+ post;
+ }
+
+ becomes (with possibly the exit conditions formulated a bit differently,
+ avoiding the need to create a new iv):
+
+ if (MAY_BE_ZERO || N < FACTOR)
+ goto rest;
+
+ do
+ {
+ x = phi (init, next);
+
+ pre;
+ post;
+ pre;
+ post;
+ ...
+ pre;
+ post;
+ N -= FACTOR;
+
+ } while (N >= FACTOR);
+
+ rest:
+ init' = phi (init, x);
+
+ while (1)
+ {
+ x = phi (init', next);
+
+ pre;
+ if (st)
+ break;
+ post;
+ }
+
+ Before the loop is unrolled, TRANSFORM is called for it (only for the
+ unrolled loop, but not for its versioned copy). DATA is passed to
+ TRANSFORM. */
+
+/* Probability in % that the unrolled loop is entered. Just a guess. */
+#define PROB_UNROLLED_LOOP_ENTERED 90
+
+void
+tree_transform_and_unroll_loop (struct loop *loop, unsigned factor,
+ edge exit, struct tree_niter_desc *desc,
+ transform_callback transform,
+ void *data)
+{
+ gimple exit_if;
+ tree ctr_before, ctr_after;
+ tree enter_main_cond, exit_base, exit_step, exit_bound;
+ enum tree_code exit_cmp;
+ gimple phi_old_loop, phi_new_loop, phi_rest;
+ gimple_stmt_iterator psi_old_loop, psi_new_loop;
+ tree init, next, new_init, var;
+ struct loop *new_loop;
+ basic_block rest, exit_bb;
+ edge old_entry, new_entry, old_latch, precond_edge, new_exit;
+ edge new_nonexit, e;
+ gimple_stmt_iterator bsi;
+ use_operand_p op;
+ bool ok;
+ unsigned est_niter, prob_entry, scale_unrolled, scale_rest, freq_e, freq_h;
+ unsigned new_est_niter, i, prob;
+ unsigned irr = loop_preheader_edge (loop)->flags & EDGE_IRREDUCIBLE_LOOP;
+ sbitmap wont_exit;
+ VEC (edge, heap) *to_remove = NULL;
+
+ est_niter = expected_loop_iterations (loop);
+ determine_exit_conditions (loop, desc, factor,
+ &enter_main_cond, &exit_base, &exit_step,
+ &exit_cmp, &exit_bound);
+
+ /* Let us assume that the unrolled loop is quite likely to be entered. */
+ if (integer_nonzerop (enter_main_cond))
+ prob_entry = REG_BR_PROB_BASE;
+ else
+ prob_entry = PROB_UNROLLED_LOOP_ENTERED * REG_BR_PROB_BASE / 100;
+
+ /* The values for scales should keep profile consistent, and somewhat close
+ to correct.
+
+ TODO: The current value of SCALE_REST makes it appear that the loop that
+ is created by splitting the remaining iterations of the unrolled loop is
+ executed the same number of times as the original loop, and with the same
+ frequencies, which is obviously wrong. This does not appear to cause
+ problems, so we do not bother with fixing it for now. To make the profile
+ correct, we would need to change the probability of the exit edge of the
+ loop, and recompute the distribution of frequencies in its body because
+ of this change (scale the frequencies of blocks before and after the exit
+ by appropriate factors). */
+ scale_unrolled = prob_entry;
+ scale_rest = REG_BR_PROB_BASE;
+
+ new_loop = loop_version (loop, enter_main_cond, NULL,
+ prob_entry, scale_unrolled, scale_rest, true);
+ gcc_assert (new_loop != NULL);
+ update_ssa (TODO_update_ssa);
+
+ /* Determine the probability of the exit edge of the unrolled loop. */
+ new_est_niter = est_niter / factor;
+
+ /* Without profile feedback, loops for that we do not know a better estimate
+ are assumed to roll 10 times. When we unroll such loop, it appears to
+ roll too little, and it may even seem to be cold. To avoid this, we
+ ensure that the created loop appears to roll at least 5 times (but at
+ most as many times as before unrolling). */
+ if (new_est_niter < 5)
+ {
+ if (est_niter < 5)
+ new_est_niter = est_niter;
+ else
+ new_est_niter = 5;
+ }
+
+ /* Prepare the cfg and update the phi nodes. Move the loop exit to the
+ loop latch (and make its condition dummy, for the moment). */
+ rest = loop_preheader_edge (new_loop)->src;
+ precond_edge = single_pred_edge (rest);
+ split_edge (loop_latch_edge (loop));
+ exit_bb = single_pred (loop->latch);
+
+ /* Since the exit edge will be removed, the frequency of all the blocks
+ in the loop that are dominated by it must be scaled by
+ 1 / (1 - exit->probability). */
+ scale_dominated_blocks_in_loop (loop, exit->src,
+ REG_BR_PROB_BASE,
+ REG_BR_PROB_BASE - exit->probability);
+
+ bsi = gsi_last_bb (exit_bb);
+ exit_if = gimple_build_cond (EQ_EXPR, integer_zero_node,
+ integer_zero_node,
+ NULL_TREE, NULL_TREE);
+
+ gsi_insert_after (&bsi, exit_if, GSI_NEW_STMT);
+ new_exit = make_edge (exit_bb, rest, EDGE_FALSE_VALUE | irr);
+ rescan_loop_exit (new_exit, true, false);
+
+ /* Set the probability of new exit to the same of the old one. Fix
+ the frequency of the latch block, by scaling it back by
+ 1 - exit->probability. */
+ new_exit->count = exit->count;
+ new_exit->probability = exit->probability;
+ new_nonexit = single_pred_edge (loop->latch);
+ new_nonexit->probability = REG_BR_PROB_BASE - exit->probability;
+ new_nonexit->flags = EDGE_TRUE_VALUE;
+ new_nonexit->count -= exit->count;
+ if (new_nonexit->count < 0)
+ new_nonexit->count = 0;
+ scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability,
+ REG_BR_PROB_BASE);
+
+ old_entry = loop_preheader_edge (loop);
+ new_entry = loop_preheader_edge (new_loop);
+ old_latch = loop_latch_edge (loop);
+ for (psi_old_loop = gsi_start_phis (loop->header),
+ psi_new_loop = gsi_start_phis (new_loop->header);
+ !gsi_end_p (psi_old_loop);
+ gsi_next (&psi_old_loop), gsi_next (&psi_new_loop))
+ {
+ phi_old_loop = gsi_stmt (psi_old_loop);
+ phi_new_loop = gsi_stmt (psi_new_loop);
+
+ init = PHI_ARG_DEF_FROM_EDGE (phi_old_loop, old_entry);
+ op = PHI_ARG_DEF_PTR_FROM_EDGE (phi_new_loop, new_entry);
+ gcc_assert (operand_equal_for_phi_arg_p (init, USE_FROM_PTR (op)));
+ next = PHI_ARG_DEF_FROM_EDGE (phi_old_loop, old_latch);
+
+ /* Prefer using original variable as a base for the new ssa name.
+ This is necessary for virtual ops, and useful in order to avoid
+ losing debug info for real ops. */
+ if (TREE_CODE (next) == SSA_NAME
+ && useless_type_conversion_p (TREE_TYPE (next),
+ TREE_TYPE (init)))
+ var = SSA_NAME_VAR (next);
+ else if (TREE_CODE (init) == SSA_NAME
+ && useless_type_conversion_p (TREE_TYPE (init),
+ TREE_TYPE (next)))
+ var = SSA_NAME_VAR (init);
+ else if (useless_type_conversion_p (TREE_TYPE (next), TREE_TYPE (init)))
+ {
+ var = create_tmp_var (TREE_TYPE (next), "unrinittmp");
+ add_referenced_var (var);
+ }
+ else
+ {
+ var = create_tmp_var (TREE_TYPE (init), "unrinittmp");
+ add_referenced_var (var);
+ }
+
+ new_init = make_ssa_name (var, NULL);
+ phi_rest = create_phi_node (new_init, rest);
+ SSA_NAME_DEF_STMT (new_init) = phi_rest;
+
+ add_phi_arg (phi_rest, init, precond_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi_rest, next, new_exit, UNKNOWN_LOCATION);
+ SET_USE (op, new_init);
+ }
+
+ remove_path (exit);
+
+ /* Transform the loop. */
+ if (transform)
+ (*transform) (loop, data);
+
+ /* Unroll the loop and remove the exits in all iterations except for the
+ last one. */
+ wont_exit = sbitmap_alloc (factor);
+ sbitmap_ones (wont_exit);
+ RESET_BIT (wont_exit, factor - 1);
+
+ ok = gimple_duplicate_loop_to_header_edge
+ (loop, loop_latch_edge (loop), factor - 1,
+ wont_exit, new_exit, &to_remove, DLTHE_FLAG_UPDATE_FREQ);
+ free (wont_exit);
+ gcc_assert (ok);
+
+ for (i = 0; VEC_iterate (edge, to_remove, i, e); i++)
+ {
+ ok = remove_path (e);
+ gcc_assert (ok);
+ }
+ VEC_free (edge, heap, to_remove);
+ update_ssa (TODO_update_ssa);
+
+ /* Ensure that the frequencies in the loop match the new estimated
+ number of iterations, and change the probability of the new
+ exit edge. */
+ freq_h = loop->header->frequency;
+ freq_e = EDGE_FREQUENCY (loop_preheader_edge (loop));
+ if (freq_h != 0)
+ scale_loop_frequencies (loop, freq_e * (new_est_niter + 1), freq_h);
+
+ exit_bb = single_pred (loop->latch);
+ new_exit = find_edge (exit_bb, rest);
+ new_exit->count = loop_preheader_edge (loop)->count;
+ new_exit->probability = REG_BR_PROB_BASE / (new_est_niter + 1);
+
+ rest->count += new_exit->count;
+ rest->frequency += EDGE_FREQUENCY (new_exit);
+
+ new_nonexit = single_pred_edge (loop->latch);
+ prob = new_nonexit->probability;
+ new_nonexit->probability = REG_BR_PROB_BASE - new_exit->probability;
+ new_nonexit->count = exit_bb->count - new_exit->count;
+ if (new_nonexit->count < 0)
+ new_nonexit->count = 0;
+ if (prob > 0)
+ scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability,
+ prob);
+
+ /* Finally create the new counter for number of iterations and add the new
+ exit instruction. */
+ bsi = gsi_last_bb (exit_bb);
+ exit_if = gsi_stmt (bsi);
+ create_iv (exit_base, exit_step, NULL_TREE, loop,
+ &bsi, false, &ctr_before, &ctr_after);
+ gimple_cond_set_code (exit_if, exit_cmp);
+ gimple_cond_set_lhs (exit_if, ctr_after);
+ gimple_cond_set_rhs (exit_if, exit_bound);
+ update_stmt (exit_if);
+
+#ifdef ENABLE_CHECKING
+ verify_flow_info ();
+ verify_dominators (CDI_DOMINATORS);
+ verify_loop_structure ();
+ verify_loop_closed_ssa ();
+#endif
+}
+
+/* Wrapper over tree_transform_and_unroll_loop for case we do not
+ want to transform the loop before unrolling. The meaning
+ of the arguments is the same as for tree_transform_and_unroll_loop. */
+
+void
+tree_unroll_loop (struct loop *loop, unsigned factor,
+ edge exit, struct tree_niter_desc *desc)
+{
+ tree_transform_and_unroll_loop (loop, factor, exit, desc,
+ NULL, NULL);
+}
+
+/* Rewrite the phi node at position PSI in function of the main
+ induction variable MAIN_IV and insert the generated code at GSI. */
+
+static void
+rewrite_phi_with_iv (loop_p loop,
+ gimple_stmt_iterator *psi,
+ gimple_stmt_iterator *gsi,
+ tree main_iv)
+{
+ affine_iv iv;
+ gimple stmt, phi = gsi_stmt (*psi);
+ tree atype, mtype, val, res = PHI_RESULT (phi);
+
+ if (!is_gimple_reg (res) || res == main_iv)
+ {
+ gsi_next (psi);
+ return;
+ }
+
+ if (!simple_iv (loop, loop, res, &iv, true))
+ {
+ gsi_next (psi);
+ return;
+ }
+
+ remove_phi_node (psi, false);
+
+ atype = TREE_TYPE (res);
+ mtype = POINTER_TYPE_P (atype) ? sizetype : atype;
+ val = fold_build2 (MULT_EXPR, mtype, unshare_expr (iv.step),
+ fold_convert (mtype, main_iv));
+ val = fold_build2 (POINTER_TYPE_P (atype)
+ ? POINTER_PLUS_EXPR : PLUS_EXPR,
+ atype, unshare_expr (iv.base), val);
+ val = force_gimple_operand_gsi (gsi, val, false, NULL_TREE, true,
+ GSI_SAME_STMT);
+ stmt = gimple_build_assign (res, val);
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+ SSA_NAME_DEF_STMT (res) = stmt;
+}
+
+/* Rewrite all the phi nodes of LOOP in function of the main induction
+ variable MAIN_IV. */
+
+static void
+rewrite_all_phi_nodes_with_iv (loop_p loop, tree main_iv)
+{
+ unsigned i;
+ basic_block *bbs = get_loop_body_in_dom_order (loop);
+ gimple_stmt_iterator psi;
+
+ for (i = 0; i < loop->num_nodes; i++)
+ {
+ basic_block bb = bbs[i];
+ gimple_stmt_iterator gsi = gsi_after_labels (bb);
+
+ if (bb->loop_father != loop)
+ continue;
+
+ for (psi = gsi_start_phis (bb); !gsi_end_p (psi); )
+ rewrite_phi_with_iv (loop, &psi, &gsi, main_iv);
+ }
+
+ free (bbs);
+}
+
+/* Bases all the induction variables in LOOP on a single induction
+ variable (unsigned with base 0 and step 1), whose final value is
+ compared with *NIT. When the IV type precision has to be larger
+ than *NIT type precision, *NIT is converted to the larger type, the
+ conversion code is inserted before the loop, and *NIT is updated to
+ the new definition. The induction variable is incremented in the
+ loop latch. Return the induction variable that was created. */
+
+tree
+canonicalize_loop_ivs (struct loop *loop, tree *nit)
+{
+ unsigned precision = TYPE_PRECISION (TREE_TYPE (*nit));
+ unsigned original_precision = precision;
+ tree type, var_before;
+ gimple_stmt_iterator gsi, psi;
+ gimple stmt;
+ edge exit = single_dom_exit (loop);
+ gimple_seq stmts;
+
+ for (psi = gsi_start_phis (loop->header);
+ !gsi_end_p (psi); gsi_next (&psi))
+ {
+ gimple phi = gsi_stmt (psi);
+ tree res = PHI_RESULT (phi);
+
+ if (is_gimple_reg (res) && TYPE_PRECISION (TREE_TYPE (res)) > precision)
+ precision = TYPE_PRECISION (TREE_TYPE (res));
+ }
+
+ type = lang_hooks.types.type_for_size (precision, 1);
+
+ if (original_precision != precision)
+ {
+ *nit = fold_convert (type, *nit);
+ *nit = force_gimple_operand (*nit, &stmts, true, NULL_TREE);
+ if (stmts)
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ }
+
+ gsi = gsi_last_bb (loop->latch);
+ create_iv (build_int_cst_type (type, 0), build_int_cst (type, 1), NULL_TREE,
+ loop, &gsi, true, &var_before, NULL);
+
+ rewrite_all_phi_nodes_with_iv (loop, var_before);
+
+ stmt = last_stmt (exit->src);
+ /* Make the loop exit if the control condition is not satisfied. */
+ if (exit->flags & EDGE_TRUE_VALUE)
+ {
+ edge te, fe;
+
+ extract_true_false_edges_from_block (exit->src, &te, &fe);
+ te->flags = EDGE_FALSE_VALUE;
+ fe->flags = EDGE_TRUE_VALUE;
+ }
+ gimple_cond_set_code (stmt, LT_EXPR);
+ gimple_cond_set_lhs (stmt, var_before);
+ gimple_cond_set_rhs (stmt, *nit);
+ update_stmt (stmt);
+
+ return var_before;
+}
OSDN Git Service
