static void combine_predictions_for_insn (rtx, basic_block);
static void dump_prediction (FILE *, enum br_predictor, int, basic_block, int);
-static void estimate_loops_at_level (struct loop *loop);
-static void propagate_freq (struct loop *);
+static void estimate_loops_at_level (struct loop *, bitmap);
+static void propagate_freq (struct loop *, bitmap);
static void estimate_bb_frequencies (struct loops *);
-static int counts_to_freqs (void);
static void predict_paths_leading_to (basic_block, int *, enum br_predictor, enum prediction);
static bool last_basic_block_p (basic_block);
static void compute_function_frequency (void);
{
return (JUMP_P (insn)
&& any_condjump_p (insn)
- && BLOCK_FOR_INSN (insn)->succ->succ_next);
+ && EDGE_COUNT (BLOCK_FOR_INSN (insn)->succs) >= 2);
}
/* Predict edge E by given predictor if possible. */
dump_prediction (FILE *file, enum br_predictor predictor, int probability,
basic_block bb, int used)
{
- edge e = bb->succ;
+ edge e;
+ edge_iterator ei;
if (!file)
return;
- while (e && (e->flags & EDGE_FALLTHRU))
- e = e->succ_next;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (! (e->flags & EDGE_FALLTHRU))
+ break;
fprintf (file, " %s heuristics%s: %.1f%%",
predictor_info[predictor].name,
{
int nedges = 0;
edge e;
+ edge_iterator ei;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (!(e->flags & (EDGE_EH | EDGE_FAKE)))
nedges ++;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (!(e->flags & (EDGE_EH | EDGE_FAKE)))
e->probability = (REG_BR_PROB_BASE + nedges / 2) / nedges;
else
/* Save the prediction into CFG in case we are seeing non-degenerated
conditional jump. */
- if (bb->succ->succ_next)
+ if (EDGE_COUNT (bb->succs) > 1)
{
BRANCH_EDGE (bb)->probability = combined_probability;
FALLTHRU_EDGE (bb)->probability
= REG_BR_PROB_BASE - combined_probability;
}
}
+ else if (EDGE_COUNT (bb->succs) > 1)
+ {
+ int prob = INTVAL (XEXP (prob_note, 0));
+
+ BRANCH_EDGE (bb)->probability = prob;
+ FALLTHRU_EDGE (bb)->probability = REG_BR_PROB_BASE - prob;
+ }
+ else
+ EDGE_SUCC (bb, 0)->probability = REG_BR_PROB_BASE;
}
/* Combine predictions into single probability and store them into CFG.
struct edge_prediction *pred;
int nedges = 0;
edge e, first = NULL, second = NULL;
+ edge_iterator ei;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (!(e->flags & (EDGE_EH | EDGE_FAKE)))
{
- nedges ++;
+ nedges ++;
if (first && !second)
second = e;
if (!first)
int predictor = pred->predictor;
int probability = pred->probability;
- if (pred->edge != bb->succ)
+ if (pred->edge != EDGE_SUCC (bb, 0))
probability = REG_BR_PROB_BASE - probability;
dump_prediction (file, predictor, probability, bb,
!first_match || best_predictor == predictor);
{
int header_found = 0;
edge e;
+ edge_iterator ei;
bb = bbs[j];
/* Loop branch heuristics - predict an edge back to a
loop's head as taken. */
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (e->dest == loop->header
&& e->src == loop->latch)
{
/* Loop exit heuristics - predict an edge exiting the loop if the
conditional has no loop header successors as not taken. */
if (!header_found)
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (e->dest->index < 0
|| !flow_bb_inside_loop_p (loop, e->dest))
predict_edge
}
if (!rtlsimpleloops)
- scev_reset ();
+ scev_finalize ();
}
/* Attempt to predict probabilities of BB outgoing edges using local
{
rtx last_insn = BB_END (bb);
edge e;
+ edge_iterator ei;
if (! can_predict_insn_p (last_insn))
continue;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
{
/* Predict early returns to be probable, as we've already taken
care for error returns and other are often used for fast paths
trought function. */
if ((e->dest == EXIT_BLOCK_PTR
- || (e->dest->succ && !e->dest->succ->succ_next
- && e->dest->succ->dest == EXIT_BLOCK_PTR))
+ || (EDGE_COUNT (e->dest->succs) == 1
+ && EDGE_SUCC (e->dest, 0)->dest == EXIT_BLOCK_PTR))
&& !predicted_by_p (bb, PRED_NULL_RETURN)
&& !predicted_by_p (bb, PRED_CONST_RETURN)
&& !predicted_by_p (bb, PRED_NEGATIVE_RETURN)
tree type;
tree val;
bitmap visited;
+ edge_iterator ei;
if (!stmt || TREE_CODE (stmt) != COND_EXPR)
return;
- for (then_edge = bb->succ; then_edge; then_edge = then_edge->succ_next)
+ FOR_EACH_EDGE (then_edge, ei, bb->succs)
if (then_edge->flags & EDGE_TRUE_VALUE)
- break;
+ break;
cond = TREE_OPERAND (stmt, 0);
if (!COMPARISON_CLASS_P (cond))
return;
int phi_num_args, i;
enum br_predictor pred;
enum prediction direction;
+ edge_iterator ei;
- for (e = EXIT_BLOCK_PTR->pred; e ; e = e->pred_next)
+ FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
{
return_stmt = last_stmt (e->src);
if (TREE_CODE (return_stmt) == RETURN_EXPR)
tree_bb_level_predictions ();
+ mark_irreducible_loops (&loops_info);
predict_loops (&loops_info, false);
FOR_EACH_BB (bb)
{
edge e;
+ edge_iterator ei;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
{
/* Predict early returns to be probable, as we've already taken
care for error returns and other cases are often used for
fast paths trought function. */
if (e->dest == EXIT_BLOCK_PTR
&& TREE_CODE (last_stmt (bb)) == RETURN_EXPR
- && bb->pred && bb->pred->pred_next)
+ && EDGE_COUNT (bb->preds) > 1)
{
edge e1;
+ edge_iterator ei1;
- for (e1 = bb->pred; e1; e1 = e1->pred_next)
+ FOR_EACH_EDGE (e1, ei1, bb->preds)
if (!predicted_by_p (e1->src, PRED_NULL_RETURN)
&& !predicted_by_p (e1->src, PRED_CONST_RETURN)
&& !predicted_by_p (e1->src, PRED_NEGATIVE_RETURN)
return (bb->next_bb == EXIT_BLOCK_PTR
|| (bb->next_bb->next_bb == EXIT_BLOCK_PTR
- && bb->succ && !bb->succ->succ_next
- && bb->succ->dest->next_bb == EXIT_BLOCK_PTR));
+ && EDGE_COUNT (bb->succs) == 1
+ && EDGE_SUCC (bb, 0)->dest->next_bb == EXIT_BLOCK_PTR));
}
/* Sets branch probabilities according to PREDiction and
enum prediction taken)
{
edge e;
+ edge_iterator ei;
int y;
if (heads[bb->index] < 0)
if (y == last_basic_block)
return;
- for (e = BASIC_BLOCK (y)->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, BASIC_BLOCK (y)->succs)
if (e->dest->index >= 0
&& dominated_by_p (CDI_POST_DOMINATORS, e->dest, bb))
predict_edge_def (e, pred, taken);
/* To keep queue of basic blocks to process. */
basic_block next;
- /* True if block needs to be visited in propagate_freq. */
- unsigned int tovisit:1;
-
/* Number of predecessors we need to visit first. */
int npredecessors;
} *block_info;
Propagate the frequencies for LOOP. */
static void
-propagate_freq (struct loop *loop)
+propagate_freq (struct loop *loop, bitmap tovisit)
{
basic_block head = loop->header;
basic_block bb;
basic_block last;
+ unsigned i;
edge e;
basic_block nextbb;
+ bitmap_iterator bi;
/* For each basic block we need to visit count number of his predecessors
we need to visit first. */
- FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR, NULL, next_bb)
+ EXECUTE_IF_SET_IN_BITMAP (tovisit, 0, i, bi)
{
- if (BLOCK_INFO (bb)->tovisit)
+ edge_iterator ei;
+ int count = 0;
+
+ /* The outermost "loop" includes the exit block, which we can not
+ look up via BASIC_BLOCK. Detect this and use EXIT_BLOCK_PTR
+ directly. Do the same for the entry block. */
+ if (i == (unsigned)ENTRY_BLOCK)
+ bb = ENTRY_BLOCK_PTR;
+ else if (i == (unsigned)EXIT_BLOCK)
+ bb = EXIT_BLOCK_PTR;
+ else
+ bb = BASIC_BLOCK (i);
+
+ FOR_EACH_EDGE (e, ei, bb->preds)
{
- int count = 0;
-
- for (e = bb->pred; e; e = e->pred_next)
- if (BLOCK_INFO (e->src)->tovisit && !(e->flags & EDGE_DFS_BACK))
- count++;
- else if (BLOCK_INFO (e->src)->tovisit
- && dump_file && !EDGE_INFO (e)->back_edge)
- fprintf (dump_file,
- "Irreducible region hit, ignoring edge to %i->%i\n",
- e->src->index, bb->index);
- BLOCK_INFO (bb)->npredecessors = count;
+ bool visit = bitmap_bit_p (tovisit, e->src->index);
+
+ if (visit && !(e->flags & EDGE_DFS_BACK))
+ count++;
+ else if (visit && dump_file && !EDGE_INFO (e)->back_edge)
+ fprintf (dump_file,
+ "Irreducible region hit, ignoring edge to %i->%i\n",
+ e->src->index, bb->index);
}
+ BLOCK_INFO (bb)->npredecessors = count;
}
memcpy (&BLOCK_INFO (head)->frequency, &real_one, sizeof (real_one));
last = head;
for (bb = head; bb; bb = nextbb)
{
+ edge_iterator ei;
sreal cyclic_probability, frequency;
memcpy (&cyclic_probability, &real_zero, sizeof (real_zero));
if (bb != head)
{
#ifdef ENABLE_CHECKING
- for (e = bb->pred; e; e = e->pred_next)
- if (BLOCK_INFO (e->src)->tovisit && !(e->flags & EDGE_DFS_BACK))
+ FOR_EACH_EDGE (e, ei, bb->preds)
+ if (bitmap_bit_p (tovisit, e->src->index)
+ && !(e->flags & EDGE_DFS_BACK))
abort ();
#endif
- for (e = bb->pred; e; e = e->pred_next)
+ FOR_EACH_EDGE (e, ei, bb->preds)
if (EDGE_INFO (e)->back_edge)
{
sreal_add (&cyclic_probability, &cyclic_probability,
}
}
- BLOCK_INFO (bb)->tovisit = 0;
+ bitmap_clear_bit (tovisit, bb->index);
/* Compute back edge frequencies. */
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (e->dest == head)
{
sreal tmp;
-
+
/* EDGE_INFO (e)->back_edge_prob
- = ((e->probability * BLOCK_INFO (bb)->frequency)
- / REG_BR_PROB_BASE); */
-
+ = ((e->probability * BLOCK_INFO (bb)->frequency)
+ / REG_BR_PROB_BASE); */
+
sreal_init (&tmp, e->probability, 0);
sreal_mul (&tmp, &tmp, &BLOCK_INFO (bb)->frequency);
sreal_mul (&EDGE_INFO (e)->back_edge_prob,
}
/* Propagate to successor blocks. */
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (!(e->flags & EDGE_DFS_BACK)
&& BLOCK_INFO (e->dest)->npredecessors)
{
nextbb = e->dest;
else
BLOCK_INFO (last)->next = e->dest;
-
+
last = e->dest;
}
- }
+ }
}
}
/* Estimate probabilities of loopback edges in loops at same nest level. */
static void
-estimate_loops_at_level (struct loop *first_loop)
+estimate_loops_at_level (struct loop *first_loop, bitmap tovisit)
{
struct loop *loop;
basic_block *bbs;
unsigned i;
- estimate_loops_at_level (loop->inner);
+ estimate_loops_at_level (loop->inner, tovisit);
- if (loop->latch->succ) /* Do not do this for dummy function loop. */
+ /* Do not do this for dummy function loop. */
+ if (EDGE_COUNT (loop->latch->succs) > 0)
{
/* Find current loop back edge and mark it. */
e = loop_latch_edge (loop);
bbs = get_loop_body (loop);
for (i = 0; i < loop->num_nodes; i++)
- BLOCK_INFO (bbs[i])->tovisit = 1;
+ bitmap_set_bit (tovisit, bbs[i]->index);
free (bbs);
- propagate_freq (loop);
+ propagate_freq (loop, tovisit);
}
}
/* Convert counts measured by profile driven feedback to frequencies.
Return nonzero iff there was any nonzero execution count. */
-static int
+int
counts_to_freqs (void)
{
gcov_type count_max, true_count_max = 0;
if (!flag_branch_probabilities || !counts_to_freqs ())
{
static int real_values_initialized = 0;
+ bitmap tovisit;
if (!real_values_initialized)
{
mark_dfs_back_edges ();
- ENTRY_BLOCK_PTR->succ->probability = REG_BR_PROB_BASE;
+ EDGE_SUCC (ENTRY_BLOCK_PTR, 0)->probability = REG_BR_PROB_BASE;
/* Set up block info for each basic block. */
+ tovisit = BITMAP_XMALLOC ();
alloc_aux_for_blocks (sizeof (struct block_info_def));
alloc_aux_for_edges (sizeof (struct edge_info_def));
FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR, NULL, next_bb)
{
edge e;
+ edge_iterator ei;
- BLOCK_INFO (bb)->tovisit = 0;
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
{
sreal_init (&EDGE_INFO (e)->back_edge_prob, e->probability, 0);
sreal_mul (&EDGE_INFO (e)->back_edge_prob,
/* First compute probabilities locally for each loop from innermost
to outermost to examine probabilities for back edges. */
- estimate_loops_at_level (loops->tree_root);
+ estimate_loops_at_level (loops->tree_root, tovisit);
memcpy (&freq_max, &real_zero, sizeof (real_zero));
FOR_EACH_BB (bb)
free_aux_for_blocks ();
free_aux_for_edges ();
+ BITMAP_XFREE (tovisit);
}
compute_function_frequency ();
if (flag_reorder_functions)
OSDN Git Service
