/* Inlining decision heuristics.
- Copyright (C) 2003, 2004, 2007, 2008, 2009 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
INLINE_SIZE,
INLINE_ALL
};
+
static bool
-cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode,
- int);
+cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode);
+static void cgraph_flatten (struct cgraph_node *node);
/* Statistics we collect about inlining algorithm. */
return size;
}
+/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
+ by NEST. */
+
+static void
+update_noncloned_frequencies (struct cgraph_node *node,
+ int freq_scale, int nest)
+{
+ struct cgraph_edge *e;
+
+ /* We do not want to ignore high loop nest after freq drops to 0. */
+ if (!freq_scale)
+ freq_scale = 1;
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ e->loop_nest += nest;
+ e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+ if (e->frequency > CGRAPH_FREQ_MAX)
+ e->frequency = CGRAPH_FREQ_MAX;
+ if (!e->inline_failed)
+ update_noncloned_frequencies (e->callee, freq_scale, nest);
+ }
+}
+
/* E is expected to be an edge being inlined. Clone destination node of
the edge and redirect it to the new clone.
DUPLICATE is used for bookkeeping on whether we are actually creating new
/* We may eliminate the need for out-of-line copy to be output.
In that case just go ahead and re-use it. */
if (!e->callee->callers->next_caller
- && !e->callee->needed
+ && cgraph_can_remove_if_no_direct_calls_p (e->callee)
+ /* Don't reuse if more than one function shares a comdat group.
+ If the other function(s) are needed, we need to emit even
+ this function out of line. */
+ && !e->callee->same_comdat_group
&& !cgraph_new_nodes)
{
gcc_assert (!e->callee->global.inlined_to);
nfunctions_inlined++;
}
duplicate = false;
+ e->callee->local.externally_visible = false;
+ update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
}
else
{
struct cgraph_node *n;
- n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
- update_original);
+ n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+ update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
}
+ inline_summary (e->callee)->estimated_self_stack_size;
if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
e->callee->global.inlined_to->global.estimated_stack_size = peak;
+ cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
struct cgraph_node *to = NULL, *what;
struct cgraph_edge *curr = e;
int freq;
- bool duplicate = false;
- int orig_size = e->callee->global.size;
gcc_assert (e->inline_failed);
e->inline_failed = CIF_OK;
DECL_POSSIBLY_INLINED (e->callee->decl) = true;
e->callee->global.inlined = true;
- if (e->callee->callers->next_caller
- || e->callee->needed)
- duplicate = true;
cgraph_clone_inlined_nodes (e, true, update_original);
what = e->callee;
gcc_assert (what->global.inlined_to == to);
if (new_size > old_size)
overall_size += new_size - old_size;
- if (!duplicate)
- overall_size -= orig_size;
ncalls_inlined++;
if (flag_indirect_inlining)
return false;
}
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
- Return following unredirected edge in the list of callers
- of EDGE->CALLEE */
+/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER. */
-static struct cgraph_edge *
+static void
cgraph_mark_inline (struct cgraph_edge *edge)
{
struct cgraph_node *to = edge->caller;
struct cgraph_node *what = edge->callee;
struct cgraph_edge *e, *next;
- gcc_assert (!gimple_call_cannot_inline_p (edge->call_stmt));
+ gcc_assert (!edge->call_stmt_cannot_inline_p);
/* Look for all calls, mark them inline and clone recursively
all inlined functions. */
for (e = what->callers; e; e = next)
edge = next;
}
}
-
- return edge;
}
/* Estimate the growth caused by inlining NODE into all callees. */
we decide to not inline for different reasons, but it is not big deal
as in that case we will keep the body around, but we will also avoid
some inlining. */
- if (!node->needed && !DECL_EXTERNAL (node->decl) && !self_recursive)
+ if (cgraph_only_called_directly_p (node)
+ && !DECL_EXTERNAL (node->decl) && !self_recursive)
growth -= node->global.size;
node->global.estimated_growth = growth;
}
/* Return false when inlining WHAT into TO is not good idea
- as it would cause too large growth of function bodies.
+ as it would cause too large growth of function bodies.
When ONE_ONLY is true, assume that only one call site is going
to be inlined, otherwise figure out how many call sites in
TO calls WHAT and verify that all can be inlined.
{
tree decl = n->decl;
+ if (n->local.disregard_inline_limits)
+ return true;
+
if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl))
{
if (reason)
of the function or function body size. */
static int
-cgraph_edge_badness (struct cgraph_edge *edge)
+cgraph_edge_badness (struct cgraph_edge *edge, bool dump)
{
gcov_type badness;
int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
- growth -= edge->caller->global.size;
+ if (dump)
+ {
+ fprintf (dump_file, " Badness calculcation for %s -> %s\n",
+ cgraph_node_name (edge->caller),
+ cgraph_node_name (edge->callee));
+ fprintf (dump_file, " growth %i, time %i-%i, size %i-%i\n",
+ growth,
+ edge->callee->global.time,
+ inline_summary (edge->callee)->time_inlining_benefit,
+ edge->callee->global.size,
+ inline_summary (edge->callee)->size_inlining_benefit);
+ }
/* Always prefer inlining saving code size. */
if (growth <= 0)
- badness = INT_MIN - growth;
+ {
+ badness = INT_MIN - growth;
+ if (dump)
+ fprintf (dump_file, " %i: Growth %i < 0\n", (int) badness,
+ growth);
+ }
/* When profiling is available, base priorities -(#calls / growth).
So we optimize for overall number of "executed" inlined calls. */
else if (max_count)
- badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1))
- * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ {
+ badness =
+ ((int)
+ ((double) edge->count * INT_MIN / max_count / (max_benefit + 1)) *
+ (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i (relative %f): profile info. Relative count %f"
+ " * Relative benefit %f\n",
+ (int) badness, (double) badness / INT_MIN,
+ (double) edge->count / max_count,
+ (double) (inline_summary (edge->callee)->
+ time_inlining_benefit + 1) / (max_benefit + 1));
+ }
+ }
/* When function local profile is available, base priorities on
growth / frequency, so we optimize for overall frequency of inlined
else if (flag_guess_branch_prob)
{
int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1;
+ int benefitperc;
+ int growth_for_all;
badness = growth * 10000;
- div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
- / (edge->callee->global.time + 1) + 1, 100);
-
+ benefitperc =
+ MIN (100 * inline_summary (edge->callee)->time_inlining_benefit /
+ (edge->callee->global.time + 1) +1, 100);
+ div *= benefitperc;
+
/* Decrease badness if call is nested. */
/* Compress the range so we don't overflow. */
div = 1;
if (badness > 0)
badness /= div;
- badness += cgraph_estimate_growth (edge->callee);
+ growth_for_all = cgraph_estimate_growth (edge->callee);
+ badness += growth_for_all;
if (badness > INT_MAX)
- badness = INT_MAX;
+ badness = INT_MAX;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i: guessed profile. frequency %i, overall growth %i,"
+ " benefit %i%%, divisor %i\n",
+ (int) badness, edge->frequency, growth_for_all, benefitperc, div);
+ }
}
/* When function local profile is not available or it does not give
useful information (ie frequency is zero), base the cost on
badness = cgraph_estimate_growth (edge->callee) * 256;
/* Decrease badness if call is nested. */
- if (badness > 0)
+ if (badness > 0)
badness >>= nest;
else
- {
+ {
badness <<= nest;
- }
+ }
+ if (dump)
+ fprintf (dump_file, " %i: no profile. nest %i\n", (int) badness,
+ nest);
}
+
+ /* Ensure that we did not overflow in all the fixed point math above. */
+ gcc_assert (badness >= INT_MIN);
+ gcc_assert (badness <= INT_MAX - 1);
/* Make recursive inlining happen always after other inlining is done. */
if (cgraph_recursive_inlining_p (edge->caller, edge->callee, NULL))
return badness + 1;
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
{
- int badness = cgraph_edge_badness (edge);
+ int badness = cgraph_edge_badness (edge, false);
if (edge->aux)
{
fibnode_t n = (fibnode_t) edge->aux;
continue;
/* fibheap_replace_key only increase the keys. */
- if (fibheap_replace_key (heap, n, badness))
- continue;
+ if (badness < n->key)
+ {
+ fibheap_replace_key (heap, n, badness);
+ gcc_assert (n->key == badness);
+ continue;
+ }
fibheap_delete_node (heap, (fibnode_t) edge->aux);
}
edge->aux = fibheap_insert (heap, badness, edge);
int depth = 0;
int n = 0;
+ /* It does not make sense to recursively inline always-inline functions
+ as we are going to sorry() on the remaining calls anyway. */
+ if (node->local.disregard_inline_limits
+ && lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)))
+ return false;
+
if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))
|| (!flag_inline_functions && !DECL_DECLARED_INLINE_P (node->decl)))
return false;
}
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" Performing recursive inlining on %s\n",
cgraph_node_name (node));
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1, false);
+ master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1,
+ false, NULL);
master_clone->needed = true;
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
if (depth > max_depth)
{
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" maximal depth reached\n");
continue;
}
if (curr->count * 100 / node->count < probability)
{
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" Probability of edge is too small\n");
continue;
}
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
" Inlining call of depth %i", depth);
if (node->count)
{
fibheap_delete (heap);
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
"\n Inlined %i times, body grown from size %i to %i, time %i to %i\n", n,
master_clone->global.size, node->global.size,
master_clone->global.time, node->global.time);
struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
for (node = cgraph_nodes; node; node = node->next)
{
- if (!node->local.inlinable || !node->callers
- || node->local.disregard_inline_limits)
+ if (!node->local.inlinable || !node->callers)
continue;
if (dump_file)
fprintf (dump_file, "Considering inline candidate %s.\n", cgraph_node_name (node));
if (edge->inline_failed)
{
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
min_size = overall_size;
while (overall_size <= max_size
- && (edge = (struct cgraph_edge *) fibheap_extract_min (heap)))
+ && !fibheap_empty (heap))
{
int old_size = overall_size;
- struct cgraph_node *where;
- int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ struct cgraph_node *where, *callee;
+ int badness = fibheap_min_key (heap);
+ int growth;
cgraph_inline_failed_t not_good = CIF_OK;
- growth -= edge->caller->global.size;
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
+ gcc_assert (edge->aux);
+ edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ callee = edge->callee;
+
+ growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
"\nConsidering %s with %i size\n",
cgraph_node_name (edge->callee),
edge->callee->global.size);
- fprintf (dump_file,
+ fprintf (dump_file,
" to be inlined into %s in %s:%i\n"
" Estimated growth after inlined into all callees is %+i insns.\n"
" Estimated badness is %i, frequency %.2f.\n",
cgraph_node_name (edge->caller),
- gimple_filename ((const_gimple) edge->call_stmt),
- gimple_lineno ((const_gimple) edge->call_stmt),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
cgraph_estimate_growth (edge->callee),
- cgraph_edge_badness (edge),
+ badness,
edge->frequency / (double)CGRAPH_FREQ_BASE);
if (edge->count)
fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
}
- gcc_assert (edge->aux);
- edge->aux = NULL;
- if (!edge->inline_failed)
- continue;
/* When not having profile info ready we don't weight by any way the
position of call in procedure itself. This means if call of
else
{
struct cgraph_node *callee;
- if (gimple_call_cannot_inline_p (edge->call_stmt)
+ if (edge->call_stmt_cannot_inline_p
|| !cgraph_check_inline_limits (edge->caller, edge->callee,
&edge->inline_failed, true))
{
called by function we inlined (since number of it inlinable callers
might change). */
update_caller_keys (heap, where, updated_nodes);
+
+ /* We removed one call of the function we just inlined. If offline
+ copy is still needed, be sure to update the keys. */
+ if (callee != where && !callee->global.inlined_to)
+ update_caller_keys (heap, callee, updated_nodes);
bitmap_clear (updated_nodes);
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
" Inlined into %s which now has size %i and self time %i,"
"net change of %+i.\n",
cgraph_node_name (edge->caller),
fprintf (dump_file, "New minimal size reached: %i\n", min_size);
}
}
- while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL)
+ while (!fibheap_empty (heap))
{
+ int badness = fibheap_min_key (heap);
+
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
gcc_assert (edge->aux);
edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "\nSkipping %s with %i size\n",
+ cgraph_node_name (edge->callee),
+ edge->callee->global.size);
+ fprintf (dump_file,
+ " called by %s in %s:%i\n"
+ " Estimated growth after inlined into all callees is %+i insns.\n"
+ " Estimated badness is %i, frequency %.2f.\n",
+ cgraph_node_name (edge->caller),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
+ cgraph_estimate_growth (edge->callee),
+ badness,
+ edge->frequency / (double)CGRAPH_FREQ_BASE);
+ if (edge->count)
+ fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
+ }
if (!edge->callee->local.disregard_inline_limits && edge->inline_failed
&& !cgraph_recursive_inlining_p (edge->caller, edge->callee,
&edge->inline_failed))
BITMAP_FREE (updated_nodes);
}
+/* Flatten NODE from the IPA inliner. */
+
+static void
+cgraph_flatten (struct cgraph_node *node)
+{
+ struct cgraph_edge *e;
+
+ /* We shouldn't be called recursively when we are being processed. */
+ gcc_assert (node->aux == NULL);
+
+ node->aux = (void *)(size_t) INLINE_ALL;
+
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ struct cgraph_node *orig_callee;
+
+ if (e->call_stmt_cannot_inline_p)
+ continue;
+
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining: Function body not available.\n");
+ continue;
+ }
+
+ /* We've hit cycle? It is time to give up. */
+ if (e->callee->aux)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining %s into %s to avoid cycle.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ e->inline_failed = CIF_RECURSIVE_INLINING;
+ continue;
+ }
+
+ /* When the edge is already inlined, we just need to recurse into
+ it in order to fully flatten the leaves. */
+ if (!e->inline_failed)
+ {
+ cgraph_flatten (e->callee);
+ continue;
+ }
+
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ continue;
+ }
+
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
+
+ /* Inline the edge and flatten the inline clone. Avoid
+ recursing through the original node if the node was cloned. */
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ orig_callee = e->callee;
+ cgraph_mark_inline_edge (e, true, NULL);
+ if (e->callee != orig_callee)
+ orig_callee->aux = (void *)(size_t) INLINE_ALL;
+ cgraph_flatten (e->callee);
+ if (e->callee != orig_callee)
+ orig_callee->aux = NULL;
+ }
+
+ node->aux = NULL;
+}
+
/* Decide on the inlining. We do so in the topological order to avoid
expenses on updating data structures. */
XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
int old_size = 0;
int i;
- bool redo_always_inline = true;
int initial_size = 0;
cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
+ if (in_lto_p && flag_indirect_inlining)
+ ipa_update_after_lto_read ();
max_count = 0;
max_benefit = 0;
struct cgraph_edge *e;
gcc_assert (inline_summary (node)->self_size == node->global.size);
- gcc_assert (node->needed || node->reachable);
initial_size += node->global.size;
for (e = node->callees; e; e = e->next_callee)
if (max_count < e->count)
if (max_benefit < inline_summary (node)->time_inlining_benefit)
max_benefit = inline_summary (node)->time_inlining_benefit;
}
- gcc_assert (!max_count || (profile_info && flag_branch_probabilities));
+ gcc_assert (in_lto_p
+ || !max_count
+ || (profile_info && flag_branch_probabilities));
overall_size = initial_size;
nnodes = cgraph_postorder (order);
node->aux = 0;
if (dump_file)
- fprintf (dump_file, "\nInlining always_inline functions:\n");
+ fprintf (dump_file, "\nFlattening functions:\n");
- /* In the first pass mark all always_inline edges. Do this with a priority
- so none of our later choices will make this impossible. */
- while (redo_always_inline)
+ /* In the first pass handle functions to be flattened. Do this with
+ a priority so none of our later choices will make this impossible. */
+ for (i = nnodes - 1; i >= 0; i--)
{
- redo_always_inline = false;
- for (i = nnodes - 1; i >= 0; i--)
+ node = order[i];
+
+ /* Handle nodes to be flattened, but don't update overall unit
+ size. Calling the incremental inliner here is lame,
+ a simple worklist should be enough. What should be left
+ here from the early inliner (if it runs) is cyclic cases.
+ Ideally when processing callees we stop inlining at the
+ entry of cycles, possibly cloning that entry point and
+ try to flatten itself turning it into a self-recursive
+ function. */
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
{
- struct cgraph_edge *e, *next;
-
- node = order[i];
-
- /* Handle nodes to be flattened, but don't update overall unit
- size. */
- if (lookup_attribute ("flatten",
- DECL_ATTRIBUTES (node->decl)) != NULL)
- {
- if (dump_file)
- fprintf (dump_file,
- "Flattening %s\n", cgraph_node_name (node));
- cgraph_decide_inlining_incrementally (node, INLINE_ALL, 0);
- }
-
- if (!node->local.disregard_inline_limits)
- continue;
if (dump_file)
fprintf (dump_file,
- "\nConsidering %s size:%i (always inline)\n",
- cgraph_node_name (node), node->global.size);
- old_size = overall_size;
- for (e = node->callers; e; e = next)
- {
- next = e->next_caller;
- if (!e->inline_failed
- || gimple_call_cannot_inline_p (e->call_stmt))
- continue;
- if (cgraph_recursive_inlining_p (e->caller, e->callee,
- &e->inline_failed))
- continue;
- if (!tree_can_inline_p (e))
- continue;
- if (cgraph_mark_inline_edge (e, true, NULL))
- redo_always_inline = true;
- if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has size %i.\n",
- cgraph_node_name (e->caller),
- e->caller->global.size);
- }
- /* Inlining self recursive function might introduce new calls to
- themselves we didn't see in the loop above. Fill in the proper
- reason why inline failed. */
- for (e = node->callers; e; e = e->next_caller)
- if (e->inline_failed)
- e->inline_failed = CIF_RECURSIVE_INLINING;
- if (dump_file)
- fprintf (dump_file,
- " Inlined for a net change of %+i size.\n",
- overall_size - old_size);
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
}
}
if (node->callers
&& !node->callers->next_caller
- && !node->needed
+ && cgraph_only_called_directly_p (node)
&& node->local.inlinable
&& node->callers->inline_failed
&& node->callers->caller != node
&& node->callers->caller->global.inlined_to != node
- && !gimple_call_cannot_inline_p (node->callers->call_stmt)
+ && !node->callers->call_stmt_cannot_inline_p
&& !DECL_EXTERNAL (node->decl)
&& !DECL_COMDAT (node->decl))
{
+ cgraph_inline_failed_t reason;
old_size = overall_size;
if (dump_file)
{
}
if (cgraph_check_inline_limits (node->callers->caller, node,
- NULL, false))
+ &reason, false))
{
+ struct cgraph_node *caller = node->callers->caller;
cgraph_mark_inline (node->callers);
if (dump_file)
fprintf (dump_file,
" Inlined into %s which now has %i size"
" for a net change of %+i size.\n",
- cgraph_node_name (node->callers->caller),
- node->callers->caller->global.size,
+ cgraph_node_name (caller),
+ caller->global.size,
overall_size - old_size);
}
else
{
if (dump_file)
fprintf (dump_file,
- " Inline limit reached, not inlined.\n");
+ " Not inlining: %s.\n",
+ cgraph_inline_failed_string (reason));
}
}
}
return 0;
}
-/* Try to inline edge E from incremental inliner. MODE specifies mode
- of inliner.
-
- We are detecting cycles by storing mode of inliner into cgraph_node last
- time we visited it in the recursion. In general when mode is set, we have
- recursive inlining, but as an special case, we want to try harder inline
- ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
- flatten, b being always inline. Flattening 'a' will collapse
- a->b->c before hitting cycle. To accommodate always inline, we however
- need to inline a->b->c->b.
-
- So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
- stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
-static bool
-try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth)
-{
- struct cgraph_node *callee = e->callee;
- enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux;
- bool always_inline = e->callee->local.disregard_inline_limits;
- bool inlined = false;
-
- /* We've hit cycle? */
- if (callee_mode)
- {
- /* It is first time we see it and we are not in ALWAY_INLINE only
- mode yet. and the function in question is always_inline. */
- if (always_inline && mode != INLINE_ALWAYS_INLINE)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Hit cycle in %s, switching to always inline only.\n",
- cgraph_node_name (callee));
- }
- mode = INLINE_ALWAYS_INLINE;
- }
- /* Otherwise it is time to give up. */
- else
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining %s into %s to avoid cycle.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
- }
- e->inline_failed = (e->callee->local.disregard_inline_limits
- ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
- return false;
- }
- }
-
- callee->aux = (void *)(size_t) mode;
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, " Inlining %s into %s.\n",
- cgraph_node_name (e->callee),
- cgraph_node_name (e->caller));
- }
- if (e->inline_failed)
- {
- cgraph_mark_inline (e);
-
- /* In order to fully inline always_inline functions, we need to
- recurse here, since the inlined functions might not be processed by
- incremental inlining at all yet.
-
- Also flattening needs to be done recursively. */
-
- if (mode == INLINE_ALL || always_inline)
- cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1);
- inlined = true;
- }
- callee->aux = (void *)(size_t) callee_mode;
- return inlined;
-}
-
/* Return true when N is leaf function. Accept cheap (pure&const) builtins
in leaf functions. */
static bool
}
/* Decide on the inlining. We do so in the topological order to avoid
- expenses on updating data structures.
- DEPTH is depth of recursion, used only for debug output. */
+ expenses on updating data structures. */
static bool
cgraph_decide_inlining_incrementally (struct cgraph_node *node,
- enum inlining_mode mode,
- int depth)
+ enum inlining_mode mode)
{
struct cgraph_edge *e;
bool inlined = false;
cgraph_inline_failed_t failed_reason;
- enum inlining_mode old_mode;
#ifdef ENABLE_CHECKING
verify_cgraph_node (node);
#endif
- old_mode = (enum inlining_mode) (size_t)node->aux;
-
if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE
&& lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Flattening %s\n", cgraph_node_name (node));
- }
+ fprintf (dump_file, "Incrementally flattening %s\n",
+ cgraph_node_name (node));
mode = INLINE_ALL;
}
- node->aux = (void *)(size_t) mode;
-
/* First of all look for always inline functions. */
if (mode != INLINE_SIZE_NORECURSIVE)
for (e = node->callees; e; e = e->next_callee)
if (!e->callee->local.disregard_inline_limits
&& (mode != INLINE_ALL || !e->callee->local.inlinable))
continue;
- if (gimple_call_cannot_inline_p (e->call_stmt))
+ if (e->call_stmt_cannot_inline_p)
continue;
- /* When the edge is already inlined, we just need to recurse into
- it in order to fully flatten the leaves. */
- if (!e->inline_failed && mode == INLINE_ALL)
- {
- inlined |= try_inline (e, mode, depth);
- continue;
- }
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Considering to always inline inline candidate %s.\n",
- cgraph_node_name (e->callee));
- }
+ fprintf (dump_file,
+ "Considering to always inline inline candidate %s.\n",
+ cgraph_node_name (e->callee));
if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
if (!tree_can_inline_p (e))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: %s",
- cgraph_inline_failed_string (e->inline_failed));
- }
+ fprintf (dump_file,
+ "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
continue;
}
if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
!= gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ fprintf (dump_file, "Not inlining: SSA form does not match.\n");
continue;
}
if (!e->callee->analyzed)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: Function body no longer available.\n");
- }
+ fprintf (dump_file,
+ "Not inlining: Function body no longer available.\n");
continue;
}
- inlined |= try_inline (e, mode, depth);
+
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
}
/* Now do the automatic inlining. */
- if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE)
- for (e = node->callees; e; e = e->next_callee)
- {
- int allowed_growth = 0;
- if (!e->callee->local.inlinable
- || !e->inline_failed
- || e->callee->local.disregard_inline_limits)
- continue;
- if (dump_file)
- fprintf (dump_file, "Considering inline candidate %s.\n",
- cgraph_node_name (e->callee));
- if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE
+ /* Never inline regular functions into always-inline functions
+ during incremental inlining. */
+ && !node->local.disregard_inline_limits)
+ {
+ bitmap visited = BITMAP_ALLOC (NULL);
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ int allowed_growth = 0;
+ if (!e->callee->local.inlinable
+ || !e->inline_failed
+ || e->callee->local.disregard_inline_limits)
continue;
- }
- if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
- != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ /* We are inlining a function to all call-sites in node
+ or to none. So visit each candidate only once. */
+ if (!bitmap_set_bit (visited, e->callee->uid))
continue;
- }
-
- if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
- && optimize_function_for_speed_p (cfun))
- allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+ if (dump_file)
+ fprintf (dump_file, "Considering inline candidate %s.\n",
+ cgraph_node_name (e->callee));
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ continue;
+ }
+ if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+ != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining: SSA form does not match.\n");
+ continue;
+ }
- /* When the function body would grow and inlining the function won't
- eliminate the need for offline copy of the function, don't inline.
- */
- if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
- || (!flag_inline_functions
- && !DECL_DECLARED_INLINE_P (e->callee->decl)))
- && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
- > e->caller->global.size + allowed_growth)
- && cgraph_estimate_growth (e->callee) > allowed_growth)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
+ && optimize_function_for_speed_p (cfun))
+ allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+
+ /* When the function body would grow and inlining the function
+ won't eliminate the need for offline copy of the function,
+ don't inline. */
+ if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
+ || (!flag_inline_functions
+ && !DECL_DECLARED_INLINE_P (e->callee->decl)))
+ && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+ > e->caller->global.size + allowed_growth)
+ && cgraph_estimate_growth (e->callee) > allowed_growth)
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: code size would grow by %i.\n",
cgraph_estimate_size_after_inlining (1, e->caller,
e->callee)
- e->caller->global.size);
- }
- continue;
- }
- if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
- false)
- || gimple_call_cannot_inline_p (e->call_stmt))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+ false)
+ || e->call_stmt_cannot_inline_p)
+ {
+ if (dump_file)
fprintf (dump_file, "Not inlining: %s.\n",
cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (!e->callee->analyzed)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: Function body no longer available.\n");
- }
- continue;
- }
- if (!tree_can_inline_p (e))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: %s.",
- cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (cgraph_default_inline_p (e->callee, &failed_reason))
- inlined |= try_inline (e, mode, depth);
- }
- node->aux = (void *)(size_t) old_mode;
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
+ if (cgraph_default_inline_p (e->callee, &failed_reason))
+ {
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
+ }
+ }
+ BITMAP_FREE (visited);
+ }
return inlined;
}
if (sorrycount || errorcount)
return 0;
- while (cgraph_decide_inlining_incrementally (node,
- iterations
- ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0)
- && iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS))
+
+ if (!optimize
+ || flag_no_inline
+ || !flag_early_inlining)
{
+ /* When not optimizing or not inlining inline only always-inline
+ functions. */
+ cgraph_decide_inlining_incrementally (node, INLINE_ALWAYS_INLINE);
timevar_push (TV_INTEGRATION);
todo |= optimize_inline_calls (current_function_decl);
- iterations++;
timevar_pop (TV_INTEGRATION);
}
- if (dump_file)
- fprintf (dump_file, "Iterations: %i\n", iterations);
+ else
+ {
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ timevar_pop (TV_INTEGRATION);
+ }
+ /* We iterate incremental inlining to get trivial cases of indirect
+ inlining. */
+ while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+ && cgraph_decide_inlining_incrementally (node,
+ iterations
+ ? INLINE_SIZE_NORECURSIVE
+ : INLINE_SIZE))
+ {
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ iterations++;
+ timevar_pop (TV_INTEGRATION);
+ }
+ if (dump_file)
+ fprintf (dump_file, "Iterations: %i\n", iterations);
+ }
+
cfun->always_inline_functions_inlined = true;
- return todo;
-}
-/* When inlining shall be performed. */
-static bool
-cgraph_gate_early_inlining (void)
-{
- return flag_early_inlining;
+ return todo;
}
-struct gimple_opt_pass pass_early_inline =
+struct gimple_opt_pass pass_early_inline =
{
{
GIMPLE_PASS,
"einline", /* name */
- cgraph_gate_early_inlining, /* gate */
+ NULL, /* gate */
cgraph_early_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
cgraph_gate_ipa_early_inlining (void)
{
return (flag_early_inlining
+ && !in_lto_p
&& (flag_branch_probabilities || flag_test_coverage
|| profile_arc_flag));
}
/* IPA pass wrapper for early inlining pass. We need to run early inlining
before tree profiling so we have stand alone IPA pass for doing so. */
-struct simple_ipa_opt_pass pass_ipa_early_inline =
+struct simple_ipa_opt_pass pass_ipa_early_inline =
{
{
SIMPLE_IPA_PASS,
while (handled_component_p (inner_rhs)
|| TREE_CODE (inner_rhs) == ADDR_EXPR || TREE_CODE (inner_rhs) == INDIRECT_REF)
inner_rhs = TREE_OPERAND (inner_rhs, 0);
-
+
if (TREE_CODE (inner_rhs) == PARM_DECL
|| (TREE_CODE (inner_rhs) == SSA_NAME
int freq;
tree funtype = TREE_TYPE (node->decl);
+ if (node->local.disregard_inline_limits)
+ {
+ inline_summary (node)->self_time = 0;
+ inline_summary (node)->self_size = 0;
+ inline_summary (node)->time_inlining_benefit = 0;
+ inline_summary (node)->size_inlining_benefit = 0;
+ }
+
if (dump_file)
fprintf (dump_file, "Analyzing function body size: %s\n",
cgraph_node_name (node));
node->global.stack_frame_offset = 0;
/* Can this function be inlined at all? */
- node->local.inlinable = tree_inlinable_function_p (current_function_decl);
+ node->local.inlinable = tree_inlinable_function_p (node->decl);
if (node->local.inlinable && !node->local.disregard_inline_limits)
node->local.disregard_inline_limits
- = DECL_DISREGARD_INLINE_LIMITS (current_function_decl);
+ = DECL_DISREGARD_INLINE_LIMITS (node->decl);
estimate_function_body_sizes (node);
/* Inlining characteristics are maintained by the cgraph_mark_inline. */
node->global.time = inline_summary (node)->self_time;
return 0;
}
-struct gimple_opt_pass pass_inline_parameters =
+struct gimple_opt_pass pass_inline_parameters =
{
{
GIMPLE_PASS,
for (node = cgraph_nodes; node; node = node->next)
if (node->analyzed)
analyze_function (node);
-
+
return;
}
unsigned int todo = 0;
struct cgraph_edge *e;
+ /* FIXME: Currently the passmanager is adding inline transform more than once to some
+ clones. This needs revisiting after WPA cleanups. */
+ if (cfun->after_inlining)
+ return 0;
+
/* We might need the body of this function so that we can expand
it inline somewhere else. */
if (cgraph_preserve_function_body_p (node->decl))
return todo | execute_fixup_cfg ();
}
+/* Read inline summary. Jump functions are shared among ipa-cp
+ and inliner, so when ipa-cp is active, we don't need to write them
+ twice. */
+
+static void
+inline_read_summary (void)
+{
+ if (flag_indirect_inlining)
+ {
+ ipa_register_cgraph_hooks ();
+ if (!flag_ipa_cp)
+ ipa_prop_read_jump_functions ();
+ }
+ function_insertion_hook_holder =
+ cgraph_add_function_insertion_hook (&add_new_function, NULL);
+}
+
+/* Write inline summary for node in SET.
+ Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
+ active, we don't need to write them twice. */
+
+static void
+inline_write_summary (cgraph_node_set set)
+{
+ if (flag_indirect_inlining && !flag_ipa_cp)
+ ipa_prop_write_jump_functions (set);
+}
+
+/* When to run IPA inlining. Inlining of always-inline functions
+ happens during early inlining. */
+
+static bool
+gate_cgraph_decide_inlining (void)
+{
+ /* ??? We'd like to skip this if not optimizing or not inlining as
+ all always-inline functions have been processed by early
+ inlining already. But this at least breaks EH with C++ as
+ we need to unconditionally run fixup_cfg even at -O0.
+ So leave it on unconditionally for now. */
+ return 1;
+}
+
struct ipa_opt_pass_d pass_ipa_inline =
{
{
IPA_PASS,
"inline", /* name */
- NULL, /* gate */
+ gate_cgraph_decide_inlining, /* gate */
cgraph_decide_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
| TODO_remove_functions /* todo_flags_finish */
},
inline_generate_summary, /* generate_summary */
- NULL, /* write_summary */
- NULL, /* read_summary */
- NULL, /* function_read_summary */
+ inline_write_summary, /* write_summary */
+ inline_read_summary, /* read_summary */
+ NULL, /* write_optimization_summary */
+ NULL, /* read_optimization_summary */
+ lto_ipa_fixup_call_notes, /* stmt_fixup */
0, /* TODOs */
inline_transform, /* function_transform */
NULL, /* variable_transform */
OSDN Git Service
