/* Inlining decision heuristics.
- Copyright (C) 2003, 2004, 2007, 2008, 2009 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "flags.h"
#include "cgraph.h"
#include "diagnostic.h"
+#include "gimple-pretty-print.h"
#include "timevar.h"
#include "params.h"
#include "fibheap.h"
INLINE_SIZE,
INLINE_ALL
};
+
static bool
-cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode,
- int);
+cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode);
+static void cgraph_flatten (struct cgraph_node *node);
/* Statistics we collect about inlining algorithm. */
else
{
struct cgraph_node *n;
- n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+ n = cgraph_clone_node (e->callee, e->callee->decl,
+ e->count, e->frequency, e->loop_nest,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
+ inline_summary (e->callee)->estimated_self_stack_size;
if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
e->callee->global.inlined_to->global.estimated_stack_size = peak;
+ cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
struct cgraph_node *to = NULL, *what;
struct cgraph_edge *curr = e;
int freq;
- bool duplicate = false;
- int orig_size = e->callee->global.size;
gcc_assert (e->inline_failed);
e->inline_failed = CIF_OK;
+ DECL_POSSIBLY_INLINED (e->callee->decl) = true;
- if (!e->callee->global.inlined)
- DECL_POSSIBLY_INLINED (e->callee->decl) = true;
- e->callee->global.inlined = true;
-
- if (e->callee->callers->next_caller
- || !cgraph_can_remove_if_no_direct_calls_p (e->callee)
- || e->callee->same_comdat_group)
- duplicate = true;
cgraph_clone_inlined_nodes (e, true, update_original);
what = e->callee;
gcc_assert (what->global.inlined_to == to);
if (new_size > old_size)
overall_size += new_size - old_size;
- if (!duplicate)
- overall_size -= orig_size;
ncalls_inlined++;
if (flag_indirect_inlining)
return false;
}
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
- Return following unredirected edge in the list of callers
- of EDGE->CALLEE */
+/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER. */
-static struct cgraph_edge *
+static void
cgraph_mark_inline (struct cgraph_edge *edge)
{
struct cgraph_node *to = edge->caller;
edge = next;
}
}
-
- return edge;
}
/* Estimate the growth caused by inlining NODE into all callees. */
{
tree decl = n->decl;
+ if (n->local.disregard_inline_limits)
+ return true;
+
if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl))
{
if (reason)
of the function or function body size. */
static int
-cgraph_edge_badness (struct cgraph_edge *edge)
+cgraph_edge_badness (struct cgraph_edge *edge, bool dump)
{
gcov_type badness;
int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
- growth -= edge->caller->global.size;
+ if (edge->callee->local.disregard_inline_limits)
+ return INT_MIN;
+
+ if (dump)
+ {
+ fprintf (dump_file, " Badness calculcation for %s -> %s\n",
+ cgraph_node_name (edge->caller),
+ cgraph_node_name (edge->callee));
+ fprintf (dump_file, " growth %i, time %i-%i, size %i-%i\n",
+ growth,
+ edge->callee->global.time,
+ inline_summary (edge->callee)->time_inlining_benefit,
+ edge->callee->global.size,
+ inline_summary (edge->callee)->size_inlining_benefit);
+ }
/* Always prefer inlining saving code size. */
if (growth <= 0)
- badness = INT_MIN - growth;
+ {
+ badness = INT_MIN - growth;
+ if (dump)
+ fprintf (dump_file, " %i: Growth %i < 0\n", (int) badness,
+ growth);
+ }
/* When profiling is available, base priorities -(#calls / growth).
So we optimize for overall number of "executed" inlined calls. */
else if (max_count)
- badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1))
- * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ {
+ badness =
+ ((int)
+ ((double) edge->count * INT_MIN / max_count / (max_benefit + 1)) *
+ (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i (relative %f): profile info. Relative count %f"
+ " * Relative benefit %f\n",
+ (int) badness, (double) badness / INT_MIN,
+ (double) edge->count / max_count,
+ (double) (inline_summary (edge->callee)->
+ time_inlining_benefit + 1) / (max_benefit + 1));
+ }
+ }
/* When function local profile is available, base priorities on
growth / frequency, so we optimize for overall frequency of inlined
else if (flag_guess_branch_prob)
{
int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1;
+ int benefitperc;
+ int growth_for_all;
badness = growth * 10000;
- div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
- / (edge->callee->global.time + 1) + 1, 100);
+ benefitperc =
+ MIN (100 * inline_summary (edge->callee)->time_inlining_benefit /
+ (edge->callee->global.time + 1) +1, 100);
+ div *= benefitperc;
/* Decrease badness if call is nested. */
div = 1;
if (badness > 0)
badness /= div;
- badness += cgraph_estimate_growth (edge->callee);
+ growth_for_all = cgraph_estimate_growth (edge->callee);
+ badness += growth_for_all;
if (badness > INT_MAX)
- badness = INT_MAX;
+ badness = INT_MAX;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i: guessed profile. frequency %i, overall growth %i,"
+ " benefit %i%%, divisor %i\n",
+ (int) badness, edge->frequency, growth_for_all, benefitperc, div);
+ }
}
/* When function local profile is not available or it does not give
useful information (ie frequency is zero), base the cost on
if (badness > 0)
badness >>= nest;
else
- {
+ {
badness <<= nest;
- }
+ }
+ if (dump)
+ fprintf (dump_file, " %i: no profile. nest %i\n", (int) badness,
+ nest);
}
+
+ /* Ensure that we did not overflow in all the fixed point math above. */
+ gcc_assert (badness >= INT_MIN);
+ gcc_assert (badness <= INT_MAX - 1);
/* Make recursive inlining happen always after other inlining is done. */
if (cgraph_recursive_inlining_p (edge->caller, edge->callee, NULL))
return badness + 1;
struct cgraph_edge *edge;
cgraph_inline_failed_t failed_reason;
- if (!node->local.inlinable || node->local.disregard_inline_limits
+ if (!node->local.inlinable
|| node->global.inlined_to)
return;
if (bitmap_bit_p (updated_nodes, node->uid))
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
{
- int badness = cgraph_edge_badness (edge);
+ int badness = cgraph_edge_badness (edge, false);
if (edge->aux)
{
fibnode_t n = (fibnode_t) edge->aux;
continue;
/* fibheap_replace_key only increase the keys. */
- if (fibheap_replace_key (heap, n, badness))
- continue;
+ if (badness < n->key)
+ {
+ fibheap_replace_key (heap, n, badness);
+ gcc_assert (n->key == badness);
+ continue;
+ }
fibheap_delete_node (heap, (fibnode_t) edge->aux);
}
edge->aux = fibheap_insert (heap, badness, edge);
int depth = 0;
int n = 0;
+ /* It does not make sense to recursively inline always-inline functions
+ as we are going to sorry() on the remaining calls anyway. */
+ if (node->local.disregard_inline_limits
+ && lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)))
+ return false;
+
if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))
|| (!flag_inline_functions && !DECL_DECLARED_INLINE_P (node->decl)))
return false;
cgraph_node_name (node));
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1,
+ master_clone = cgraph_clone_node (node, node->decl,
+ node->count, CGRAPH_FREQ_BASE, 1,
false, NULL);
master_clone->needed = true;
for (e = master_clone->callees; e; e = e->next_callee)
struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
for (node = cgraph_nodes; node; node = node->next)
{
- if (!node->local.inlinable || !node->callers
- || node->local.disregard_inline_limits)
+ if (!node->local.inlinable || !node->callers)
continue;
if (dump_file)
fprintf (dump_file, "Considering inline candidate %s.\n", cgraph_node_name (node));
if (edge->inline_failed)
{
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
min_size = overall_size;
while (overall_size <= max_size
- && (edge = (struct cgraph_edge *) fibheap_extract_min (heap)))
+ && !fibheap_empty (heap))
{
int old_size = overall_size;
- struct cgraph_node *where;
- int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ struct cgraph_node *where, *callee;
+ int badness = fibheap_min_key (heap);
+ int growth;
cgraph_inline_failed_t not_good = CIF_OK;
- growth -= edge->caller->global.size;
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
+ gcc_assert (edge->aux);
+ edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ callee = edge->callee;
+
+ growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
if (dump_file)
{
" Estimated growth after inlined into all callees is %+i insns.\n"
" Estimated badness is %i, frequency %.2f.\n",
cgraph_node_name (edge->caller),
- gimple_filename ((const_gimple) edge->call_stmt),
- gimple_lineno ((const_gimple) edge->call_stmt),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
cgraph_estimate_growth (edge->callee),
- cgraph_edge_badness (edge),
+ badness,
edge->frequency / (double)CGRAPH_FREQ_BASE);
if (edge->count)
fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
}
- gcc_assert (edge->aux);
- edge->aux = NULL;
- if (!edge->inline_failed)
- continue;
/* When not having profile info ready we don't weight by any way the
position of call in procedure itself. This means if call of
}
}
- if (!cgraph_maybe_hot_edge_p (edge))
+ if (edge->callee->local.disregard_inline_limits)
+ ;
+ else if (!cgraph_maybe_hot_edge_p (edge))
not_good = CIF_UNLIKELY_CALL;
- if (!flag_inline_functions
+ else if (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (edge->callee->decl))
not_good = CIF_NOT_DECLARED_INLINED;
- if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
+ else if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
not_good = CIF_OPTIMIZING_FOR_SIZE;
if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0)
{
called by function we inlined (since number of it inlinable callers
might change). */
update_caller_keys (heap, where, updated_nodes);
+
+ /* We removed one call of the function we just inlined. If offline
+ copy is still needed, be sure to update the keys. */
+ if (callee != where && !callee->global.inlined_to)
+ update_caller_keys (heap, callee, updated_nodes);
bitmap_clear (updated_nodes);
if (dump_file)
fprintf (dump_file, "New minimal size reached: %i\n", min_size);
}
}
- while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL)
+ while (!fibheap_empty (heap))
{
+ int badness = fibheap_min_key (heap);
+
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
gcc_assert (edge->aux);
edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "\nSkipping %s with %i size\n",
+ cgraph_node_name (edge->callee),
+ edge->callee->global.size);
+ fprintf (dump_file,
+ " called by %s in %s:%i\n"
+ " Estimated growth after inlined into all callees is %+i insns.\n"
+ " Estimated badness is %i, frequency %.2f.\n",
+ cgraph_node_name (edge->caller),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
+ cgraph_estimate_growth (edge->callee),
+ badness,
+ edge->frequency / (double)CGRAPH_FREQ_BASE);
+ if (edge->count)
+ fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
+ }
if (!edge->callee->local.disregard_inline_limits && edge->inline_failed
&& !cgraph_recursive_inlining_p (edge->caller, edge->callee,
&edge->inline_failed))
BITMAP_FREE (updated_nodes);
}
+/* Flatten NODE from the IPA inliner. */
+
+static void
+cgraph_flatten (struct cgraph_node *node)
+{
+ struct cgraph_edge *e;
+
+ /* We shouldn't be called recursively when we are being processed. */
+ gcc_assert (node->aux == NULL);
+
+ node->aux = (void *)(size_t) INLINE_ALL;
+
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ struct cgraph_node *orig_callee;
+
+ if (e->call_stmt_cannot_inline_p)
+ continue;
+
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining: Function body not available.\n");
+ continue;
+ }
+
+ /* We've hit cycle? It is time to give up. */
+ if (e->callee->aux)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining %s into %s to avoid cycle.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ e->inline_failed = CIF_RECURSIVE_INLINING;
+ continue;
+ }
+
+ /* When the edge is already inlined, we just need to recurse into
+ it in order to fully flatten the leaves. */
+ if (!e->inline_failed)
+ {
+ cgraph_flatten (e->callee);
+ continue;
+ }
+
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ continue;
+ }
+
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
+
+ /* Inline the edge and flatten the inline clone. Avoid
+ recursing through the original node if the node was cloned. */
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ orig_callee = e->callee;
+ cgraph_mark_inline_edge (e, true, NULL);
+ if (e->callee != orig_callee)
+ orig_callee->aux = (void *)(size_t) INLINE_ALL;
+ cgraph_flatten (e->callee);
+ if (e->callee != orig_callee)
+ orig_callee->aux = NULL;
+ }
+
+ node->aux = NULL;
+}
+
/* Decide on the inlining. We do so in the topological order to avoid
expenses on updating data structures. */
XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
int old_size = 0;
int i;
- bool redo_always_inline = true;
int initial_size = 0;
cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
if (in_lto_p && flag_indirect_inlining)
ipa_update_after_lto_read ();
+ if (flag_indirect_inlining)
+ ipa_create_all_structures_for_iinln ();
max_count = 0;
max_benefit = 0;
node->aux = 0;
if (dump_file)
- fprintf (dump_file, "\nInlining always_inline functions:\n");
+ fprintf (dump_file, "\nFlattening functions:\n");
- /* In the first pass mark all always_inline edges. Do this with a priority
- so none of our later choices will make this impossible. */
- while (redo_always_inline)
+ /* In the first pass handle functions to be flattened. Do this with
+ a priority so none of our later choices will make this impossible. */
+ for (i = nnodes - 1; i >= 0; i--)
{
- redo_always_inline = false;
- for (i = nnodes - 1; i >= 0; i--)
+ node = order[i];
+
+ /* Handle nodes to be flattened, but don't update overall unit
+ size. Calling the incremental inliner here is lame,
+ a simple worklist should be enough. What should be left
+ here from the early inliner (if it runs) is cyclic cases.
+ Ideally when processing callees we stop inlining at the
+ entry of cycles, possibly cloning that entry point and
+ try to flatten itself turning it into a self-recursive
+ function. */
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
{
- struct cgraph_edge *e, *next;
-
- node = order[i];
-
- /* Handle nodes to be flattened, but don't update overall unit
- size. */
- if (lookup_attribute ("flatten",
- DECL_ATTRIBUTES (node->decl)) != NULL)
- {
- if (dump_file)
- fprintf (dump_file,
- "Flattening %s\n", cgraph_node_name (node));
- cgraph_decide_inlining_incrementally (node, INLINE_ALL, 0);
- }
-
- if (!node->local.disregard_inline_limits)
- continue;
if (dump_file)
fprintf (dump_file,
- "\nConsidering %s size:%i (always inline)\n",
- cgraph_node_name (node), node->global.size);
- old_size = overall_size;
- for (e = node->callers; e; e = next)
- {
- next = e->next_caller;
- if (!e->inline_failed || e->call_stmt_cannot_inline_p)
- continue;
- if (cgraph_recursive_inlining_p (e->caller, e->callee,
- &e->inline_failed))
- continue;
- if (!tree_can_inline_p (e))
- continue;
- if (cgraph_mark_inline_edge (e, true, NULL))
- redo_always_inline = true;
- if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has size %i.\n",
- cgraph_node_name (e->caller),
- e->caller->global.size);
- }
- /* Inlining self recursive function might introduce new calls to
- themselves we didn't see in the loop above. Fill in the proper
- reason why inline failed. */
- for (e = node->callers; e; e = e->next_caller)
- if (e->inline_failed)
- e->inline_failed = CIF_RECURSIVE_INLINING;
- if (dump_file)
- fprintf (dump_file,
- " Inlined for a net change of %+i size.\n",
- overall_size - old_size);
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
}
}
if (cgraph_check_inline_limits (node->callers->caller, node,
&reason, false))
{
+ struct cgraph_node *caller = node->callers->caller;
cgraph_mark_inline (node->callers);
if (dump_file)
fprintf (dump_file,
" Inlined into %s which now has %i size"
" for a net change of %+i size.\n",
- cgraph_node_name (node->callers->caller),
- node->callers->caller->global.size,
+ cgraph_node_name (caller),
+ caller->global.size,
overall_size - old_size);
}
else
/* Free ipa-prop structures if they are no longer needed. */
if (flag_indirect_inlining)
- free_all_ipa_structures_after_iinln ();
+ ipa_free_all_structures_after_iinln ();
if (dump_file)
fprintf (dump_file,
return 0;
}
-/* Try to inline edge E from incremental inliner. MODE specifies mode
- of inliner.
-
- We are detecting cycles by storing mode of inliner into cgraph_node last
- time we visited it in the recursion. In general when mode is set, we have
- recursive inlining, but as an special case, we want to try harder inline
- ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
- flatten, b being always inline. Flattening 'a' will collapse
- a->b->c before hitting cycle. To accommodate always inline, we however
- need to inline a->b->c->b.
-
- So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
- stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
-static bool
-try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth)
-{
- struct cgraph_node *callee = e->callee;
- enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux;
- bool always_inline = e->callee->local.disregard_inline_limits;
- bool inlined = false;
-
- /* We've hit cycle? */
- if (callee_mode)
- {
- /* It is first time we see it and we are not in ALWAY_INLINE only
- mode yet. and the function in question is always_inline. */
- if (always_inline && mode != INLINE_ALWAYS_INLINE)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Hit cycle in %s, switching to always inline only.\n",
- cgraph_node_name (callee));
- }
- mode = INLINE_ALWAYS_INLINE;
- }
- /* Otherwise it is time to give up. */
- else
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining %s into %s to avoid cycle.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
- }
- e->inline_failed = (e->callee->local.disregard_inline_limits
- ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
- return false;
- }
- }
-
- callee->aux = (void *)(size_t) mode;
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, " Inlining %s into %s.\n",
- cgraph_node_name (e->callee),
- cgraph_node_name (e->caller));
- }
- if (e->inline_failed)
- {
- cgraph_mark_inline (e);
-
- /* In order to fully inline always_inline functions, we need to
- recurse here, since the inlined functions might not be processed by
- incremental inlining at all yet.
-
- Also flattening needs to be done recursively. */
-
- if (mode == INLINE_ALL || always_inline)
- cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1);
- inlined = true;
- }
- callee->aux = (void *)(size_t) callee_mode;
- return inlined;
-}
-
/* Return true when N is leaf function. Accept cheap (pure&const) builtins
in leaf functions. */
static bool
}
/* Decide on the inlining. We do so in the topological order to avoid
- expenses on updating data structures.
- DEPTH is depth of recursion, used only for debug output. */
+ expenses on updating data structures. */
static bool
cgraph_decide_inlining_incrementally (struct cgraph_node *node,
- enum inlining_mode mode,
- int depth)
+ enum inlining_mode mode)
{
struct cgraph_edge *e;
bool inlined = false;
cgraph_inline_failed_t failed_reason;
- enum inlining_mode old_mode;
#ifdef ENABLE_CHECKING
verify_cgraph_node (node);
#endif
- old_mode = (enum inlining_mode) (size_t)node->aux;
-
if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE
&& lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Flattening %s\n", cgraph_node_name (node));
- }
+ fprintf (dump_file, "Incrementally flattening %s\n",
+ cgraph_node_name (node));
mode = INLINE_ALL;
}
- node->aux = (void *)(size_t) mode;
-
/* First of all look for always inline functions. */
if (mode != INLINE_SIZE_NORECURSIVE)
for (e = node->callees; e; e = e->next_callee)
continue;
if (e->call_stmt_cannot_inline_p)
continue;
- /* When the edge is already inlined, we just need to recurse into
- it in order to fully flatten the leaves. */
- if (!e->inline_failed && mode == INLINE_ALL)
- {
- inlined |= try_inline (e, mode, depth);
- continue;
- }
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Considering to always inline inline candidate %s.\n",
- cgraph_node_name (e->callee));
- }
+ fprintf (dump_file,
+ "Considering to always inline inline candidate %s.\n",
+ cgraph_node_name (e->callee));
if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
if (!tree_can_inline_p (e))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: %s",
- cgraph_inline_failed_string (e->inline_failed));
- }
+ fprintf (dump_file,
+ "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
continue;
}
if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
!= gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ fprintf (dump_file, "Not inlining: SSA form does not match.\n");
continue;
}
if (!e->callee->analyzed)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: Function body no longer available.\n");
- }
+ fprintf (dump_file,
+ "Not inlining: Function body no longer available.\n");
continue;
}
- inlined |= try_inline (e, mode, depth);
+
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
}
/* Now do the automatic inlining. */
- if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE)
- for (e = node->callees; e; e = e->next_callee)
- {
- int allowed_growth = 0;
- if (!e->callee->local.inlinable
- || !e->inline_failed
- || e->callee->local.disregard_inline_limits)
- continue;
- if (dump_file)
- fprintf (dump_file, "Considering inline candidate %s.\n",
- cgraph_node_name (e->callee));
- if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE
+ /* Never inline regular functions into always-inline functions
+ during incremental inlining. */
+ && !node->local.disregard_inline_limits)
+ {
+ bitmap visited = BITMAP_ALLOC (NULL);
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ int allowed_growth = 0;
+ if (!e->callee->local.inlinable
+ || !e->inline_failed
+ || e->callee->local.disregard_inline_limits)
continue;
- }
- if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
- != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ /* We are inlining a function to all call-sites in node
+ or to none. So visit each candidate only once. */
+ if (!bitmap_set_bit (visited, e->callee->uid))
continue;
- }
-
- if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
- && optimize_function_for_speed_p (cfun))
- allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+ if (dump_file)
+ fprintf (dump_file, "Considering inline candidate %s.\n",
+ cgraph_node_name (e->callee));
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ continue;
+ }
+ if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+ != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining: SSA form does not match.\n");
+ continue;
+ }
- /* When the function body would grow and inlining the function won't
- eliminate the need for offline copy of the function, don't inline.
- */
- if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
- || (!flag_inline_functions
- && !DECL_DECLARED_INLINE_P (e->callee->decl)))
- && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
- > e->caller->global.size + allowed_growth)
- && cgraph_estimate_growth (e->callee) > allowed_growth)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
+ && optimize_function_for_speed_p (cfun))
+ allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+
+ /* When the function body would grow and inlining the function
+ won't eliminate the need for offline copy of the function,
+ don't inline. */
+ if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
+ || (!flag_inline_functions
+ && !DECL_DECLARED_INLINE_P (e->callee->decl)))
+ && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+ > e->caller->global.size + allowed_growth)
+ && cgraph_estimate_growth (e->callee) > allowed_growth)
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: code size would grow by %i.\n",
cgraph_estimate_size_after_inlining (1, e->caller,
e->callee)
- e->caller->global.size);
- }
- continue;
- }
- if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
- false)
- || e->call_stmt_cannot_inline_p)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+ false)
+ || e->call_stmt_cannot_inline_p)
+ {
+ if (dump_file)
fprintf (dump_file, "Not inlining: %s.\n",
cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (!e->callee->analyzed)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: Function body no longer available.\n");
- }
- continue;
- }
- if (!tree_can_inline_p (e))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
+ continue;
+ }
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
fprintf (dump_file,
"Not inlining: %s.",
- cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (cgraph_default_inline_p (e->callee, &failed_reason))
- inlined |= try_inline (e, mode, depth);
- }
- node->aux = (void *)(size_t) old_mode;
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
+ if (cgraph_default_inline_p (e->callee, &failed_reason))
+ {
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
+ }
+ }
+ BITMAP_FREE (visited);
+ }
return inlined;
}
if (sorrycount || errorcount)
return 0;
- while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
- && cgraph_decide_inlining_incrementally (node,
- iterations
- ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0))
+
+ if (!optimize
+ || flag_no_inline
+ || !flag_early_inlining)
{
+ /* When not optimizing or not inlining inline only always-inline
+ functions. */
+ cgraph_decide_inlining_incrementally (node, INLINE_ALWAYS_INLINE);
timevar_push (TV_INTEGRATION);
todo |= optimize_inline_calls (current_function_decl);
- iterations++;
timevar_pop (TV_INTEGRATION);
}
- if (dump_file)
- fprintf (dump_file, "Iterations: %i\n", iterations);
+ else
+ {
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ timevar_pop (TV_INTEGRATION);
+ }
+ /* We iterate incremental inlining to get trivial cases of indirect
+ inlining. */
+ while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+ && cgraph_decide_inlining_incrementally (node,
+ iterations
+ ? INLINE_SIZE_NORECURSIVE
+ : INLINE_SIZE))
+ {
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ iterations++;
+ timevar_pop (TV_INTEGRATION);
+ }
+ if (dump_file)
+ fprintf (dump_file, "Iterations: %i\n", iterations);
+ }
+
cfun->always_inline_functions_inlined = true;
- return todo;
-}
-/* When inlining shall be performed. */
-static bool
-cgraph_gate_early_inlining (void)
-{
- return flag_early_inlining;
+ return todo;
}
struct gimple_opt_pass pass_early_inline =
{
GIMPLE_PASS,
"einline", /* name */
- cgraph_gate_early_inlining, /* gate */
+ NULL, /* gate */
cgraph_early_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
static void
inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
{
- struct cgraph_edge *cs;
-
- if (!flag_ipa_cp)
- {
- ipa_initialize_node_params (node);
- ipa_detect_param_modifications (node);
- }
+ ipa_initialize_node_params (node);
+ ipa_detect_param_modifications (node);
ipa_analyze_params_uses (node);
-
- if (!flag_ipa_cp)
- for (cs = node->callees; cs; cs = cs->next_callee)
- {
- ipa_count_arguments (cs);
- ipa_compute_jump_functions (cs);
- }
+ ipa_compute_jump_functions (node);
if (dump_file)
{
active, we don't need to write them twice. */
static void
-inline_write_summary (cgraph_node_set set)
+inline_write_summary (cgraph_node_set set,
+ varpool_node_set vset ATTRIBUTE_UNUSED)
{
if (flag_indirect_inlining && !flag_ipa_cp)
ipa_prop_write_jump_functions (set);
}
+/* When to run IPA inlining. Inlining of always-inline functions
+ happens during early inlining. */
+
+static bool
+gate_cgraph_decide_inlining (void)
+{
+ /* ??? We'd like to skip this if not optimizing or not inlining as
+ all always-inline functions have been processed by early
+ inlining already. But this at least breaks EH with C++ as
+ we need to unconditionally run fixup_cfg even at -O0.
+ So leave it on unconditionally for now. */
+ return 1;
+}
+
struct ipa_opt_pass_d pass_ipa_inline =
{
{
IPA_PASS,
"inline", /* name */
- NULL, /* gate */
+ gate_cgraph_decide_inlining, /* gate */
cgraph_decide_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
0, /* properties_destroyed */
TODO_remove_functions, /* todo_flags_finish */
TODO_dump_cgraph | TODO_dump_func
- | TODO_remove_functions /* todo_flags_finish */
+ | TODO_remove_functions | TODO_ggc_collect /* todo_flags_finish */
},
inline_generate_summary, /* generate_summary */
inline_write_summary, /* write_summary */
inline_read_summary, /* read_summary */
- NULL, /* function_read_summary */
- lto_ipa_fixup_call_notes, /* stmt_fixup */
+ NULL, /* write_optimization_summary */
+ NULL, /* read_optimization_summary */
+ NULL, /* stmt_fixup */
0, /* TODOs */
inline_transform, /* function_transform */
NULL, /* variable_transform */
OSDN Git Service
