void
cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, bool update_original)
{
- struct cgraph_node *n;
-
- /* We may eliminate the need for out-of-line copy to be output. In that
- case just go ahead and re-use it. */
- if (!e->callee->callers->next_caller
- && (!e->callee->needed || DECL_EXTERNAL (e->callee->decl))
- && duplicate
- && flag_unit_at_a_time)
+ HOST_WIDE_INT peak;
+ if (duplicate)
{
- gcc_assert (!e->callee->global.inlined_to);
- if (!DECL_EXTERNAL (e->callee->decl))
- overall_insns -= e->callee->global.insns, nfunctions_inlined++;
- duplicate = 0;
- }
- else if (duplicate)
- {
- n = cgraph_clone_node (e->callee, e->count, e->loop_nest, update_original);
- cgraph_redirect_edge_callee (e, n);
+ /* We may eliminate the need for out-of-line copy to be output.
+ In that case just go ahead and re-use it. */
+ if (!e->callee->callers->next_caller
+ && !e->callee->needed
+ && flag_unit_at_a_time)
+ {
+ gcc_assert (!e->callee->global.inlined_to);
+ if (DECL_SAVED_TREE (e->callee->decl))
+ overall_insns -= e->callee->global.insns, nfunctions_inlined++;
+ duplicate = false;
+ }
+ else
+ {
+ struct cgraph_node *n;
+ n = cgraph_clone_node (e->callee, e->count, e->loop_nest,
+ update_original);
+ cgraph_redirect_edge_callee (e, n);
+ }
}
if (e->caller->global.inlined_to)
e->callee->global.inlined_to = e->caller->global.inlined_to;
else
e->callee->global.inlined_to = e->caller;
+ e->callee->global.stack_frame_offset
+ = e->caller->global.stack_frame_offset + e->caller->local.estimated_self_stack_size;
+ peak = e->callee->global.stack_frame_offset + e->callee->local.estimated_self_stack_size;
+ if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
+ e->callee->global.inlined_to->global.estimated_stack_size = peak;
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
int old_insns = 0, new_insns = 0;
struct cgraph_node *to = NULL, *what;
+ if (e->callee->inline_decl)
+ cgraph_redirect_edge_callee (e, cgraph_node (e->callee->inline_decl));
+
gcc_assert (e->inline_failed);
e->inline_failed = NULL;
}
/* Return false when inlining WHAT into TO is not good idea
- as it would cause too large growth of function bodies. */
+ as it would cause too large growth of function bodies.
+ When ONE_ONLY is true, assume that only one call site is going
+ to be inlined, otherwise figure out how many call sites in
+ TO calls WHAT and verify that all can be inlined.
+ */
static bool
cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what,
- const char **reason)
+ const char **reason, bool one_only)
{
int times = 0;
struct cgraph_edge *e;
int newsize;
int limit;
+ HOST_WIDE_INT stack_size_limit, inlined_stack;
+
+ if (one_only)
+ times = 1;
+ else
+ for (e = to->callees; e; e = e->next_callee)
+ if (e->callee == what)
+ times++;
if (to->global.inlined_to)
to = to->global.inlined_to;
- for (e = to->callees; e; e = e->next_callee)
- if (e->callee == what)
- times++;
-
/* When inlining large function body called once into small function,
take the inlined function as base for limiting the growth. */
if (to->local.self_insns > what->local.self_insns)
limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100;
+ /* Check the size after inlining against the function limits. But allow
+ the function to shrink if it went over the limits by forced inlining. */
newsize = cgraph_estimate_size_after_inlining (times, to, what);
- if (newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)
+ if (newsize >= to->global.insns
+ && newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)
&& newsize > limit)
{
if (reason)
*reason = N_("--param large-function-growth limit reached");
return false;
}
+
+ stack_size_limit = to->local.estimated_self_stack_size;
+
+ stack_size_limit += stack_size_limit * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH) / 100;
+
+ inlined_stack = (to->global.stack_frame_offset
+ + to->local.estimated_self_stack_size
+ + what->global.estimated_stack_size);
+ if (inlined_stack > stack_size_limit
+ && inlined_stack > PARAM_VALUE (PARAM_LARGE_STACK_FRAME))
+ {
+ if (reason)
+ *reason = N_("--param large-stack-frame-growth limit reached");
+ return false;
+ }
return true;
}
bool
cgraph_default_inline_p (struct cgraph_node *n, const char **reason)
{
- if (!DECL_INLINE (n->decl))
+ tree decl = n->decl;
+
+ if (n->inline_decl)
+ decl = n->inline_decl;
+ if (!DECL_INLINE (decl))
{
if (reason)
*reason = N_("function not inlinable");
return false;
}
- if (!DECL_SAVED_TREE (n->decl))
+ if (!DECL_STRUCT_FUNCTION (decl)->cfg)
{
if (reason)
*reason = N_("function body not available");
return false;
}
- if (DECL_DECLARED_INLINE_P (n->decl))
+ if (DECL_DECLARED_INLINE_P (decl))
{
if (n->global.insns >= MAX_INLINE_INSNS_SINGLE)
{
bitmap updated_nodes)
{
struct cgraph_edge *edge;
+ const char *failed_reason;
if (!node->local.inlinable || node->local.disregard_inline_limits
|| node->global.inlined_to)
bitmap_set_bit (updated_nodes, node->uid);
node->global.estimated_growth = INT_MIN;
+ if (!node->local.inlinable)
+ return;
+ /* Prune out edges we won't inline into anymore. */
+ if (!cgraph_default_inline_p (node, &failed_reason))
+ {
+ for (edge = node->callers; edge; edge = edge->next_caller)
+ if (edge->aux)
+ {
+ fibheap_delete_node (heap, edge->aux);
+ edge->aux = NULL;
+ if (edge->inline_failed)
+ edge->inline_failed = failed_reason;
+ }
+ return;
+ }
+
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
{
node->aux = 0;
}
-/* Leafify the cgraph node. We have to be careful in recursing
+/* Flatten the cgraph node. We have to be careful in recursing
as to not run endlessly in circles of the callgraph.
We do so by using a hashtab of cycle entering nodes as generated
by cgraph_find_cycles. */
int probability = PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY);
fibheap_t heap;
struct cgraph_edge *e;
- struct cgraph_node *master_clone;
+ struct cgraph_node *master_clone, *next;
int depth = 0;
int n = 0;
into master clone gets queued just before master clone so we don't
need recursion. */
for (node = cgraph_nodes; node != master_clone;
- node = node->next)
- if (node->global.inlined_to == master_clone)
- cgraph_remove_node (node);
+ node = next)
+ {
+ next = node->next;
+ if (node->global.inlined_to == master_clone)
+ cgraph_remove_node (node);
+ }
cgraph_remove_node (master_clone);
/* FIXME: Recursive inlining actually reduces number of calls of the
function. At this place we should probably walk the function and
inline clones and compensate the counts accordingly. This probably
doesn't matter much in practice. */
- return true;
+ return n > 0;
}
/* Set inline_failed for all callers of given function to REASON. */
if (dump_file)
{
fprintf (dump_file,
- "\nConsidering %s with %i insns to be inlined into %s\n"
+ "\nConsidering %s with %i insns\n",
+ cgraph_node_name (edge->callee),
+ edge->callee->global.insns);
+ fprintf (dump_file,
+ " to be inlined into %s\n"
" Estimated growth after inlined into all callees is %+i insns.\n"
" Estimated badness is %i.\n",
- cgraph_node_name (edge->callee),
- edge->callee->global.insns,
cgraph_node_name (edge->caller),
cgraph_estimate_growth (edge->callee),
cgraph_edge_badness (edge));
{
struct cgraph_node *callee;
if (!cgraph_check_inline_limits (edge->caller, edge->callee,
- &edge->inline_failed))
+ &edge->inline_failed, true))
{
if (dump_file)
fprintf (dump_file, " Not inlining into %s:%s.\n",
bitmap_clear (updated_nodes);
if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has %i insns.\n",
- cgraph_node_name (edge->caller),
- edge->caller->global.insns);
- if (dump_file)
- fprintf (dump_file,
- " Inlined for a net change of %+i insns.\n",
- overall_insns - old_insns);
+ {
+ fprintf (dump_file,
+ " Inlined into %s which now has %i insns,"
+ "net change of %+i insns.\n",
+ cgraph_node_name (edge->caller),
+ edge->caller->global.insns,
+ overall_insns - old_insns);
+ }
}
while ((edge = fibheap_extract_min (heap)) != NULL)
{
/* Decide on the inlining. We do so in the topological order to avoid
expenses on updating data structures. */
-static void
+static unsigned int
cgraph_decide_inlining (void)
{
struct cgraph_node *node;
int nnodes;
struct cgraph_node **order =
- xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
+ XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
int old_insns = 0;
int i;
- timevar_push (TV_INLINE_HEURISTICS);
max_count = 0;
for (node = cgraph_nodes; node; node = node->next)
- {
- struct cgraph_edge *e;
- initial_insns += node->local.self_insns;
- for (e = node->callees; e; e = e->next_callee)
- if (max_count < e->count)
- max_count = e->count;
- }
+ if (node->analyzed && (node->needed || node->reachable))
+ {
+ struct cgraph_edge *e;
+
+ /* At the moment, no IPA passes change function bodies before inlining.
+ Save some time by not recomputing function body sizes if early inlining
+ already did so. */
+ if (!flag_early_inlining)
+ node->local.self_insns = node->global.insns
+ = estimate_num_insns (node->decl);
+
+ initial_insns += node->local.self_insns;
+ gcc_assert (node->local.self_insns == node->global.insns);
+ for (e = node->callees; e; e = e->next_callee)
+ if (max_count < e->count)
+ max_count = e->count;
+ }
overall_insns = initial_insns;
gcc_assert (!max_count || (profile_info && flag_branch_probabilities));
- max_insns = ((HOST_WIDEST_INT) overall_insns
+ max_insns = overall_insns;
+ if (max_insns < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS))
+ max_insns = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS);
+
+ max_insns = ((HOST_WIDEST_INT) max_insns
* (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH)) / 100);
nnodes = cgraph_postorder (order);
htab_t cycles;
if (dump_file)
fprintf (dump_file,
- "Leafifying %s\n", cgraph_node_name (node));
+ "Flattening %s\n", cgraph_node_name (node));
cycles = htab_create (7, htab_hash_pointer, htab_eq_pointer, NULL);
cgraph_find_cycles (node, cycles);
cgraph_flatten_node (node, cycles);
&& node->local.inlinable && node->callers->inline_failed
&& !DECL_EXTERNAL (node->decl) && !DECL_COMDAT (node->decl))
{
- bool ok = true;
- struct cgraph_node *node1;
-
- /* Verify that we won't duplicate the caller. */
- for (node1 = node->callers->caller;
- node1->callers && !node1->callers->inline_failed
- && ok; node1 = node1->callers->caller)
- if (node1->callers->next_caller || node1->needed)
- ok = false;
- if (ok)
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "\nConsidering %s %i insns.\n",
+ cgraph_node_name (node), node->global.insns);
+ fprintf (dump_file,
+ " Called once from %s %i insns.\n",
+ cgraph_node_name (node->callers->caller),
+ node->callers->caller->global.insns);
+ }
+
+ old_insns = overall_insns;
+
+ if (cgraph_check_inline_limits (node->callers->caller, node,
+ NULL, false))
{
+ cgraph_mark_inline (node->callers);
if (dump_file)
fprintf (dump_file,
- "\nConsidering %s %i insns.\n"
- " Called once from %s %i insns.\n",
- cgraph_node_name (node), node->global.insns,
+ " Inlined into %s which now has %i insns"
+ " for a net change of %+i insns.\n",
cgraph_node_name (node->callers->caller),
- node->callers->caller->global.insns);
-
- old_insns = overall_insns;
-
- if (cgraph_check_inline_limits (node->callers->caller, node,
- NULL))
- {
- cgraph_mark_inline (node->callers);
- if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has %i insns"
- " for a net change of %+i insns.\n",
- cgraph_node_name (node->callers->caller),
- node->callers->caller->global.insns,
- overall_insns - old_insns);
- }
- else
- {
- if (dump_file)
- fprintf (dump_file,
- " Inline limit reached, not inlined.\n");
- }
+ node->callers->caller->global.insns,
+ overall_insns - old_insns);
+ }
+ else
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ " Inline limit reached, not inlined.\n");
}
}
}
ncalls_inlined, nfunctions_inlined, initial_insns,
overall_insns);
free (order);
- timevar_pop (TV_INLINE_HEURISTICS);
+ return 0;
}
/* Decide on the inlining. We do so in the topological order to avoid
&& !cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed)
/* ??? It is possible that renaming variable removed the function body
in duplicate_decls. See gcc.c-torture/compile/20011119-2.c */
- && DECL_SAVED_TREE (e->callee->decl))
+ && (DECL_SAVED_TREE (e->callee->decl) || e->callee->inline_decl))
{
if (dump_file && early)
- fprintf (dump_file, " Early inlining %s into %s\n",
- cgraph_node_name (e->callee), cgraph_node_name (node));
+ {
+ fprintf (dump_file, " Early inlining %s",
+ cgraph_node_name (e->callee));
+ fprintf (dump_file, " into %s\n", cgraph_node_name (node));
+ }
cgraph_mark_inline (e);
inlined = true;
}
&& !e->callee->local.disregard_inline_limits
&& !cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed)
&& (!early
- || (cgraph_estimate_size_after_inlining (1, e->caller, node)
+ || (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
<= e->caller->global.insns))
- && cgraph_check_inline_limits (node, e->callee, &e->inline_failed)
- && DECL_SAVED_TREE (e->callee->decl))
+ && cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+ false)
+ && (DECL_SAVED_TREE (e->callee->decl) || e->callee->inline_decl))
{
if (cgraph_default_inline_p (e->callee, &failed_reason))
{
if (dump_file && early)
- fprintf (dump_file, " Early inlining %s into %s\n",
- cgraph_node_name (e->callee), cgraph_node_name (node));
+ {
+ fprintf (dump_file, " Early inlining %s",
+ cgraph_node_name (e->callee));
+ fprintf (dump_file, " into %s\n", cgraph_node_name (node));
+ }
cgraph_mark_inline (e);
inlined = true;
}
}
if (early && inlined)
{
+ timevar_push (TV_INTEGRATION);
push_cfun (DECL_STRUCT_FUNCTION (node->decl));
tree_register_cfg_hooks ();
current_function_decl = node->decl;
node->local.self_insns = node->global.insns;
current_function_decl = NULL;
pop_cfun ();
+ timevar_pop (TV_INTEGRATION);
}
return inlined;
}
NULL, /* sub */
NULL, /* next */
0, /* static_pass_number */
- TV_INTEGRATION, /* tv_id */
+ TV_INLINE_HEURISTICS, /* tv_id */
0, /* properties_required */
PROP_cfg, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
- TODO_dump_cgraph | TODO_dump_func, /* todo_flags_finish */
+ TODO_dump_cgraph | TODO_dump_func
+ | TODO_remove_functions, /* todo_flags_finish */
0 /* letter */
};
+/* Because inlining might remove no-longer reachable nodes, we need to
+ keep the array visible to garbage collector to avoid reading collected
+ out nodes. */
+static int nnodes;
+static GTY ((length ("nnodes"))) struct cgraph_node **order;
+
/* Do inlining of small functions. Doing so early helps profiling and other
passes to be somewhat more effective and avoids some code duplication in
later real inlining pass for testcases with very many function calls. */
-static void
+static unsigned int
cgraph_early_inlining (void)
{
struct cgraph_node *node;
- int nnodes;
- struct cgraph_node **order =
- xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
int i;
if (sorrycount || errorcount)
- return;
+ return 0;
#ifdef ENABLE_CHECKING
for (node = cgraph_nodes; node; node = node->next)
gcc_assert (!node->aux);
#endif
+ order = ggc_alloc (sizeof (*order) * cgraph_n_nodes);
nnodes = cgraph_postorder (order);
for (i = nnodes - 1; i >= 0; i--)
{
node = order[i];
+ if (node->analyzed && (node->needed || node->reachable))
+ node->local.self_insns = node->global.insns
+ = estimate_num_insns (node->decl);
+ }
+ for (i = nnodes - 1; i >= 0; i--)
+ {
+ node = order[i];
if (node->analyzed && node->local.inlinable
&& (node->needed || node->reachable)
&& node->callers)
- cgraph_decide_inlining_incrementally (node, true);
+ {
+ if (cgraph_decide_inlining_incrementally (node, true))
+ ggc_collect ();
+ }
}
- cgraph_remove_unreachable_nodes (true, dump_file);
#ifdef ENABLE_CHECKING
for (node = cgraph_nodes; node; node = node->next)
gcc_assert (!node->global.inlined_to);
#endif
- free (order);
+ ggc_free (order);
+ order = NULL;
+ nnodes = 0;
+ return 0;
}
/* When inlining shall be performed. */
NULL, /* sub */
NULL, /* next */
0, /* static_pass_number */
- TV_INTEGRATION, /* tv_id */
+ TV_INLINE_HEURISTICS, /* tv_id */
0, /* properties_required */
PROP_cfg, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
- TODO_dump_cgraph | TODO_dump_func, /* todo_flags_finish */
+ TODO_dump_cgraph | TODO_dump_func
+ | TODO_remove_functions, /* todo_flags_finish */
0 /* letter */
};
+
+#include "gt-ipa-inline.h"
OSDN Git Service
