#include "ipa-prop.h"
#include "ipa-inline.h"
#include "tree-inline.h"
+#include "tree-pass.h"
int ncalls_inlined;
int nfunctions_inlined;
-/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
- by NEST. */
+/* Scale frequency of NODE edges by FREQ_SCALE. */
static void
update_noncloned_frequencies (struct cgraph_node *node,
- int freq_scale, int nest)
+ int freq_scale)
{
struct cgraph_edge *e;
freq_scale = 1;
for (e = node->callees; e; e = e->next_callee)
{
- e->loop_nest += nest;
e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
if (e->frequency > CGRAPH_FREQ_MAX)
e->frequency = CGRAPH_FREQ_MAX;
if (!e->inline_failed)
- update_noncloned_frequencies (e->callee, freq_scale, nest);
+ update_noncloned_frequencies (e->callee, freq_scale);
}
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ {
+ e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+ if (e->frequency > CGRAPH_FREQ_MAX)
+ e->frequency = CGRAPH_FREQ_MAX;
+ }
+}
+
+/* We removed or are going to remove the last call to NODE.
+ Return true if we can and want proactively remove the NODE now.
+ This is important to do, since we want inliner to know when offline
+ copy of function was removed. */
+
+static bool
+can_remove_node_now_p_1 (struct cgraph_node *node)
+{
+ /* FIXME: When address is taken of DECL_EXTERNAL function we still
+ can remove its offline copy, but we would need to keep unanalyzed node in
+ the callgraph so references can point to it. */
+ return (!node->address_taken
+ && !ipa_ref_has_aliases_p (&node->ref_list)
+ && cgraph_can_remove_if_no_direct_calls_p (node)
+ /* Inlining might enable more devirtualizing, so we want to remove
+ those only after all devirtualizable virtual calls are processed.
+ Lacking may edges in callgraph we just preserve them post
+ inlining. */
+ && (!DECL_VIRTUAL_P (node->decl)
+ || (!DECL_COMDAT (node->decl)
+ && !DECL_EXTERNAL (node->decl)))
+ /* During early inlining some unanalyzed cgraph nodes might be in the
+ callgraph and they might reffer the function in question. */
+ && !cgraph_new_nodes);
+}
+
+/* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
+ Verify that the NODE can be removed from unit and if it is contained in comdat
+ group that the whole comdat group is removable. */
+
+static bool
+can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
+{
+ struct cgraph_node *next;
+ if (!can_remove_node_now_p_1 (node))
+ return false;
+
+ /* When we see same comdat group, we need to be sure that all
+ items can be removed. */
+ if (!node->same_comdat_group)
+ return true;
+ for (next = node->same_comdat_group;
+ next != node; next = next->same_comdat_group)
+ if ((next->callers && next->callers != e)
+ || !can_remove_node_now_p_1 (next))
+ return false;
+ return true;
}
clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
bool update_original, int *overall_size)
{
- HOST_WIDE_INT peak;
- struct inline_summary *caller_info, *callee_info;
-
if (duplicate)
{
/* We may eliminate the need for out-of-line copy to be output.
/* Recursive inlining never wants the master clone to
be overwritten. */
&& update_original
- /* FIXME: When address is taken of DECL_EXTERNAL function we still
- can remove its offline copy, but we would need to keep unanalyzed
- node in the callgraph so references can point to it. */
- && !e->callee->address_taken
- && cgraph_can_remove_if_no_direct_calls_p (e->callee)
- /* Inlining might enable more devirtualizing, so we want to remove
- those only after all devirtualizable virtual calls are processed.
- Lacking may edges in callgraph we just preserve them post
- inlining. */
- && (!DECL_VIRTUAL_P (e->callee->decl)
- || (!DECL_COMDAT (e->callee->decl)
- && !DECL_EXTERNAL (e->callee->decl)))
- /* Don't reuse if more than one function shares a comdat group.
- If the other function(s) are needed, we need to emit even
- this function out of line. */
- && !e->callee->same_comdat_group
- /* During early inlining some unanalyzed cgraph nodes might be in the
- callgraph and they might reffer the function in question. */
- && !cgraph_new_nodes)
+ && can_remove_node_now_p (e->callee, e))
{
+ /* TODO: When callee is in a comdat group, we could remove all of it,
+ including all inline clones inlined into it. That would however
+ need small function inlining to register edge removal hook to
+ maintain the priority queue.
+
+ For now we keep the ohter functions in the group in program until
+ cgraph_remove_unreachable_functions gets rid of them. */
gcc_assert (!e->callee->global.inlined_to);
if (e->callee->analyzed && !DECL_EXTERNAL (e->callee->decl))
{
}
duplicate = false;
e->callee->local.externally_visible = false;
- update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
+ update_noncloned_frequencies (e->callee, e->frequency);
}
else
{
struct cgraph_node *n;
n = cgraph_clone_node (e->callee, e->callee->decl,
- e->count, e->frequency, e->loop_nest,
- update_original, NULL);
+ e->count, e->frequency,
+ update_original, NULL, true);
cgraph_redirect_edge_callee (e, n);
}
}
- callee_info = inline_summary (e->callee);
- caller_info = inline_summary (e->caller);
-
if (e->caller->global.inlined_to)
e->callee->global.inlined_to = e->caller->global.inlined_to;
else
e->callee->global.inlined_to = e->caller;
- callee_info->stack_frame_offset
- = caller_info->stack_frame_offset
- + caller_info->estimated_self_stack_size;
- peak = callee_info->stack_frame_offset
- + callee_info->estimated_self_stack_size;
- if (inline_summary (e->callee->global.inlined_to)->estimated_stack_size
- < peak)
- inline_summary (e->callee->global.inlined_to)->estimated_stack_size = peak;
- cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
int old_size = 0, new_size = 0;
struct cgraph_node *to = NULL;
struct cgraph_edge *curr = e;
- struct inline_summary *info;
+ struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
/* Don't inline inlined edges. */
gcc_assert (e->inline_failed);
/* Don't even think of inlining inline clone. */
- gcc_assert (!e->callee->global.inlined_to);
+ gcc_assert (!callee->global.inlined_to);
e->inline_failed = CIF_OK;
- DECL_POSSIBLY_INLINED (e->callee->decl) = true;
+ DECL_POSSIBLY_INLINED (callee->decl) = true;
- clone_inlined_nodes (e, true, update_original, overall_size);
+ to = e->caller;
+ if (to->global.inlined_to)
+ to = to->global.inlined_to;
- /* Now update size of caller and all functions caller is inlined into. */
- for (;e && !e->inline_failed; e = e->caller->callers)
+ /* If aliases are involved, redirect edge to the actual destination and
+ possibly remove the aliases. */
+ if (e->callee != callee)
{
- to = e->caller;
- info = inline_summary (to);
- old_size = info->size;
- new_size = estimate_size_after_inlining (to, curr);
- info->size = new_size;
- info->time = estimate_time_after_inlining (to, curr);
+ struct cgraph_node *alias = e->callee, *next_alias;
+ cgraph_redirect_edge_callee (e, callee);
+ while (alias && alias != callee)
+ {
+ if (!alias->callers
+ && can_remove_node_now_p (alias, e))
+ {
+ next_alias = cgraph_alias_aliased_node (alias);
+ cgraph_remove_node (alias);
+ alias = next_alias;
+ }
+ else
+ break;
+ }
}
+
+ clone_inlined_nodes (e, true, update_original, overall_size);
+
gcc_assert (curr->callee->global.inlined_to == to);
- if (overall_size && new_size > old_size)
+
+ old_size = inline_summary (to)->size;
+ inline_merge_summary (e);
+ new_size = inline_summary (to)->size;
+ if (overall_size)
*overall_size += new_size - old_size;
ncalls_inlined++;
- if (flag_indirect_inlining && optimize)
+ /* This must happen after inline_merge_summary that rely on jump
+ functions of callee to not be updated. */
+ if (optimize)
return ipa_propagate_indirect_call_infos (curr, new_edges);
else
return false;
/* Copy the OLD_VERSION_NODE function tree to the new version. */
tree_function_versioning (node->decl, first_clone->decl, NULL, true, NULL,
- NULL, NULL);
+ false, NULL, NULL);
/* The function will be short lived and removed after we inline all the clones,
but make it internal so we won't confuse ourself. */
{
unsigned int todo = 0;
struct cgraph_edge *e;
- bool inline_p = false;
-
+
/* FIXME: Currently the pass manager is adding inline transform more than
once to some clones. This needs revisiting after WPA cleanups. */
if (cfun->after_inlining)
/* We might need the body of this function so that we can expand
it inline somewhere else. */
- if (cgraph_preserve_function_body_p (node->decl))
+ if (cgraph_preserve_function_body_p (node))
save_inline_function_body (node);
for (e = node->callees; e; e = e->next_callee)
- {
- cgraph_redirect_edge_call_stmt_to_callee (e);
- if (!e->inline_failed || warn_inline)
- inline_p = true;
- }
+ cgraph_redirect_edge_call_stmt_to_callee (e);
+
+ timevar_push (TV_INTEGRATION);
+ if (node->callees)
+ todo = optimize_inline_calls (current_function_decl);
+ timevar_pop (TV_INTEGRATION);
- if (inline_p)
- {
- timevar_push (TV_INTEGRATION);
- todo = optimize_inline_calls (current_function_decl);
- timevar_pop (TV_INTEGRATION);
- }
cfun->always_inline_functions_inlined = true;
cfun->after_inlining = true;
- return todo | execute_fixup_cfg ();
+ todo |= execute_fixup_cfg ();
+
+ if (!(todo & TODO_update_ssa_any))
+ /* Redirecting edges might lead to a need for vops to be recomputed. */
+ todo |= TODO_update_ssa_only_virtuals;
+
+ return todo;
}
OSDN Git Service
