return size;
}
+/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
+ by NEST. */
+
+static void
+update_noncloned_frequencies (struct cgraph_node *node,
+ int freq_scale, int nest)
+{
+ struct cgraph_edge *e;
+
+ /* We do not want to ignore high loop nest after freq drops to 0. */
+ if (!freq_scale)
+ freq_scale = 1;
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ e->loop_nest += nest;
+ e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+ if (e->frequency > CGRAPH_FREQ_MAX)
+ e->frequency = CGRAPH_FREQ_MAX;
+ if (!e->inline_failed)
+ update_noncloned_frequencies (e->callee, freq_scale, nest);
+ }
+}
+
/* E is expected to be an edge being inlined. Clone destination node of
the edge and redirect it to the new clone.
DUPLICATE is used for bookkeeping on whether we are actually creating new
In that case just go ahead and re-use it. */
if (!e->callee->callers->next_caller
&& cgraph_can_remove_if_no_direct_calls_p (e->callee)
+ /* Don't reuse if more than one function shares a comdat group.
+ If the other function(s) are needed, we need to emit even
+ this function out of line. */
+ && !e->callee->same_comdat_group
&& !cgraph_new_nodes)
{
gcc_assert (!e->callee->global.inlined_to);
}
duplicate = false;
e->callee->local.externally_visible = false;
+ update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
}
else
{
struct cgraph_node *n;
- n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+ n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
e->callee->global.inlined = true;
if (e->callee->callers->next_caller
- || !cgraph_can_remove_if_no_direct_calls_p (e->callee))
+ || !cgraph_can_remove_if_no_direct_calls_p (e->callee)
+ || e->callee->same_comdat_group)
duplicate = true;
cgraph_clone_inlined_nodes (e, true, update_original);
}
/* Return false when inlining WHAT into TO is not good idea
- as it would cause too large growth of function bodies.
+ as it would cause too large growth of function bodies.
When ONE_ONLY is true, assume that only one call site is going
to be inlined, otherwise figure out how many call sites in
TO calls WHAT and verify that all can be inlined.
badness = growth * 10000;
div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
/ (edge->callee->global.time + 1) + 1, 100);
-
+
/* Decrease badness if call is nested. */
/* Compress the range so we don't overflow. */
badness = cgraph_estimate_growth (edge->callee) * 256;
/* Decrease badness if call is nested. */
- if (badness > 0)
+ if (badness > 0)
badness >>= nest;
else
{
}
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" Performing recursive inlining on %s\n",
cgraph_node_name (node));
if (depth > max_depth)
{
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" maximal depth reached\n");
continue;
}
if (curr->count * 100 / node->count < probability)
{
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" Probability of edge is too small\n");
continue;
}
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
" Inlining call of depth %i", depth);
if (node->count)
{
fibheap_delete (heap);
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
"\n Inlined %i times, body grown from size %i to %i, time %i to %i\n", n,
master_clone->global.size, node->global.size,
master_clone->global.time, node->global.time);
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
"\nConsidering %s with %i size\n",
cgraph_node_name (edge->callee),
edge->callee->global.size);
- fprintf (dump_file,
+ fprintf (dump_file,
" to be inlined into %s in %s:%i\n"
" Estimated growth after inlined into all callees is %+i insns.\n"
" Estimated badness is %i, frequency %.2f.\n",
if (dump_file)
{
- fprintf (dump_file,
+ fprintf (dump_file,
" Inlined into %s which now has size %i and self time %i,"
"net change of %+i.\n",
cgraph_node_name (edge->caller),
if (e->inline_failed)
e->inline_failed = CIF_RECURSIVE_INLINING;
if (dump_file)
- fprintf (dump_file,
+ fprintf (dump_file,
" Inlined for a net change of %+i size.\n",
overall_size - old_size);
}
&& !DECL_EXTERNAL (node->decl)
&& !DECL_COMDAT (node->decl))
{
+ cgraph_inline_failed_t reason;
old_size = overall_size;
if (dump_file)
{
}
if (cgraph_check_inline_limits (node->callers->caller, node,
- NULL, false))
+ &reason, false))
{
cgraph_mark_inline (node->callers);
if (dump_file)
{
if (dump_file)
fprintf (dump_file,
- " Inline limit reached, not inlined.\n");
+ " Not inlining: %s.\n",
+ cgraph_inline_failed_string (reason));
}
}
}
return false;
}
}
-
+
callee->aux = (void *)(size_t) mode;
if (dump_file)
{
/* In order to fully inline always_inline functions, we need to
recurse here, since the inlined functions might not be processed by
- incremental inlining at all yet.
+ incremental inlining at all yet.
Also flattening needs to be done recursively. */
}
/* Decide on the inlining. We do so in the topological order to avoid
- expenses on updating data structures.
+ expenses on updating data structures.
DEPTH is depth of recursion, used only for debug output. */
static bool
}
/* Now do the automatic inlining. */
- if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE)
- for (e = node->callees; e; e = e->next_callee)
- {
- int allowed_growth = 0;
- if (!e->callee->local.inlinable
- || !e->inline_failed
- || e->callee->local.disregard_inline_limits)
- continue;
- if (dump_file)
- fprintf (dump_file, "Considering inline candidate %s.\n",
- cgraph_node_name (e->callee));
- if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ if (mode != INLINE_ALL && mode != INLINE_ALWAYS_INLINE
+ /* Never inline regular functions into always-inline functions
+ during incremental inlining. */
+ && !node->local.disregard_inline_limits)
+ {
+ bitmap visited = BITMAP_ALLOC (NULL);
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ int allowed_growth = 0;
+ if (!e->callee->local.inlinable
+ || !e->inline_failed
+ || e->callee->local.disregard_inline_limits)
continue;
- }
- if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
- != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ /* We are inlining a function to all call-sites in node
+ or to none. So visit each candidate only once. */
+ if (!bitmap_set_bit (visited, e->callee->uid))
continue;
- }
-
- if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
- && optimize_function_for_speed_p (cfun))
- allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+ if (dump_file)
+ fprintf (dump_file, "Considering inline candidate %s.\n",
+ cgraph_node_name (e->callee));
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ }
+ continue;
+ }
+ if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+ != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file,
+ "Not inlining: SSA form does not match.\n");
+ }
+ continue;
+ }
- /* When the function body would grow and inlining the function won't
- eliminate the need for offline copy of the function, don't inline.
- */
- if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
- || (!flag_inline_functions
- && !DECL_DECLARED_INLINE_P (e->callee->decl)))
- && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
- > e->caller->global.size + allowed_growth)
- && cgraph_estimate_growth (e->callee) > allowed_growth)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: code size would grow by %i.\n",
- cgraph_estimate_size_after_inlining (1, e->caller,
- e->callee)
- - e->caller->global.size);
- }
- continue;
- }
- if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
- false)
- || e->call_stmt_cannot_inline_p)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: %s.\n",
- cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (!e->callee->analyzed)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: Function body no longer available.\n");
- }
- continue;
- }
- if (!tree_can_inline_p (e))
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: %s.",
- cgraph_inline_failed_string (e->inline_failed));
- }
- continue;
- }
- if (cgraph_default_inline_p (e->callee, &failed_reason))
- inlined |= try_inline (e, mode, depth);
- }
+ if (cgraph_maybe_hot_edge_p (e) && leaf_node_p (e->callee)
+ && optimize_function_for_speed_p (cfun))
+ allowed_growth = PARAM_VALUE (PARAM_EARLY_INLINING_INSNS);
+
+ /* When the function body would grow and inlining the function
+ won't eliminate the need for offline copy of the function,
+ don't inline. */
+ if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
+ || (!flag_inline_functions
+ && !DECL_DECLARED_INLINE_P (e->callee->decl)))
+ && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+ > e->caller->global.size + allowed_growth)
+ && cgraph_estimate_growth (e->callee) > allowed_growth)
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file,
+ "Not inlining: code size would grow by %i.\n",
+ cgraph_estimate_size_after_inlining (1, e->caller,
+ e->callee)
+ - e->caller->global.size);
+ }
+ continue;
+ }
+ if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
+ false)
+ || e->call_stmt_cannot_inline_p)
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file, "Not inlining: %s.\n",
+ cgraph_inline_failed_string (e->inline_failed));
+ }
+ continue;
+ }
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file,
+ "Not inlining: Function body no longer available.\n");
+ }
+ continue;
+ }
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
+ {
+ indent_to (dump_file, depth);
+ fprintf (dump_file,
+ "Not inlining: %s.",
+ cgraph_inline_failed_string (e->inline_failed));
+ }
+ continue;
+ }
+ if (cgraph_default_inline_p (e->callee, &failed_reason))
+ inlined |= try_inline (e, mode, depth);
+ }
+ BITMAP_FREE (visited);
+ }
node->aux = (void *)(size_t) old_mode;
return inlined;
}
if (sorrycount || errorcount)
return 0;
- while (cgraph_decide_inlining_incrementally (node,
- iterations
- ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0)
- && iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS))
+ while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+ && cgraph_decide_inlining_incrementally (node,
+ iterations
+ ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0))
{
timevar_push (TV_INTEGRATION);
todo |= optimize_inline_calls (current_function_decl);
return flag_early_inlining;
}
-struct gimple_opt_pass pass_early_inline =
+struct gimple_opt_pass pass_early_inline =
{
{
GIMPLE_PASS,
/* IPA pass wrapper for early inlining pass. We need to run early inlining
before tree profiling so we have stand alone IPA pass for doing so. */
-struct simple_ipa_opt_pass pass_ipa_early_inline =
+struct simple_ipa_opt_pass pass_ipa_early_inline =
{
{
SIMPLE_IPA_PASS,
while (handled_component_p (inner_rhs)
|| TREE_CODE (inner_rhs) == ADDR_EXPR || TREE_CODE (inner_rhs) == INDIRECT_REF)
inner_rhs = TREE_OPERAND (inner_rhs, 0);
-
+
if (TREE_CODE (inner_rhs) == PARM_DECL
|| (TREE_CODE (inner_rhs) == SSA_NAME
node->global.stack_frame_offset = 0;
/* Can this function be inlined at all? */
- node->local.inlinable = tree_inlinable_function_p (current_function_decl);
+ node->local.inlinable = tree_inlinable_function_p (node->decl);
if (node->local.inlinable && !node->local.disregard_inline_limits)
node->local.disregard_inline_limits
- = DECL_DISREGARD_INLINE_LIMITS (current_function_decl);
+ = DECL_DISREGARD_INLINE_LIMITS (node->decl);
estimate_function_body_sizes (node);
/* Inlining characteristics are maintained by the cgraph_mark_inline. */
node->global.time = inline_summary (node)->self_time;
return 0;
}
-struct gimple_opt_pass pass_inline_parameters =
+struct gimple_opt_pass pass_inline_parameters =
{
{
GIMPLE_PASS,
for (node = cgraph_nodes; node; node = node->next)
if (node->analyzed)
analyze_function (node);
-
+
return;
}
unsigned int todo = 0;
struct cgraph_edge *e;
+ /* FIXME: Currently the passmanager is adding inline transform more than once to some
+ clones. This needs revisiting after WPA cleanups. */
+ if (cfun->after_inlining)
+ return 0;
+
/* We might need the body of this function so that we can expand
it inline somewhere else. */
if (cgraph_preserve_function_body_p (node->decl))
and inliner, so when ipa-cp is active, we don't need to write them
twice. */
-static void
+static void
inline_read_summary (void)
{
if (flag_indirect_inlining)
Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
active, we don't need to write them twice. */
-static void
+static void
inline_write_summary (cgraph_node_set set)
{
if (flag_indirect_inlining && !flag_ipa_cp)
inline_write_summary, /* write_summary */
inline_read_summary, /* read_summary */
NULL, /* function_read_summary */
+ lto_ipa_fixup_call_notes, /* stmt_fixup */
0, /* TODOs */
inline_transform, /* function_transform */
NULL, /* variable_transform */
OSDN Git Service
