There are three major parts of this file:
- cgraph_mark_inline implementation
+ cgraph_mark_inline_edge implementation
This function allows to mark given call inline and performs necessary
modifications of cgraph (production of the clones and updating overall
optimized allowing it to unfold abstraction penalty on C++ effectively and
cheaply.
- pass_ipa_early_inlining
-
- With profiling, the early inlining is also necessary to reduce
- instrumentation costs on program with high abstraction penalty (doing
- many redundant calls). This can't happen in parallel with early
- optimization and profile instrumentation, because we would end up
- re-instrumenting already instrumented function bodies we brought in via
- inlining.
-
- To avoid this, this pass is executed as IPA pass before profiling. It is
- simple wrapper to pass_early_inlining and ensures first inlining.
-
pass_ipa_inline
This is the main pass implementing simple greedy algorithm to do inlining
inlining of functions called once. The pass compute just so called inline
plan (representation of inlining to be done in callgraph) and unlike early
inlining it is not performing the inlining itself.
-
- pass_apply_inline
-
- This pass performs actual inlining according to pass_ipa_inline on given
- function. Possible the function body before inlining is saved when it is
- needed for further inlining later.
*/
#include "config.h"
#include "flags.h"
#include "cgraph.h"
#include "diagnostic.h"
+#include "gimple-pretty-print.h"
#include "timevar.h"
#include "params.h"
#include "fibheap.h"
return time;
}
-/* Estimate self time of the function after inlining WHAT into TO. */
+/* Estimate self size of the function after inlining WHAT into TO. */
-static int
-cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to,
+static inline int
+cgraph_estimate_size_after_inlining (struct cgraph_node *to,
struct cgraph_node *what)
{
- int size = (what->global.size - inline_summary (what)->size_inlining_benefit) * times + to->global.size;
+ int size = ((what->global.size - inline_summary (what)->size_inlining_benefit)
+ + to->global.size);
gcc_assert (size >= 0);
return size;
}
/* We may eliminate the need for out-of-line copy to be output.
In that case just go ahead and re-use it. */
if (!e->callee->callers->next_caller
+ /* Recursive inlining never wants the master clone to be overwritten. */
+ && update_original
+ /* FIXME: When address is taken of DECL_EXTERNAL function we still can remove its
+ offline copy, but we would need to keep unanalyzed node in the callgraph so
+ references can point to it. */
+ && !e->callee->address_taken
&& cgraph_can_remove_if_no_direct_calls_p (e->callee)
+ /* Inlining might enable more devirtualizing, so we want to remove
+ those only after all devirtualizable virtual calls are processed.
+ Lacking may edges in callgraph we just preserve them post
+ inlining. */
+ && (!DECL_VIRTUAL_P (e->callee->decl)
+ || (!DECL_COMDAT (e->callee->decl) && !DECL_EXTERNAL (e->callee->decl)))
/* Don't reuse if more than one function shares a comdat group.
If the other function(s) are needed, we need to emit even
this function out of line. */
&& !cgraph_new_nodes)
{
gcc_assert (!e->callee->global.inlined_to);
- if (e->callee->analyzed)
+ if (e->callee->analyzed && !DECL_EXTERNAL (e->callee->decl))
{
overall_size -= e->callee->global.size;
nfunctions_inlined++;
else
{
struct cgraph_node *n;
- n = cgraph_clone_node (e->callee, e->count, e->frequency, e->loop_nest,
+ n = cgraph_clone_node (e->callee, e->callee->decl,
+ e->count, e->frequency, e->loop_nest,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
struct cgraph_edge *curr = e;
int freq;
+ /* Don't inline inlined edges. */
gcc_assert (e->inline_failed);
+ /* Don't even think of inlining inline clone. */
+ gcc_assert (!e->callee->global.inlined_to);
+
e->inline_failed = CIF_OK;
DECL_POSSIBLY_INLINED (e->callee->decl) = true;
{
to = e->caller;
old_size = e->caller->global.size;
- new_size = cgraph_estimate_size_after_inlining (1, to, what);
+ new_size = cgraph_estimate_size_after_inlining (to, what);
to->global.size = new_size;
to->global.time = cgraph_estimate_time_after_inlining (freq, to, what);
}
overall_size += new_size - old_size;
ncalls_inlined++;
- if (flag_indirect_inlining)
+ /* FIXME: We should remove the optimize check after we ensure we never run
+ IPA passes when not optimizng. */
+ if (flag_indirect_inlining && optimize)
return ipa_propagate_indirect_call_infos (curr, new_edges);
else
return false;
}
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER. */
-
-static void
-cgraph_mark_inline (struct cgraph_edge *edge)
-{
- struct cgraph_node *to = edge->caller;
- struct cgraph_node *what = edge->callee;
- struct cgraph_edge *e, *next;
-
- gcc_assert (!edge->call_stmt_cannot_inline_p);
- /* Look for all calls, mark them inline and clone recursively
- all inlined functions. */
- for (e = what->callers; e; e = next)
- {
- next = e->next_caller;
- if (e->caller == to && e->inline_failed)
- {
- cgraph_mark_inline_edge (e, true, NULL);
- if (e == edge)
- edge = next;
- }
- }
-}
-
/* Estimate the growth caused by inlining NODE into all callees. */
static int
if (e->caller == node)
self_recursive = true;
if (e->inline_failed)
- growth += (cgraph_estimate_size_after_inlining (1, e->caller, node)
+ growth += (cgraph_estimate_size_after_inlining (e->caller, node)
- e->caller->global.size);
}
we decide to not inline for different reasons, but it is not big deal
as in that case we will keep the body around, but we will also avoid
some inlining. */
- if (cgraph_only_called_directly_p (node)
+ if (cgraph_will_be_removed_from_program_if_no_direct_calls (node)
&& !DECL_EXTERNAL (node->decl) && !self_recursive)
growth -= node->global.size;
+ /* COMDAT functions are very often not shared across multiple units since they
+ come from various template instantiations. Take this into account. */
+ else if (DECL_COMDAT (node->decl) && !self_recursive
+ && cgraph_can_remove_if_no_direct_calls_p (node))
+ growth -= (node->global.size
+ * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY)) + 50) / 100;
node->global.estimated_growth = growth;
return growth;
static bool
cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what,
- cgraph_inline_failed_t *reason, bool one_only)
+ cgraph_inline_failed_t *reason)
{
- int times = 0;
- struct cgraph_edge *e;
int newsize;
int limit;
HOST_WIDE_INT stack_size_limit, inlined_stack;
- if (one_only)
- times = 1;
- else
- for (e = to->callees; e; e = e->next_callee)
- if (e->callee == what)
- times++;
-
if (to->global.inlined_to)
to = to->global.inlined_to;
/* Check the size after inlining against the function limits. But allow
the function to shrink if it went over the limits by forced inlining. */
- newsize = cgraph_estimate_size_after_inlining (times, to, what);
+ newsize = cgraph_estimate_size_after_inlining (to, what);
if (newsize >= to->global.size
&& newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)
&& newsize > limit)
*reason = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
return false;
}
-
if (!n->analyzed)
{
if (reason)
*reason = CIF_BODY_NOT_AVAILABLE;
return false;
}
+ if (cgraph_function_body_availability (n) <= AVAIL_OVERWRITABLE)
+ {
+ if (reason)
+ *reason = CIF_OVERWRITABLE;
+ return false;
+ }
+
if (DECL_DECLARED_INLINE_P (decl))
{
We call recursive inlining all cases where same function appears more than
once in the single recursion nest path in the inline graph. */
-static bool
+static inline bool
cgraph_recursive_inlining_p (struct cgraph_node *to,
struct cgraph_node *what,
cgraph_inline_failed_t *reason)
{
gcov_type badness;
int growth =
- (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ (cgraph_estimate_size_after_inlining (edge->caller, edge->callee)
- edge->caller->global.size);
+ if (edge->callee->local.disregard_inline_limits)
+ return INT_MIN;
+
if (dump)
{
fprintf (dump_file, " Badness calculcation for %s -> %s\n",
return badness;
}
+/* Recompute badness of EDGE and update its key in HEAP if needed. */
+static void
+update_edge_key (fibheap_t heap, struct cgraph_edge *edge)
+{
+ int badness = cgraph_edge_badness (edge, false);
+ if (edge->aux)
+ {
+ fibnode_t n = (fibnode_t) edge->aux;
+ gcc_checking_assert (n->data == edge);
+
+ /* fibheap_replace_key only decrease the keys.
+ When we increase the key we do not update heap
+ and instead re-insert the element once it becomes
+ a minium of heap. */
+ if (badness < n->key)
+ {
+ fibheap_replace_key (heap, n, badness);
+ gcc_checking_assert (n->key == badness);
+ }
+ }
+ else
+ edge->aux = fibheap_insert (heap, badness, edge);
+}
+
/* Recompute heap nodes for each of caller edge. */
static void
struct cgraph_edge *edge;
cgraph_inline_failed_t failed_reason;
- if (!node->local.inlinable || node->local.disregard_inline_limits
+ if (!node->local.inlinable
+ || cgraph_function_body_availability (node) <= AVAIL_OVERWRITABLE
|| node->global.inlined_to)
return;
- if (bitmap_bit_p (updated_nodes, node->uid))
+ if (!bitmap_set_bit (updated_nodes, node->uid))
return;
- bitmap_set_bit (updated_nodes, node->uid);
node->global.estimated_growth = INT_MIN;
- if (!node->local.inlinable)
+ /* See if there is something to do. */
+ for (edge = node->callers; edge; edge = edge->next_caller)
+ if (edge->inline_failed)
+ break;
+ if (!edge)
return;
/* Prune out edges we won't inline into anymore. */
if (!cgraph_default_inline_p (node, &failed_reason))
{
- for (edge = node->callers; edge; edge = edge->next_caller)
+ for (; edge; edge = edge->next_caller)
if (edge->aux)
{
fibheap_delete_node (heap, (fibnode_t) edge->aux);
return;
}
- for (edge = node->callers; edge; edge = edge->next_caller)
+ for (; edge; edge = edge->next_caller)
if (edge->inline_failed)
+ update_edge_key (heap, edge);
+}
+
+/* Recompute heap nodes for each uninlined call.
+ This is used when we know that edge badnesses are going only to increase
+ (we introduced new call site) and thus all we need is to insert newly
+ created edges into heap. */
+
+static void
+update_callee_keys (fibheap_t heap, struct cgraph_node *node,
+ bitmap updated_nodes)
+{
+ struct cgraph_edge *e = node->callees;
+ node->global.estimated_growth = INT_MIN;
+
+ if (!e)
+ return;
+ while (true)
+ if (!e->inline_failed && e->callee->callees)
+ e = e->callee->callees;
+ else
{
- int badness = cgraph_edge_badness (edge, false);
- if (edge->aux)
+ if (e->inline_failed
+ && e->callee->local.inlinable
+ && cgraph_function_body_availability (e->callee) >= AVAIL_AVAILABLE
+ && !bitmap_bit_p (updated_nodes, e->callee->uid))
{
- fibnode_t n = (fibnode_t) edge->aux;
- gcc_assert (n->data == edge);
- if (n->key == badness)
- continue;
-
- /* fibheap_replace_key only increase the keys. */
- if (badness < n->key)
+ node->global.estimated_growth = INT_MIN;
+ /* If function becomes uninlinable, we need to remove it from the heap. */
+ if (!cgraph_default_inline_p (e->callee, &e->inline_failed))
+ update_caller_keys (heap, e->callee, updated_nodes);
+ else
+ /* Otherwise update just edge E. */
+ update_edge_key (heap, e);
+ }
+ if (e->next_callee)
+ e = e->next_callee;
+ else
+ {
+ do
{
- fibheap_replace_key (heap, n, badness);
- gcc_assert (n->key == badness);
- continue;
+ if (e->caller == node)
+ return;
+ e = e->caller->callers;
}
- fibheap_delete_node (heap, (fibnode_t) edge->aux);
+ while (!e->next_callee);
+ e = e->next_callee;
}
- edge->aux = fibheap_insert (heap, badness, edge);
}
}
-/* Recompute heap nodes for each of caller edges of each of callees. */
+/* Recompute heap nodes for each of caller edges of each of callees.
+ Walk recursively into all inline clones. */
static void
-update_callee_keys (fibheap_t heap, struct cgraph_node *node,
- bitmap updated_nodes)
+update_all_callee_keys (fibheap_t heap, struct cgraph_node *node,
+ bitmap updated_nodes)
{
- struct cgraph_edge *e;
+ struct cgraph_edge *e = node->callees;
node->global.estimated_growth = INT_MIN;
- for (e = node->callees; e; e = e->next_callee)
- if (e->inline_failed)
- update_caller_keys (heap, e->callee, updated_nodes);
- else if (!e->inline_failed)
- update_callee_keys (heap, e->callee, updated_nodes);
+ if (!e)
+ return;
+ while (true)
+ if (!e->inline_failed && e->callee->callees)
+ e = e->callee->callees;
+ else
+ {
+ if (e->inline_failed)
+ update_caller_keys (heap, e->callee, updated_nodes);
+ if (e->next_callee)
+ e = e->next_callee;
+ else
+ {
+ do
+ {
+ if (e->caller == node)
+ return;
+ e = e->caller->callers;
+ }
+ while (!e->next_callee);
+ e = e->next_callee;
+ }
+ }
}
/* Enqueue all recursive calls from NODE into priority queue depending on
/* Make sure that function is small enough to be considered for inlining. */
if (!max_depth
- || cgraph_estimate_size_after_inlining (1, node, node) >= limit)
+ || cgraph_estimate_size_after_inlining (node, node) >= limit)
return false;
heap = fibheap_new ();
lookup_recursive_calls (node, node, heap);
cgraph_node_name (node));
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, node->count, CGRAPH_FREQ_BASE, 1,
+ master_clone = cgraph_clone_node (node, node->decl,
+ node->count, CGRAPH_FREQ_BASE, 1,
false, NULL);
- master_clone->needed = true;
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
cgraph_clone_inlined_nodes (e, true, false);
/* Do the inlining and update list of recursive call during process. */
while (!fibheap_empty (heap)
- && (cgraph_estimate_size_after_inlining (1, node, master_clone)
+ && (cgraph_estimate_size_after_inlining (node, master_clone)
<= limit))
{
struct cgraph_edge *curr
struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
+ if (edge->callee->local.inlinable
+ && edge->inline_failed
+ && cgraph_default_inline_p (edge->callee, &edge->inline_failed))
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
int old_size = overall_size;
struct cgraph_node *where, *callee;
int badness = fibheap_min_key (heap);
+ int current_badness;
int growth;
cgraph_inline_failed_t not_good = CIF_OK;
edge->aux = NULL;
if (!edge->inline_failed)
continue;
-#ifdef ENABLE_CHECKING
- gcc_assert (cgraph_edge_badness (edge, false) == badness);
-#endif
+
+ /* When updating the edge costs, we only decrease badness in the keys.
+ When the badness increase, we keep the heap as it is and re-insert
+ key now. */
+ current_badness = cgraph_edge_badness (edge, false);
+ gcc_assert (current_badness >= badness);
+ if (current_badness != badness)
+ {
+ edge->aux = fibheap_insert (heap, current_badness, edge);
+ continue;
+ }
+
callee = edge->callee;
- growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ growth = (cgraph_estimate_size_after_inlining (edge->caller, edge->callee)
- edge->caller->global.size);
if (dump_file)
}
}
- if (!cgraph_maybe_hot_edge_p (edge))
+ if (edge->callee->local.disregard_inline_limits)
+ ;
+ else if (!cgraph_maybe_hot_edge_p (edge))
not_good = CIF_UNLIKELY_CALL;
- if (!flag_inline_functions
+ else if (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (edge->callee->decl))
not_good = CIF_NOT_DECLARED_INLINED;
- if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
+ else if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge->caller->decl)))
not_good = CIF_OPTIMIZING_FOR_SIZE;
if (not_good && growth > 0 && cgraph_estimate_growth (edge->callee) > 0)
{
}
continue;
}
- if (!tree_can_inline_p (edge))
+ if (!tree_can_inline_p (edge)
+ || edge->call_stmt_cannot_inline_p)
{
if (dump_file)
fprintf (dump_file, " inline_failed:%s.\n",
continue;
if (flag_indirect_inlining)
add_new_edges_to_heap (heap, new_indirect_edges);
- update_callee_keys (heap, where, updated_nodes);
+ update_all_callee_keys (heap, where, updated_nodes);
}
else
{
struct cgraph_node *callee;
- if (edge->call_stmt_cannot_inline_p
- || !cgraph_check_inline_limits (edge->caller, edge->callee,
- &edge->inline_failed, true))
+ if (!cgraph_check_inline_limits (edge->caller, edge->callee,
+ &edge->inline_failed))
{
if (dump_file)
fprintf (dump_file, " Not inlining into %s:%s.\n",
continue;
}
callee = edge->callee;
+ gcc_checking_assert (!callee->global.inlined_to);
cgraph_mark_inline_edge (edge, true, &new_indirect_edges);
if (flag_indirect_inlining)
add_new_edges_to_heap (heap, new_indirect_edges);
- update_callee_keys (heap, callee, updated_nodes);
+ /* We inlined last offline copy to the body. This might lead
+ to callees of function having fewer call sites and thus they
+ may need updating. */
+ if (callee->global.inlined_to)
+ update_all_callee_keys (heap, callee, updated_nodes);
+ else
+ update_callee_keys (heap, edge->callee, updated_nodes);
}
where = edge->caller;
if (where->global.inlined_to)
if (!edge->inline_failed)
continue;
#ifdef ENABLE_CHECKING
- gcc_assert (cgraph_edge_badness (edge, false) == badness);
+ gcc_assert (cgraph_edge_badness (edge, false) >= badness);
#endif
if (dump_file)
{
struct cgraph_node *orig_callee;
if (e->call_stmt_cannot_inline_p)
- continue;
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
if (!e->callee->analyzed)
{
continue;
}
+ if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+ != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: SSA form does not match.\n");
+ continue;
+ }
+
/* Inline the edge and flatten the inline clone. Avoid
recursing through the original node if the node was cloned. */
if (dump_file)
struct cgraph_edge *e;
gcc_assert (inline_summary (node)->self_size == node->global.size);
- initial_size += node->global.size;
+ if (!DECL_EXTERNAL (node->decl))
+ initial_size += node->global.size;
for (e = node->callees; e; e = e->next_callee)
if (max_count < e->count)
max_count = e->count;
if (node->callers
&& !node->callers->next_caller
- && cgraph_only_called_directly_p (node)
+ && !node->global.inlined_to
+ && cgraph_will_be_removed_from_program_if_no_direct_calls (node)
&& node->local.inlinable
+ && cgraph_function_body_availability (node) >= AVAIL_AVAILABLE
&& node->callers->inline_failed
&& node->callers->caller != node
&& node->callers->caller->global.inlined_to != node
&& !node->callers->call_stmt_cannot_inline_p
- && !DECL_EXTERNAL (node->decl)
- && !DECL_COMDAT (node->decl))
+ && !DECL_EXTERNAL (node->decl))
{
cgraph_inline_failed_t reason;
old_size = overall_size;
}
if (cgraph_check_inline_limits (node->callers->caller, node,
- &reason, false))
+ &reason))
{
struct cgraph_node *caller = node->callers->caller;
- cgraph_mark_inline (node->callers);
+ cgraph_mark_inline_edge (node->callers, true, NULL);
if (dump_file)
fprintf (dump_file,
" Inlined into %s which now has %i size"
return 0;
}
-/* Return true when N is leaf function. Accept cheap (pure&const) builtins
+/* Return true when N is leaf function. Accept cheap builtins
in leaf functions. */
+
static bool
leaf_node_p (struct cgraph_node *n)
{
struct cgraph_edge *e;
for (e = n->callees; e; e = e->next_callee)
- if (!DECL_BUILT_IN (e->callee->decl)
- || (!TREE_READONLY (e->callee->decl)
- || DECL_PURE_P (e->callee->decl)))
+ if (!is_inexpensive_builtin (e->callee->decl))
return false;
return true;
}
if (!e->callee->local.disregard_inline_limits
&& (mode != INLINE_ALL || !e->callee->local.inlinable))
continue;
- if (e->call_stmt_cannot_inline_p)
- continue;
if (dump_file)
fprintf (dump_file,
"Considering to always inline inline candidate %s.\n",
fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
- if (!tree_can_inline_p (e))
+ if (!tree_can_inline_p (e)
+ || e->call_stmt_cannot_inline_p)
{
if (dump_file)
fprintf (dump_file,
fprintf (dump_file, " Inlining %s into %s.\n",
cgraph_node_name (e->callee),
cgraph_node_name (e->caller));
- cgraph_mark_inline (e);
+ cgraph_mark_inline_edge (e, true, NULL);
inlined = true;
}
if (((mode == INLINE_SIZE || mode == INLINE_SIZE_NORECURSIVE)
|| (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (e->callee->decl)))
- && (cgraph_estimate_size_after_inlining (1, e->caller, e->callee)
+ && (cgraph_estimate_size_after_inlining (e->caller, e->callee)
> e->caller->global.size + allowed_growth)
&& cgraph_estimate_growth (e->callee) > allowed_growth)
{
if (dump_file)
fprintf (dump_file,
"Not inlining: code size would grow by %i.\n",
- cgraph_estimate_size_after_inlining (1, e->caller,
+ cgraph_estimate_size_after_inlining (e->caller,
e->callee)
- e->caller->global.size);
continue;
}
- if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
- false)
- || e->call_stmt_cannot_inline_p)
+ if (e->call_stmt_cannot_inline_p
+ || !tree_can_inline_p (e))
{
if (dump_file)
- fprintf (dump_file, "Not inlining: %s.\n",
- cgraph_inline_failed_string (e->inline_failed));
+ fprintf (dump_file,
+ "Not inlining: call site not inlinable.\n");
continue;
}
if (!e->callee->analyzed)
"Not inlining: Function body no longer available.\n");
continue;
}
- if (!tree_can_inline_p (e))
+ if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed))
{
if (dump_file)
- fprintf (dump_file,
- "Not inlining: %s.",
+ fprintf (dump_file, "Not inlining: %s.\n",
cgraph_inline_failed_string (e->inline_failed));
continue;
}
fprintf (dump_file, " Inlining %s into %s.\n",
cgraph_node_name (e->callee),
cgraph_node_name (e->caller));
- cgraph_mark_inline (e);
+ cgraph_mark_inline_edge (e, true, NULL);
inlined = true;
}
}
unsigned int todo = 0;
int iterations = 0;
- if (sorrycount || errorcount)
+ if (seen_error ())
return 0;
if (!optimize
}
};
-/* When inlining shall be performed. */
-static bool
-cgraph_gate_ipa_early_inlining (void)
-{
- return (flag_early_inlining
- && !in_lto_p
- && (flag_branch_probabilities || flag_test_coverage
- || profile_arc_flag));
-}
-
-/* IPA pass wrapper for early inlining pass. We need to run early inlining
- before tree profiling so we have stand alone IPA pass for doing so. */
-struct simple_ipa_opt_pass pass_ipa_early_inline =
-{
- {
- SIMPLE_IPA_PASS,
- "einline_ipa", /* name */
- cgraph_gate_ipa_early_inlining, /* gate */
- NULL, /* execute */
- NULL, /* sub */
- NULL, /* next */
- 0, /* static_pass_number */
- TV_INLINE_HEURISTICS, /* tv_id */
- 0, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- TODO_dump_cgraph /* todo_flags_finish */
- }
-};
-/* See if statement might disappear after inlining. We are not terribly
- sophisficated, basically looking for simple abstraction penalty wrappers. */
+/* See if statement might disappear after inlining.
+ 0 - means not eliminated
+ 1 - half of statements goes away
+ 2 - for sure it is eliminated.
+ We are not terribly sophisficated, basically looking for simple abstraction
+ penalty wrappers. */
-static bool
-likely_eliminated_by_inlining_p (gimple stmt)
+static int
+eliminated_by_inlining_prob (gimple stmt)
{
enum gimple_code code = gimple_code (stmt);
switch (code)
{
case GIMPLE_RETURN:
- return true;
+ return 2;
case GIMPLE_ASSIGN:
if (gimple_num_ops (stmt) != 2)
- return false;
+ return 0;
/* Casts of parameters, loads from parameters passed by reference
- and stores to return value or parameters are probably free after
- inlining. */
+ and stores to return value or parameters are often free after
+ inlining dua to SRA and further combining.
+ Assume that half of statements goes away. */
if (gimple_assign_rhs_code (stmt) == CONVERT_EXPR
|| gimple_assign_rhs_code (stmt) == NOP_EXPR
|| gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
bool rhs_free = false;
bool lhs_free = false;
- while (handled_component_p (inner_lhs) || TREE_CODE (inner_lhs) == INDIRECT_REF)
+ while (handled_component_p (inner_lhs)
+ || TREE_CODE (inner_lhs) == MEM_REF)
inner_lhs = TREE_OPERAND (inner_lhs, 0);
while (handled_component_p (inner_rhs)
- || TREE_CODE (inner_rhs) == ADDR_EXPR || TREE_CODE (inner_rhs) == INDIRECT_REF)
+ || TREE_CODE (inner_rhs) == ADDR_EXPR
+ || TREE_CODE (inner_rhs) == MEM_REF)
inner_rhs = TREE_OPERAND (inner_rhs, 0);
|| (TREE_CODE (inner_lhs) == SSA_NAME
&& TREE_CODE (SSA_NAME_VAR (inner_lhs)) == RESULT_DECL))
lhs_free = true;
- if (lhs_free && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
+ if (lhs_free
+ && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
rhs_free = true;
if (lhs_free && rhs_free)
- return true;
+ return 1;
}
- return false;
+ return 0;
default:
- return false;
+ return 0;
}
}
{
gcov_type time = 0;
gcov_type time_inlining_benefit = 0;
- int size = 0;
- int size_inlining_benefit = 0;
+ /* Estimate static overhead for function prologue/epilogue and alignment. */
+ int size = 2;
+ /* Benefits are scaled by probability of elimination that is in range
+ <0,2>. */
+ int size_inlining_benefit = 2 * 2;
basic_block bb;
gimple_stmt_iterator bsi;
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
int freq;
tree funtype = TREE_TYPE (node->decl);
- if (node->local.disregard_inline_limits)
- {
- inline_summary (node)->self_time = 0;
- inline_summary (node)->self_size = 0;
- inline_summary (node)->time_inlining_benefit = 0;
- inline_summary (node)->size_inlining_benefit = 0;
- }
-
if (dump_file)
fprintf (dump_file, "Analyzing function body size: %s\n",
cgraph_node_name (node));
gimple stmt = gsi_stmt (bsi);
int this_size = estimate_num_insns (stmt, &eni_size_weights);
int this_time = estimate_num_insns (stmt, &eni_time_weights);
+ int prob;
if (dump_file && (dump_flags & TDF_DETAILS))
{
this_time *= freq;
time += this_time;
size += this_size;
- if (likely_eliminated_by_inlining_p (stmt))
- {
- size_inlining_benefit += this_size;
- time_inlining_benefit += this_time;
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, " Likely eliminated\n");
- }
+ prob = eliminated_by_inlining_prob (stmt);
+ if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, " 50%% will be eliminated by inlining\n");
+ if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, " will eliminated by inlining\n");
+ size_inlining_benefit += this_size * prob;
+ time_inlining_benefit += this_time * prob;
gcc_assert (time >= 0);
gcc_assert (size >= 0);
}
}
time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
- time_inlining_benefit = ((time_inlining_benefit + CGRAPH_FREQ_BASE / 2)
- / CGRAPH_FREQ_BASE);
+ time_inlining_benefit = ((time_inlining_benefit + CGRAPH_FREQ_BASE)
+ / (CGRAPH_FREQ_BASE * 2));
+ size_inlining_benefit = (size_inlining_benefit + 1) / 2;
if (dump_file)
fprintf (dump_file, "Overall function body time: %i-%i size: %i-%i\n",
(int)time, (int)time_inlining_benefit,
time_inlining_benefit += cost;
size_inlining_benefit += cost;
}
- for (arg = DECL_ARGUMENTS (node->decl); arg; arg = TREE_CHAIN (arg))
+ for (arg = DECL_ARGUMENTS (node->decl); arg; arg = DECL_CHAIN (arg))
if (!VOID_TYPE_P (TREE_TYPE (arg)))
{
int cost = estimate_move_cost (TREE_TYPE (arg));
/* Estimate the stack size for the function. But not at -O0
because estimated_stack_frame_size is a quadratic problem. */
- self_stack_size = optimize ? estimated_stack_frame_size () : 0;
+ self_stack_size = optimize ? estimated_stack_frame_size (node->decl) : 0;
inline_summary (node)->estimated_self_stack_size = self_stack_size;
node->global.estimated_stack_size = self_stack_size;
node->global.stack_frame_offset = 0;
static void
inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
{
- struct cgraph_edge *cs;
-
- if (!flag_ipa_cp)
- {
- ipa_initialize_node_params (node);
- ipa_detect_param_modifications (node);
- }
- ipa_analyze_params_uses (node);
-
- if (!flag_ipa_cp)
- for (cs = node->callees; cs; cs = cs->next_callee)
- {
- ipa_count_arguments (cs);
- ipa_compute_jump_functions (cs);
- }
-
- if (dump_file)
+ ipa_analyze_node (node);
+ if (dump_file && (dump_flags & TDF_DETAILS))
{
ipa_print_node_params (dump_file, node);
ipa_print_node_jump_functions (dump_file, node);
current_function_decl = node->decl;
compute_inline_parameters (node);
- if (flag_indirect_inlining)
+ /* FIXME: We should remove the optimize check after we ensure we never run
+ IPA passes when not optimizng. */
+ if (flag_indirect_inlining && optimize)
inline_indirect_intraprocedural_analysis (node);
current_function_decl = NULL;
{
unsigned int todo = 0;
struct cgraph_edge *e;
+ bool inline_p = false;
/* FIXME: Currently the passmanager is adding inline transform more than once to some
clones. This needs revisiting after WPA cleanups. */
save_inline_function_body (node);
for (e = node->callees; e; e = e->next_callee)
- if (!e->inline_failed || warn_inline)
- break;
+ {
+ cgraph_redirect_edge_call_stmt_to_callee (e);
+ if (!e->inline_failed || warn_inline)
+ inline_p = true;
+ }
- if (e)
+ if (inline_p)
{
timevar_push (TV_INTEGRATION);
todo = optimize_inline_calls (current_function_decl);
OSDN Git Service
