if (dump_file)
{
fprintf (dump_file, " not inlinable: %s/%i -> %s/%i, %s\n",
- cgraph_node_name (e->caller), e->caller->uid,
- cgraph_node_name (e->callee), e->callee->uid,
+ xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
+ xstrdup (cgraph_node_name (e->callee)), e->callee->uid,
cgraph_inline_failed_string (e->inline_failed));
}
}
{
bool inlinable = true;
enum availability avail;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, &avail);
+ struct cgraph_node *callee
+ = cgraph_function_or_thunk_node (e->callee, &avail);
tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e->caller->decl);
- tree callee_tree = callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->decl) : NULL;
+ tree callee_tree
+ = callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->decl) : NULL;
+ struct function *caller_cfun = DECL_STRUCT_FUNCTION (e->caller->decl);
+ struct function *callee_cfun
+ = callee ? DECL_STRUCT_FUNCTION (callee->decl) : NULL;
+
+ if (!caller_cfun && e->caller->clone_of)
+ caller_cfun = DECL_STRUCT_FUNCTION (e->caller->clone_of->decl);
+
+ if (!callee_cfun && callee && callee->clone_of)
+ callee_cfun = DECL_STRUCT_FUNCTION (callee->clone_of->decl);
gcc_assert (e->inline_failed);
e->inline_failed = CIF_EH_PERSONALITY;
inlinable = false;
}
+ /* TM pure functions should not be inlined into non-TM_pure
+ functions. */
+ else if (is_tm_pure (callee->decl)
+ && !is_tm_pure (e->caller->decl))
+ {
+ e->inline_failed = CIF_UNSPECIFIED;
+ inlinable = false;
+ }
/* Don't inline if the callee can throw non-call exceptions but the
caller cannot.
FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing.
Move the flag into cgraph node or mirror it in the inline summary. */
- else if (DECL_STRUCT_FUNCTION (callee->decl)
- && DECL_STRUCT_FUNCTION
- (callee->decl)->can_throw_non_call_exceptions
- && !(DECL_STRUCT_FUNCTION (e->caller->decl)
- && DECL_STRUCT_FUNCTION
- (e->caller->decl)->can_throw_non_call_exceptions))
+ else if (callee_cfun && callee_cfun->can_throw_non_call_exceptions
+ && !(caller_cfun && caller_cfun->can_throw_non_call_exceptions))
{
e->inline_failed = CIF_NON_CALL_EXCEPTIONS;
inlinable = false;
? callee_tree
: optimization_default_node);
- if ((caller_opt->x_optimize > callee_opt->x_optimize)
- || (caller_opt->x_optimize_size != callee_opt->x_optimize_size))
+ if (((caller_opt->x_optimize > callee_opt->x_optimize)
+ || (caller_opt->x_optimize_size != callee_opt->x_optimize_size))
+ /* gcc.dg/pr43564.c. Look at forced inline even in -O0. */
+ && !DECL_DISREGARD_INLINE_LIMITS (e->callee->decl))
{
- e->inline_failed = CIF_TARGET_OPTIMIZATION_MISMATCH;
+ e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
inlinable = false;
}
}
- /* Be sure that the cannot_inline_p flag is up to date. */
- gcc_checking_assert (!e->call_stmt
- || (gimple_call_cannot_inline_p (e->call_stmt)
- == e->call_stmt_cannot_inline_p)
- /* In -flto-partition=none mode we really keep things out of
- sync because call_stmt_cannot_inline_p is set at cgraph
- merging when function bodies are not there yet. */
- || (in_lto_p && !gimple_call_cannot_inline_p (e->call_stmt)));
if (!inlinable && report)
report_inline_failed_reason (e);
return inlinable;
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"call is cold and code would grow by %i\n",
- cgraph_node_name (e->caller), e->caller->uid,
- cgraph_node_name (callee), callee->uid,
+ xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
+ xstrdup (cgraph_node_name (callee)), callee->uid,
growth);
want_inline = false;
}
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"callee is not leaf and code would grow by %i\n",
- cgraph_node_name (e->caller), e->caller->uid,
- cgraph_node_name (callee), callee->uid,
+ xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
+ xstrdup (cgraph_node_name (callee)), callee->uid,
growth);
want_inline = false;
}
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"growth %i exceeds --param early-inlining-insns\n",
- cgraph_node_name (e->caller), e->caller->uid,
- cgraph_node_name (callee), callee->uid,
+ xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
+ xstrdup (cgraph_node_name (callee)), callee->uid,
growth);
want_inline = false;
}
e->inline_failed = CIF_MAX_INLINE_INSNS_SINGLE_LIMIT;
want_inline = false;
}
- else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && !flag_inline_functions)
- {
- e->inline_failed = CIF_NOT_DECLARED_INLINED;
- want_inline = false;
- }
- else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && growth >= MAX_INLINE_INSNS_AUTO)
- {
- e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
- want_inline = false;
- }
- /* If call is cold, do not inline when function body would grow.
- Still inline when the overall unit size will shrink because the offline
- copy of function being eliminated.
+ /* Before giving up based on fact that caller size will grow, allow
+ functions that are called few times and eliminating the offline
+ copy will lead to overall code size reduction.
+ Not all of these will be handled by subsequent inlining of functions
+ called once: in particular weak functions are not handled or funcitons
+ that inline to multiple calls but a lot of bodies is optimized out.
+ Finally we want to inline earlier to allow inlining of callbacks.
This is slightly wrong on aggressive side: it is entirely possible
that function is called many times with a context where inlining
first, this situation is not a problem at all: after inlining all
"good" calls, we will realize that keeping the function around is
better. */
- else if (!cgraph_maybe_hot_edge_p (e)
- && (DECL_EXTERNAL (callee->decl)
-
- /* Unlike for functions called once, we play unsafe with
- COMDATs. We can allow that since we know functions
- in consideration are small (and thus risk is small) and
- moreover grow estimates already accounts that COMDAT
- functions may or may not disappear when eliminated from
- current unit. With good probability making aggressive
- choice in all units is going to make overall program
- smaller.
-
- Consequently we ask cgraph_can_remove_if_no_direct_calls_p
- instead of
- cgraph_will_be_removed_from_program_if_no_direct_calls */
-
- || !cgraph_can_remove_if_no_direct_calls_p (callee)
- || estimate_growth (callee) > 0))
+ else if (growth <= MAX_INLINE_INSNS_SINGLE
+ /* Unlike for functions called once, we play unsafe with
+ COMDATs. We can allow that since we know functions
+ in consideration are small (and thus risk is small) and
+ moreover grow estimates already accounts that COMDAT
+ functions may or may not disappear when eliminated from
+ current unit. With good probability making aggressive
+ choice in all units is going to make overall program
+ smaller.
+
+ Consequently we ask cgraph_can_remove_if_no_direct_calls_p
+ instead of
+ cgraph_will_be_removed_from_program_if_no_direct_calls */
+ && !DECL_EXTERNAL (callee->decl)
+ && cgraph_can_remove_if_no_direct_calls_p (callee)
+ && estimate_growth (callee) <= 0)
+ ;
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
+ && !flag_inline_functions)
+ {
+ e->inline_failed = CIF_NOT_DECLARED_INLINED;
+ want_inline = false;
+ }
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
+ && growth >= MAX_INLINE_INSNS_AUTO)
+ {
+ e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
+ want_inline = false;
+ }
+ /* If call is cold, do not inline when function body would grow. */
+ else if (!cgraph_maybe_hot_edge_p (e))
{
e->inline_failed = CIF_UNLIKELY_CALL;
want_inline = false;
return want_inline;
}
+/* Return true when NODE has caller other than EDGE.
+ Worker for cgraph_for_node_and_aliases. */
+
+static bool
+check_caller_edge (struct cgraph_node *node, void *edge)
+{
+ return (node->callers
+ && node->callers != edge);
+}
+
/* Decide if NODE is called once inlining it would eliminate need
for the offline copy of function. */
static bool
want_inline_function_called_once_p (struct cgraph_node *node)
{
- if (node->alias)
- return false;
+ struct cgraph_node *function = cgraph_function_or_thunk_node (node, NULL);
/* Already inlined? */
- if (node->global.inlined_to)
+ if (function->global.inlined_to)
return false;
/* Zero or more then one callers? */
if (!node->callers
|| node->callers->next_caller)
return false;
+ /* Maybe other aliases has more direct calls. */
+ if (cgraph_for_node_and_aliases (node, check_caller_edge, node->callers, true))
+ return false;
/* Recursive call makes no sense to inline. */
- if (node->callers->caller == node)
+ if (cgraph_edge_recursive_p (node->callers))
return false;
/* External functions are not really in the unit, so inlining
them when called once would just increase the program size. */
- if (DECL_EXTERNAL (node->decl))
+ if (DECL_EXTERNAL (function->decl))
return false;
/* Offline body must be optimized out. */
- if (!cgraph_will_be_removed_from_program_if_no_direct_calls (node))
+ if (!cgraph_will_be_removed_from_program_if_no_direct_calls (function))
return false;
if (!can_inline_edge_p (node->callers, true))
return false;
uninlined_call_time =
((gcov_type)
(callee_info->time
- + inline_edge_summary (edge)->call_stmt_time
- + CGRAPH_FREQ_BASE / 2) * edge->frequency
- / CGRAPH_FREQ_BASE);
+ + inline_edge_summary (edge)->call_stmt_time) * edge->frequency
+ + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
/* Compute relative time benefit, i.e. how much the call becomes faster.
??? perhaps computing how much the caller+calle together become faster
would lead to more realistic results. */
if (dump)
{
fprintf (dump_file, " Badness calculation for %s -> %s\n",
- cgraph_node_name (edge->caller),
- cgraph_node_name (callee));
+ xstrdup (cgraph_node_name (edge->caller)),
+ xstrdup (cgraph_node_name (callee)));
fprintf (dump_file, " size growth %i, time growth %i\n",
growth,
time_growth);
else if (flag_guess_branch_prob)
{
int div = edge->frequency * (1<<10) / CGRAPH_FREQ_MAX;
- int growth_for_all;
div = MAX (div, 1);
gcc_checking_assert (edge->frequency <= CGRAPH_FREQ_MAX);
/* Result must be integer in range 0...INT_MAX.
Set the base of fixed point calculation so we don't lose much of
precision for small bandesses (those are interesting) yet we don't
- overflow for growths that are still in interesting range. */
- badness = ((gcov_type)growth) * (1<<18);
+ overflow for growths that are still in interesting range.
+
+ Fixed point arithmetic with point at 8th bit. */
+ badness = ((gcov_type)growth) * (1<<(19+8));
badness = (badness + div / 2) / div;
/* Overall growth of inlining all calls of function matters: we want to
We might mix the valud into the fraction by taking into account
relative growth of the unit, but for now just add the number
into resulting fraction. */
- growth_for_all = estimate_growth (callee);
- badness += growth_for_all;
- if (badness > INT_MAX - 1)
- badness = INT_MAX - 1;
+ if (badness > INT_MAX / 2)
+ {
+ badness = INT_MAX / 2;
+ if (dump)
+ fprintf (dump_file, "Badness overflow\n");
+ }
if (dump)
{
fprintf (dump_file,
- " %i: guessed profile. frequency %f, overall growth %i,"
+ " %i: guessed profile. frequency %f,"
" benefit %f%%, divisor %i\n",
- (int) badness, (double)edge->frequency / CGRAPH_FREQ_BASE, growth_for_all,
+ (int) badness, (double)edge->frequency / CGRAPH_FREQ_BASE,
relative_time_benefit (callee_info, edge, time_growth) * 100 / 256.0, div);
}
}
else
{
int nest = MIN (inline_edge_summary (edge)->loop_depth, 8);
- badness = estimate_growth (callee) * 256;
+ badness = growth * 256;
/* Decrease badness if call is nested. */
if (badness > 0)
{
fprintf (dump_file,
" decreasing badness %s/%i -> %s/%i, %i to %i\n",
- cgraph_node_name (edge->caller), edge->caller->uid,
- cgraph_node_name (edge->callee), edge->callee->uid,
+ xstrdup (cgraph_node_name (edge->caller)),
+ edge->caller->uid,
+ xstrdup (cgraph_node_name (edge->callee)),
+ edge->callee->uid,
(int)n->key,
badness);
}
{
fprintf (dump_file,
" enqueuing call %s/%i -> %s/%i, badness %i\n",
- cgraph_node_name (edge->caller), edge->caller->uid,
- cgraph_node_name (edge->callee), edge->callee->uid,
+ xstrdup (cgraph_node_name (edge->caller)),
+ edge->caller->uid,
+ xstrdup (cgraph_node_name (edge->callee)),
+ edge->callee->uid,
badness);
}
edge->aux = fibheap_insert (heap, badness, edge);
struct cgraph_edge *edge;
struct cgraph_edge *e = node->callees;
struct cgraph_node *where = node;
+ int i;
+ struct ipa_ref *ref;
if (where->global.inlined_to)
where = where->global.inlined_to;
for (edge = where->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
reset_edge_growth_cache (edge);
+ for (i = 0; ipa_ref_list_refering_iterate (&where->ref_list, i, ref); i++)
+ if (ref->use == IPA_REF_ALIAS)
+ reset_edge_caches (ipa_ref_refering_node (ref));
if (!e)
return;
struct cgraph_edge *check_inlinablity_for)
{
struct cgraph_edge *edge;
+ int i;
+ struct ipa_ref *ref;
- if (!inline_summary (node)->inlinable
+ if ((!node->alias && !inline_summary (node)->inlinable)
|| cgraph_function_body_availability (node) <= AVAIL_OVERWRITABLE
|| node->global.inlined_to)
return;
if (!bitmap_set_bit (updated_nodes, node->uid))
return;
+ for (i = 0; ipa_ref_list_refering_iterate (&node->ref_list, i, ref); i++)
+ if (ref->use == IPA_REF_ALIAS)
+ {
+ struct cgraph_node *alias = ipa_ref_refering_node (ref);
+ update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for);
+ }
+
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
{
depth = 1;
for (cnode = curr->caller;
cnode->global.inlined_to; cnode = cnode->callers->caller)
- if (node->decl == curr->callee->decl)
- depth++;
+ if (node->decl
+ == cgraph_function_or_thunk_node (curr->callee, NULL)->decl)
+ depth++;
if (!want_inline_self_recursive_call_p (curr, node, false, depth))
continue;
struct cgraph_node *where, *callee;
int badness = fibheap_min_key (heap);
int current_badness;
+ int cached_badness;
int growth;
edge = (struct cgraph_edge *) fibheap_extract_min (heap);
if (!edge->inline_failed)
continue;
- /* Be sure that caches are maintained consistent. */
-#ifdef ENABLE_CHECKING
+ /* Be sure that caches are maintained consistent.
+ We can not make this ENABLE_CHECKING only because it cause differnt
+ updates of the fibheap queue. */
+ cached_badness = edge_badness (edge, false);
reset_edge_growth_cache (edge);
reset_node_growth_cache (edge->callee);
-#endif
/* When updating the edge costs, we only decrease badness in the keys.
Increases of badness are handled lazilly; when we see key with out
of date value on it, we re-insert it now. */
current_badness = edge_badness (edge, false);
+ gcc_assert (cached_badness == current_badness);
gcc_assert (current_badness >= badness);
if (current_badness != badness)
{
where = edge->caller;
while (where->global.inlined_to)
{
- if (where->decl == edge->callee->decl)
+ if (where->decl == callee->decl)
outer_node = where, depth++;
where = where->callers->caller;
}
/* We inlined last offline copy to the body. This might lead
to callees of function having fewer call sites and thus they
- may need updating. */
- if (callee->global.inlined_to)
+ may need updating.
+
+ FIXME: the callee size could also shrink because more information
+ is propagated from caller. We don't track when this happen and
+ thus we need to recompute everything all the time. Once this is
+ solved, "|| 1" should go away. */
+ if (callee->global.inlined_to || 1)
update_all_callee_keys (heap, callee, updated_nodes);
else
update_callee_keys (heap, edge->callee, updated_nodes);
if (dump_file)
fprintf (dump_file,
"Not inlining %s into %s to avoid cycle.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
+ xstrdup (cgraph_node_name (callee)),
+ xstrdup (cgraph_node_name (e->caller)));
e->inline_failed = CIF_RECURSIVE_INLINING;
continue;
}
recursing through the original node if the node was cloned. */
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
+ xstrdup (cgraph_node_name (callee)),
+ xstrdup (cgraph_node_name (e->caller)));
orig_callee = callee;
inline_call (e, true, NULL, NULL);
if (e->callee != orig_callee)
XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
int i;
- if (in_lto_p && flag_indirect_inlining)
+ if (in_lto_p && optimize)
ipa_update_after_lto_read ();
- if (flag_indirect_inlining)
- ipa_create_all_structures_for_iinln ();
if (dump_file)
dump_inline_summaries (dump_file);
{
fprintf (dump_file,
"\nInlining %s size %i.\n",
- cgraph_node_name (node), inline_summary (node)->size);
+ cgraph_node_name (node),
+ inline_summary (node)->size);
fprintf (dump_file,
" Called once from %s %i insns.\n",
cgraph_node_name (node->callers->caller),
}
/* Free ipa-prop structures if they are no longer needed. */
- if (flag_indirect_inlining)
+ if (optimize)
ipa_free_all_structures_after_iinln ();
if (dump_file)
if (dump_file)
fprintf (dump_file, " Inlining %s into %s (always_inline).\n",
- cgraph_node_name (e->callee),
- cgraph_node_name (e->caller));
+ xstrdup (cgraph_node_name (e->callee)),
+ xstrdup (cgraph_node_name (e->caller)));
inline_call (e, true, NULL, NULL);
inlined = true;
}
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
+ xstrdup (cgraph_node_name (callee)),
+ xstrdup (cgraph_node_name (e->caller)));
inline_call (e, true, NULL, NULL);
inlined = true;
}
= estimate_num_insns (edge->call_stmt, &eni_size_weights);
es->call_stmt_time
= estimate_num_insns (edge->call_stmt, &eni_time_weights);
+ if (edge->callee->decl
+ && !gimple_check_call_matching_types (edge->call_stmt,
+ edge->callee->decl))
+ edge->call_stmt_cannot_inline_p = true;
}
timevar_pop (TV_INTEGRATION);
iterations++;
0, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
- TODO_dump_func /* todo_flags_finish */
+ 0 /* todo_flags_finish */
}
};
/* When to run IPA inlining. Inlining of always-inline functions
- happens during early inlining. */
+ happens during early inlining.
+
+ Enable inlining unconditoinally at -flto. We need size estimates to
+ drive partitioning. */
static bool
gate_ipa_inline (void)
{
- /* ??? We'd like to skip this if not optimizing or not inlining as
- all always-inline functions have been processed by early
- inlining already. But this at least breaks EH with C++ as
- we need to unconditionally run fixup_cfg even at -O0.
- So leave it on unconditionally for now. */
- return 1;
+ return optimize || flag_lto || flag_wpa;
}
struct ipa_opt_pass_d pass_ipa_inline =
0, /* properties_provided */
0, /* properties_destroyed */
TODO_remove_functions, /* todo_flags_finish */
- TODO_dump_cgraph | TODO_dump_func
+ TODO_dump_cgraph
| TODO_remove_functions | TODO_ggc_collect /* todo_flags_finish */
},
inline_generate_summary, /* generate_summary */
OSDN Git Service
