/* Callgraph based interprocedural optimizations.
- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009
Free Software Foundation, Inc.
Contributed by Jan Hubicka
#include "timevar.h"
#include "params.h"
#include "fibheap.h"
-#include "c-common.h"
#include "intl.h"
#include "function.h"
#include "ipa-prop.h"
#include "tree-iterator.h"
#include "tree-pass.h"
#include "output.h"
+#include "coverage.h"
static void cgraph_expand_all_functions (void);
static void cgraph_mark_functions_to_output (void);
case CGRAPH_STATE_EXPANSION:
/* Functions created during expansion shall be compiled
directly. */
- node->output = 0;
+ node->process = 0;
cgraph_expand_function (node);
break;
gcc_unreachable ();
break;
}
+ cgraph_call_function_insertion_hooks (node);
}
return output;
}
static void
cgraph_reset_node (struct cgraph_node *node)
{
- /* If node->output is set, then we have already begun whole-unit analysis.
+ /* If node->process is set, then we have already begun whole-unit analysis.
This is *not* testing for whether we've already emitted the function.
That case can be sort-of legitimately seen with real function redefinition
errors. I would argue that the front end should never present us with
such a case, but don't enforce that for now. */
- gcc_assert (!node->output);
+ gcc_assert (!node->process);
/* Reset our data structures so we can analyze the function again. */
memset (&node->local, 0, sizeof (node->local));
notice_global_symbol (decl);
node->local.finalized = true;
node->lowered = DECL_STRUCT_FUNCTION (decl)->cfg != NULL;
+ node->finalized_by_frontend = true;
record_cdtor_fn (node->decl);
if (node->nested)
lower_nested_functions (decl);
cgraph_mark_needed_node (node);
}
+/* Return TRUE if NODE2 is equivalent to NODE or its clone. */
+static bool
+clone_of_p (struct cgraph_node *node, struct cgraph_node *node2)
+{
+ while (node != node2 && node2)
+ node2 = node2->clone_of;
+ return node2 != NULL;
+}
+
/* Verify cgraph nodes of given cgraph node. */
void
verify_cgraph_node (struct cgraph_node *node)
{
struct cgraph_edge *e;
- struct cgraph_node *main_clone;
struct function *this_cfun = DECL_STRUCT_FUNCTION (node->decl);
struct function *saved_cfun = cfun;
basic_block this_block;
if (e->aux)
{
error ("aux field set for edge %s->%s",
- cgraph_node_name (e->caller), cgraph_node_name (e->callee));
+ identifier_to_locale (cgraph_node_name (e->caller)),
+ identifier_to_locale (cgraph_node_name (e->callee)));
error_found = true;
}
if (node->count < 0)
error_found = true;
}
- for (main_clone = cgraph_node (node->decl); main_clone;
- main_clone = main_clone->next_clone)
- if (main_clone == node)
- break;
if (!cgraph_node (node->decl))
{
error ("node not found in cgraph_hash");
error_found = true;
}
- if (node->analyzed
- && gimple_body (node->decl)
+ if (node->clone_of)
+ {
+ struct cgraph_node *n;
+ for (n = node->clone_of->clones; n; n = n->next_sibling_clone)
+ if (n == node)
+ break;
+ if (!n)
+ {
+ error ("node has wrong clone_of");
+ error_found = true;
+ }
+ }
+ if (node->clones)
+ {
+ struct cgraph_node *n;
+ for (n = node->clones; n; n = n->next_sibling_clone)
+ if (n->clone_of != node)
+ break;
+ if (n)
+ {
+ error ("node has wrong clone list");
+ error_found = true;
+ }
+ }
+ if ((node->prev_sibling_clone || node->next_sibling_clone) && !node->clone_of)
+ {
+ error ("node is in clone list but it is not clone");
+ error_found = true;
+ }
+ if (!node->prev_sibling_clone && node->clone_of && node->clone_of->clones != node)
+ {
+ error ("node has wrong prev_clone pointer");
+ error_found = true;
+ }
+ if (node->prev_sibling_clone && node->prev_sibling_clone->next_sibling_clone != node)
+ {
+ error ("double linked list of clones corrupted");
+ error_found = true;
+ }
+
+ if (node->analyzed && gimple_has_body_p (node->decl)
&& !TREE_ASM_WRITTEN (node->decl)
&& (!DECL_EXTERNAL (node->decl) || node->global.inlined_to))
{
debug_gimple_stmt (stmt);
error_found = true;
}
- if (e->callee->decl != cgraph_node (decl)->decl
- && e->inline_failed)
+ if (!clone_of_p (cgraph_node (decl), e->callee)
+ && !e->callee->global.inlined_to)
{
error ("edge points to wrong declaration:");
debug_tree (e->callee->decl);
if (!e->aux && !e->indirect_call)
{
error ("edge %s->%s has no corresponding call_stmt",
- cgraph_node_name (e->caller),
- cgraph_node_name (e->callee));
+ identifier_to_locale (cgraph_node_name (e->caller)),
+ identifier_to_locale (cgraph_node_name (e->callee)));
debug_gimple_stmt (e->call_stmt);
error_found = true;
}
{
fprintf (cgraph_dump_file, "Initial entry points:");
for (node = cgraph_nodes; node != first_analyzed; node = node->next)
- if (node->needed && gimple_body (node->decl))
+ if (node->needed)
fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
fprintf (cgraph_dump_file, "\n");
}
if (!edge->callee->reachable)
cgraph_mark_reachable_node (edge->callee);
+ /* If decl is a clone of an abstract function, mark that abstract
+ function so that we don't release its body. The DECL_INITIAL() of that
+ abstract function declaration will be later needed to output debug info. */
+ if (DECL_ABSTRACT_ORIGIN (decl))
+ {
+ struct cgraph_node *origin_node = cgraph_node (DECL_ABSTRACT_ORIGIN (decl));
+ origin_node->abstract_and_needed = true;
+ }
+
/* We finalize local static variables during constructing callgraph
edges. Process their attributes too. */
process_function_and_variable_attributes (first_processed,
{
fprintf (cgraph_dump_file, "Unit entry points:");
for (node = cgraph_nodes; node != first_analyzed; node = node->next)
- if (node->needed && gimple_body (node->decl))
+ if (node->needed)
fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
fprintf (cgraph_dump_file, "\n\nInitial ");
dump_cgraph (cgraph_dump_file);
tree decl = node->decl;
next = node->next;
- if (node->local.finalized && !gimple_body (decl))
+ if (node->local.finalized && !gimple_has_body_p (decl))
cgraph_reset_node (node);
- if (!node->reachable && gimple_body (decl))
+ if (!node->reachable && gimple_has_body_p (decl))
{
if (cgraph_dump_file)
fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
}
else
node->next_needed = NULL;
- gcc_assert (!node->local.finalized || gimple_body (decl));
+ gcc_assert (!node->local.finalized || gimple_has_body_p (decl));
gcc_assert (node->analyzed == node->local.finalized);
}
if (cgraph_dump_file)
cgraph_analyze_functions ();
timevar_pop (TV_CGRAPH);
}
+
+
/* Figure out what functions we want to assemble. */
static void
tree decl = node->decl;
struct cgraph_edge *e;
- gcc_assert (!node->output);
+ gcc_assert (!node->process);
for (e = node->callers; e; e = e->next_caller)
if (e->inline_failed)
/* We need to output all local functions that are used and not
always inlined, as well as those that are reachable from
outside the current compilation unit. */
- if (gimple_body (decl)
+ if (node->analyzed
&& !node->global.inlined_to
&& (node->needed
|| (e && node->reachable))
&& !TREE_ASM_WRITTEN (decl)
&& !DECL_EXTERNAL (decl))
- node->output = 1;
+ node->process = 1;
else
{
/* We should've reclaimed all functions that are not needed. */
#ifdef ENABLE_CHECKING
if (!node->global.inlined_to
- && gimple_body (decl)
+ && gimple_has_body_p (decl)
&& !DECL_EXTERNAL (decl))
{
dump_cgraph_node (stderr, node);
}
#endif
gcc_assert (node->global.inlined_to
- || !gimple_body (decl)
+ || !gimple_has_body_p (decl)
|| DECL_EXTERNAL (decl));
}
gcc_assert (!node->global.inlined_to);
announce_function (decl);
+ node->process = 0;
gcc_assert (node->lowered);
/* Generate RTL for the body of DECL. */
- if (lang_hooks.callgraph.emit_associated_thunks)
+ if (lang_hooks.callgraph.emit_associated_thunks
+ && node->finalized_by_frontend)
lang_hooks.callgraph.emit_associated_thunks (decl);
tree_rest_of_compilation (decl);
/* Make sure that BE didn't give up on compiling. */
- /* ??? Can happen with nested function of extern inline. */
gcc_assert (TREE_ASM_WRITTEN (decl));
current_function_decl = NULL;
- if (!cgraph_preserve_function_body_p (decl))
- {
- cgraph_release_function_body (node);
- /* Eliminate all call edges. This is important so the call_expr no longer
- points to the dead function body. */
- cgraph_node_remove_callees (node);
- }
+ gcc_assert (!cgraph_preserve_function_body_p (decl));
+ cgraph_release_function_body (node);
+ /* Eliminate all call edges. This is important so the GIMPLE_CALL no longer
+ points to the dead function body. */
+ cgraph_node_remove_callees (node);
cgraph_function_flags_ready = true;
}
/* Return true when CALLER_DECL should be inlined into CALLEE_DECL. */
bool
-cgraph_inline_p (struct cgraph_edge *e, const char **reason)
+cgraph_inline_p (struct cgraph_edge *e, cgraph_inline_failed_t *reason)
{
*reason = e->inline_failed;
return !e->inline_failed;
/* Garbage collector may remove inline clones we eliminate during
optimization. So we must be sure to not reference them. */
for (i = 0; i < order_pos; i++)
- if (order[i]->output)
+ if (order[i]->process)
order[new_order_pos++] = order[i];
for (i = new_order_pos - 1; i >= 0; i--)
{
node = order[i];
- if (node->output)
+ if (node->process)
{
gcc_assert (node->reachable);
- node->output = 0;
+ node->process = 0;
cgraph_expand_function (node);
}
}
/* This is used to sort the node types by the cgraph order number. */
+enum cgraph_order_sort_kind
+{
+ ORDER_UNDEFINED = 0,
+ ORDER_FUNCTION,
+ ORDER_VAR,
+ ORDER_ASM
+};
+
struct cgraph_order_sort
{
- enum { ORDER_UNDEFINED = 0, ORDER_FUNCTION, ORDER_VAR, ORDER_ASM } kind;
+ enum cgraph_order_sort_kind kind;
union
{
struct cgraph_node *f;
for (pf = cgraph_nodes; pf; pf = pf->next)
{
- if (pf->output)
+ if (pf->process)
{
i = pf->order;
gcc_assert (nodes[i].kind == ORDER_UNDEFINED);
switch (nodes[i].kind)
{
case ORDER_FUNCTION:
- nodes[i].u.f->output = 0;
+ nodes[i].u.f->process = 0;
cgraph_expand_function (nodes[i].u.f);
break;
gcc_assert (cgraph_global_info_ready);
/* Look if there is any clone around. */
- for (node = cgraph_node (decl); node; node = node->next_clone)
- if (node->global.inlined_to)
- return true;
+ node = cgraph_node (decl);
+ if (node->clones)
+ return true;
return false;
}
gimple_register_cfg_hooks ();
bitmap_obstack_initialize (NULL);
execute_ipa_pass_list (all_ipa_passes);
+
+ /* Generate coverage variables and constructors. */
+ coverage_finish ();
+
+ /* Process new functions added. */
+ set_cfun (NULL);
+ current_function_decl = NULL;
+ cgraph_process_new_functions ();
+
bitmap_obstack_release (NULL);
}
verify_cgraph ();
#endif
+ cgraph_materialize_all_clones ();
cgraph_mark_functions_to_output ();
cgraph_state = CGRAPH_STATE_EXPANSION;
varpool_assemble_pending_decls ();
}
- varpool_output_debug_info ();
cgraph_process_new_functions ();
cgraph_state = CGRAPH_STATE_FINISHED;
for (node = cgraph_nodes; node; node = node->next)
if (node->analyzed
&& (node->global.inlined_to
- || gimple_body (node->decl)))
+ || gimple_has_body_p (node->decl)))
{
error_found = true;
dump_cgraph_node (stderr, node);
resdecl = build_decl (RESULT_DECL, NULL_TREE, void_type_node);
DECL_ARTIFICIAL (resdecl) = 1;
DECL_RESULT (decl) = resdecl;
+ DECL_CONTEXT (resdecl) = decl;
allocate_struct_function (decl, false);
/* Update the call expr on the edges to call the new version. */
for (e = new_version->callers; e; e = e->next_caller)
- gimple_call_set_fndecl (e->call_stmt, new_version->decl);
+ {
+ struct function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl);
+ gimple_call_set_fndecl (e->call_stmt, new_version->decl);
+ /* Update EH information too, just in case. */
+ if (!stmt_could_throw_p (e->call_stmt)
+ && lookup_stmt_eh_region_fn (inner_function, e->call_stmt))
+ remove_stmt_from_eh_region_fn (inner_function, e->call_stmt);
+ }
}
TREE_MAP is a mapping of tree nodes we want to replace with
new ones (according to results of prior analysis).
OLD_VERSION_NODE is the node that is versioned.
- It returns the new version's cgraph node. */
+ It returns the new version's cgraph node.
+ ARGS_TO_SKIP lists arguments to be omitted from functions
+ */
struct cgraph_node *
cgraph_function_versioning (struct cgraph_node *old_version_node,
VEC(cgraph_edge_p,heap) *redirect_callers,
- varray_type tree_map)
+ VEC (ipa_replace_map_p,gc)* tree_map,
+ bitmap args_to_skip)
{
tree old_decl = old_version_node->decl;
struct cgraph_node *new_version_node = NULL;
/* Make a new FUNCTION_DECL tree node for the
new version. */
- new_decl = copy_node (old_decl);
+ if (!args_to_skip)
+ new_decl = copy_node (old_decl);
+ else
+ new_decl = build_function_decl_skip_args (old_decl, args_to_skip);
/* Create the new version's call-graph node.
and update the edges of the new node. */
redirect_callers);
/* Copy the OLD_VERSION_NODE function tree to the new version. */
- tree_function_versioning (old_decl, new_decl, tree_map, false);
- /* Update the call_expr on the edges to call the new version node. */
- update_call_expr (new_version_node);
+ tree_function_versioning (old_decl, new_decl, tree_map, false, args_to_skip);
/* Update the new version's properties.
- Make The new version visible only within this translation unit.
+ Make The new version visible only within this translation unit. Make sure
+ that is not weak also.
??? We cannot use COMDAT linkage because there is no
ABI support for this. */
DECL_EXTERNAL (new_version_node->decl) = 0;
DECL_ONE_ONLY (new_version_node->decl) = 0;
TREE_PUBLIC (new_version_node->decl) = 0;
DECL_COMDAT (new_version_node->decl) = 0;
+ DECL_WEAK (new_version_node->decl) = 0;
+ DECL_VIRTUAL_P (new_version_node->decl) = 0;
new_version_node->local.externally_visible = 0;
new_version_node->local.local = 1;
new_version_node->lowered = true;
+
+ /* Update the call_expr on the edges to call the new version node. */
+ update_call_expr (new_version_node);
+
+ cgraph_call_function_insertion_hooks (new_version_node);
return new_version_node;
}
struct cgraph_node *
save_inline_function_body (struct cgraph_node *node)
{
- struct cgraph_node *first_clone;
+ struct cgraph_node *first_clone, *n;
gcc_assert (node == cgraph_node (node->decl));
cgraph_lower_function (node);
- first_clone = node->next_clone;
+ first_clone = node->clones;
first_clone->decl = copy_node (node->decl);
- node->next_clone = NULL;
- first_clone->prev_clone = NULL;
cgraph_insert_node_to_hashtable (first_clone);
gcc_assert (first_clone == cgraph_node (first_clone->decl));
+ if (first_clone->next_sibling_clone)
+ {
+ for (n = first_clone->next_sibling_clone; n->next_sibling_clone; n = n->next_sibling_clone)
+ n->clone_of = first_clone;
+ n->clone_of = first_clone;
+ n->next_sibling_clone = first_clone->clones;
+ if (first_clone->clones)
+ first_clone->clones->prev_sibling_clone = n;
+ first_clone->clones = first_clone->next_sibling_clone;
+ first_clone->next_sibling_clone->prev_sibling_clone = NULL;
+ first_clone->next_sibling_clone = NULL;
+ gcc_assert (!first_clone->prev_sibling_clone);
+ }
+ first_clone->clone_of = NULL;
+ node->clones = NULL;
+
+ if (first_clone->clones)
+ for (n = first_clone->clones; n != first_clone;)
+ {
+ gcc_assert (n->decl == node->decl);
+ n->decl = first_clone->decl;
+ if (n->clones)
+ n = n->clones;
+ else if (n->next_sibling_clone)
+ n = n->next_sibling_clone;
+ else
+ {
+ while (n != first_clone && !n->next_sibling_clone)
+ n = n->clone_of;
+ if (n != first_clone)
+ n = n->next_sibling_clone;
+ }
+ }
/* Copy the OLD_VERSION_NODE function tree to the new version. */
- tree_function_versioning (node->decl, first_clone->decl, NULL, true);
+ tree_function_versioning (node->decl, first_clone->decl, NULL, true, NULL);
DECL_EXTERNAL (first_clone->decl) = 0;
DECL_ONE_ONLY (first_clone->decl) = 0;
TREE_PUBLIC (first_clone->decl) = 0;
DECL_COMDAT (first_clone->decl) = 0;
+ VEC_free (ipa_opt_pass, heap,
+ DECL_STRUCT_FUNCTION (first_clone->decl)->ipa_transforms_to_apply);
+ DECL_STRUCT_FUNCTION (first_clone->decl)->ipa_transforms_to_apply = NULL;
- for (node = first_clone->next_clone; node; node = node->next_clone)
- node->decl = first_clone->decl;
#ifdef ENABLE_CHECKING
verify_cgraph_node (first_clone);
#endif
return first_clone;
}
+/* Given virtual clone, turn it into actual clone. */
+static void
+cgraph_materialize_clone (struct cgraph_node *node)
+{
+ bitmap_obstack_initialize (NULL);
+ /* Copy the OLD_VERSION_NODE function tree to the new version. */
+ tree_function_versioning (node->clone_of->decl, node->decl,
+ node->clone.tree_map, true,
+ node->clone.args_to_skip);
+
+ /* Function is no longer clone. */
+ if (node->next_sibling_clone)
+ node->next_sibling_clone->prev_sibling_clone = node->prev_sibling_clone;
+ if (node->prev_sibling_clone)
+ node->prev_sibling_clone->next_sibling_clone = node->next_sibling_clone;
+ else
+ node->clone_of->clones = node->next_sibling_clone;
+ node->next_sibling_clone = NULL;
+ node->prev_sibling_clone = NULL;
+ node->clone_of = NULL;
+ bitmap_obstack_release (NULL);
+}
+
+/* Once all functions from compilation unit are in memory, produce all clones
+ and update all calls.
+ We might also do this on demand if we don't want to bring all functions to
+ memory prior compilation, but current WHOPR implementation does that and it is
+ is bit easier to keep everything right in this order. */
+void
+cgraph_materialize_all_clones (void)
+{
+ struct cgraph_node *node;
+ bool stabilized = false;
+
+ if (cgraph_dump_file)
+ fprintf (cgraph_dump_file, "Materializing clones\n");
+#ifdef ENABLE_CHECKING
+ verify_cgraph ();
+#endif
+
+ /* We can also do topological order, but number of iterations should be
+ bounded by number of IPA passes since single IPA pass is probably not
+ going to create clones of clones it created itself. */
+ while (!stabilized)
+ {
+ stabilized = true;
+ for (node = cgraph_nodes; node; node = node->next)
+ {
+ if (node->clone_of && node->decl != node->clone_of->decl
+ && !gimple_has_body_p (node->decl))
+ {
+ if (gimple_has_body_p (node->clone_of->decl))
+ {
+ if (cgraph_dump_file)
+ fprintf (cgraph_dump_file, " clonning %s to %s",
+ cgraph_node_name (node->clone_of),
+ cgraph_node_name (node));
+ cgraph_materialize_clone (node);
+ }
+ else
+ stabilized = false;
+ }
+ }
+ }
+ if (cgraph_dump_file)
+ fprintf (cgraph_dump_file, "Updating call sites\n");
+ for (node = cgraph_nodes; node; node = node->next)
+ if (node->analyzed && gimple_has_body_p (node->decl)
+ && (!node->clone_of || node->clone_of->decl != node->decl))
+ {
+ struct cgraph_edge *e;
+
+ current_function_decl = node->decl;
+ push_cfun (DECL_STRUCT_FUNCTION (node->decl));
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ tree decl = gimple_call_fndecl (e->call_stmt);
+ /* When function gets inlined, indirect inlining might've invented
+ new edge for orginally indirect stmt. Since we are not
+ preserving clones in the original form, we must not update here
+ since other inline clones don't need to contain call to the same
+ call. Inliner will do the substitution for us later. */
+ if (decl && decl != e->callee->decl)
+ {
+ gimple new_stmt;
+ gimple_stmt_iterator gsi;
+
+ if (cgraph_dump_file)
+ {
+ fprintf (cgraph_dump_file, "updating call of %s in %s:",
+ cgraph_node_name (node),
+ cgraph_node_name (e->callee));
+ print_gimple_stmt (cgraph_dump_file, e->call_stmt, 0, dump_flags);
+ }
+
+ if (e->callee->clone.args_to_skip)
+ new_stmt = gimple_call_copy_skip_args (e->call_stmt,
+ e->callee->clone.args_to_skip);
+ else
+ new_stmt = e->call_stmt;
+ if (gimple_vdef (new_stmt)
+ && TREE_CODE (gimple_vdef (new_stmt)) == SSA_NAME)
+ SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
+ gimple_call_set_fndecl (new_stmt, e->callee->decl);
+
+ gsi = gsi_for_stmt (e->call_stmt);
+ gsi_replace (&gsi, new_stmt, true);
+
+ /* Update EH information too, just in case. */
+ if (!stmt_could_throw_p (new_stmt)
+ && lookup_stmt_eh_region (new_stmt))
+ remove_stmt_from_eh_region (new_stmt);
+
+ cgraph_set_call_stmt_including_clones (node, e->call_stmt, new_stmt);
+
+ if (cgraph_dump_file)
+ {
+ fprintf (cgraph_dump_file, " updated to:");
+ print_gimple_stmt (cgraph_dump_file, e->call_stmt, 0, dump_flags);
+ }
+ }
+ }
+ pop_cfun ();
+ current_function_decl = NULL;
+#ifdef ENABLE_CHECKING
+ verify_cgraph_node (node);
+#endif
+ }
+#ifdef ENABLE_CHECKING
+ verify_cgraph ();
+#endif
+ cgraph_remove_unreachable_nodes (false, cgraph_dump_file);
+}
+
#include "gt-cgraphunit.h"