#include "fibheap.h"
#include "params.h"
+/* Number of functions identified as candidates for cloning. When not cloning
+ we can simplify iterate stage not forcing it to go through the decision
+ on what is profitable and what not. */
+static int n_cloning_candidates;
+
+/* Maximal count found in program. */
+static gcov_type max_count;
+
+/* Cgraph nodes that has been completely replaced by cloning during iterate
+ * stage and will be removed after ipcp is finished. */
+static bitmap dead_nodes;
+
+static void ipcp_print_profile_data (FILE *);
+static void ipcp_function_scale_print (FILE *);
+
/* Get the original node field of ipa_node_params associated with node NODE. */
static inline struct cgraph_node *
ipcp_get_orig_node (struct cgraph_node *node)
struct cgraph_node *new_node)
{
ipa_check_create_node_params ();
+ ipa_initialize_node_params (new_node);
IPA_NODE_REF (new_node)->ipcp_orig_node = orig_node;
- ipa_count_formal_params (new_node);
- ipa_create_param_decls_array (new_node);
}
/* Perform intraprocedrual analysis needed for ipcp. */
/* Unreachable nodes should have been eliminated before ipcp. */
gcc_assert (node->needed || node->reachable);
- ipa_count_formal_params (node);
- ipa_create_param_decls_array (node);
+ ipa_initialize_node_params (node);
ipa_detect_param_modifications (node);
}
/* Recompute all local information since node might've got new
- direct calls after clonning. */
+ direct calls after cloning. */
static void
ipcp_update_cloned_node (struct cgraph_node *new_node)
{
/* Return the lattice corresponding to the Ith formal parameter of the function
described by INFO. */
static inline struct ipcp_lattice *
-ipcp_get_ith_lattice (struct ipa_node_params *info, int i)
+ipcp_get_lattice (struct ipa_node_params *info, int i)
{
- return &(info->ipcp_lattices[i]);
+ return &(info->params[i].ipcp_lattice);
}
/* Given the jump function JFUNC, compute the lattice LAT that describes the
{
struct ipcp_lattice *caller_lat;
- caller_lat = ipcp_get_ith_lattice (info, jfunc->value.formal_id);
+ caller_lat = ipcp_get_lattice (info, jfunc->value.formal_id);
lat->type = caller_lat->type;
lat->constant = caller_lat->constant;
}
struct cgraph_node *node;
int i, count;
- fprintf (f, "\nLATTICE PRINT\n");
+ fprintf (f, "\nLattice:\n");
for (node = cgraph_nodes; node; node = node->next)
{
struct ipa_node_params *info;
if (!node->analyzed)
continue;
info = IPA_NODE_REF (node);
- fprintf (f, "Printing lattices %s:\n", cgraph_node_name (node));
+ fprintf (f, " Node: %s:\n", cgraph_node_name (node));
count = ipa_get_param_count (info);
for (i = 0; i < count; i++)
{
- struct ipcp_lattice *lat = ipcp_get_ith_lattice (info, i);
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
- fprintf (f, " param [%d]: ", i);
+ fprintf (f, " param [%d]: ", i);
if (lat->type == IPA_CONST_VALUE)
{
fprintf (f, "type is CONST ");
}
}
+/* Return true if this NODE is viable candidate for cloning. */
+static bool
+ipcp_cloning_candidate_p (struct cgraph_node *node)
+{
+ int n_calls = 0;
+ int n_hot_calls = 0;
+ gcov_type direct_call_sum = 0;
+ struct cgraph_edge *e;
+
+ /* We never clone functions that are not visible from outside.
+ FIXME: in future we should clone such functions when they are called with
+ different constants, but current ipcp implementation is not good on this.
+ */
+ if (!node->needed || !node->analyzed)
+ return false;
+
+ if (cgraph_function_body_availability (node) <= AVAIL_OVERWRITABLE)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; body is overwrittable.\n",
+ cgraph_node_name (node));
+ return false;
+ }
+ if (!tree_versionable_function_p (node->decl))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; body is not versionable.\n",
+ cgraph_node_name (node));
+ return false;
+ }
+ for (e = node->callers; e; e = e->next_caller)
+ {
+ direct_call_sum += e->count;
+ n_calls ++;
+ if (cgraph_maybe_hot_edge_p (e))
+ n_hot_calls ++;
+ }
+
+ if (!n_calls)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; no direct calls.\n",
+ cgraph_node_name (node));
+ return false;
+ }
+ if (node->local.inline_summary.self_insns < n_calls)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Considering %s for cloning; code would shrink.\n",
+ cgraph_node_name (node));
+ return true;
+ }
+
+ if (!flag_ipa_cp_clone)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; -fipa-cp-clone disabled.\n",
+ cgraph_node_name (node));
+ return false;
+ }
+
+ if (!optimize_function_for_speed_p (DECL_STRUCT_FUNCTION (node->decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; optimizing it for size.\n",
+ cgraph_node_name (node));
+ return false;
+ }
+
+ /* When profile is available and function is hot, propagate into it even if
+ calls seems cold; constant propagation can improve function's speed
+ significandly. */
+ if (max_count)
+ {
+ if (direct_call_sum > node->count * 90 / 100)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Considering %s for cloning; usually called directly.\n",
+ cgraph_node_name (node));
+ return true;
+ }
+ }
+ if (!n_hot_calls)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not considering %s for cloning; no hot calls.\n",
+ cgraph_node_name (node));
+ }
+ if (dump_file)
+ fprintf (dump_file, "Considering %s for cloning.\n",
+ cgraph_node_name (node));
+ return true;
+}
+
/* Initialize ipcp_lattices array. The lattices corresponding to supported
types (integers, real types and Fortran constants defined as const_decls)
are initialized to IPA_TOP, the rest of them to IPA_BOTTOM. */
{
int i;
struct ipa_node_params *info = IPA_NODE_REF (node);
+ enum ipa_lattice_type type;
- info->ipcp_lattices = XCNEWVEC (struct ipcp_lattice,
- ipa_get_param_count (info));
-
+ if (ipa_is_called_with_var_arguments (info))
+ type = IPA_BOTTOM;
+ else if (!node->needed)
+ type = IPA_TOP;
/* When cloning is allowed, we can assume that externally visible functions
are not called. We will compensate this by cloning later. */
- if (flag_ipa_cp_clone || !node->needed)
- for (i = 0; i < ipa_get_param_count (info) ; i++)
- ipcp_get_ith_lattice (info, i)->type = IPA_TOP;
+ else if (ipcp_cloning_candidate_p (node))
+ type = IPA_TOP, n_cloning_candidates ++;
else
- for (i = 0; i < ipa_get_param_count (info) ; i++)
- ipcp_get_ith_lattice (info, i)->type = IPA_BOTTOM;
+ type = IPA_BOTTOM;
+
+ for (i = 0; i < ipa_get_param_count (info) ; i++)
+ ipcp_get_lattice (info, i)->type = type;
}
/* build INTEGER_CST tree with type TREE_TYPE and value according to LAT.
for (node = cgraph_nodes; node; node = node->next)
if (node->analyzed)
- {
- ipcp_analyze_node (node);
- ipcp_initialize_node_lattices (node);
- ipcp_compute_node_scale (node);
- }
+ ipcp_analyze_node (node);
for (node = cgraph_nodes; node; node = node->next)
{
if (!node->analyzed)
count = ipa_get_param_count (info);
for (i = 0; i < count; i++)
{
- struct ipcp_lattice *lat = ipcp_get_ith_lattice (info, i);
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
if (lat->type == IPA_TOP)
{
prop_again = true;
+ if (dump_file)
+ {
+ fprintf (dump_file, "Forcing param ");
+ print_generic_expr (dump_file, ipa_get_param (info, i), 0);
+ fprintf (dump_file, " of node %s to bottom.\n",
+ cgraph_node_name (node));
+ }
lat->type = IPA_BOTTOM;
}
}
ipa_check_create_node_params ();
ipa_check_create_edge_args ();
+
/* Initialize worklist to contain all functions. */
wl = ipa_init_func_list ();
while (wl)
{
jump_func = ipa_get_ith_jump_func (args, i);
ipcp_lattice_from_jfunc (info, &inc_lat, jump_func);
- dest_lat = ipcp_get_ith_lattice (callee_info, i);
+ dest_lat = ipcp_get_lattice (callee_info, i);
ipa_lattice_meet (&new_lat, &inc_lat, dest_lat);
if (ipcp_lattice_changed (&new_lat, dest_lat))
{
static void
ipcp_iterate_stage (void)
{
+ struct cgraph_node *node;
+ n_cloning_candidates = 0;
+
+ if (dump_file)
+ fprintf (dump_file, "\nIPA iterate stage:\n\n");
+ for (node = cgraph_nodes; node; node = node->next)
+ {
+ ipcp_initialize_node_lattices (node);
+ ipcp_compute_node_scale (node);
+ }
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ ipcp_print_all_lattices (dump_file);
+ ipcp_function_scale_print (dump_file);
+ }
+
ipcp_propagate_stage ();
if (ipcp_change_tops_to_bottom ())
/* Some lattices have changed from IPA_TOP to IPA_BOTTOM.
This change should be propagated. */
- ipcp_propagate_stage ();
+ {
+ gcc_assert (n_cloning_candidates);
+ ipcp_propagate_stage ();
+ }
+ if (dump_file)
+ {
+ fprintf (dump_file, "\nIPA lattices after propagation:\n");
+ ipcp_print_all_lattices (dump_file);
+ if (dump_flags & TDF_DETAILS)
+ ipcp_print_profile_data (dump_file);
+ }
}
/* Check conditions to forbid constant insertion to function described by
NODE. */
static inline bool
-ipcp_node_not_modifiable_p (struct cgraph_node *node)
+ipcp_node_modifiable_p (struct cgraph_node *node)
{
- /* ??? Handle pending sizes case. */
- if (DECL_UNINLINABLE (node->decl))
- return true;
- return false;
+ /* Once we will be able to do in-place replacement, we can be more
+ lax here. */
+ return tree_versionable_function_p (node->decl);
}
/* Print count scale data structures. */
}
}
-/* Print all IPCP data structures to F. */
-static void
-ipcp_print_all_structures (FILE * f)
-{
- ipcp_print_all_lattices (f);
- ipcp_function_scale_print (f);
- ipa_print_all_tree_maps (f);
- ipa_print_all_param_flags (f);
- ipa_print_all_jump_functions (f);
-}
-
/* Print profile info for all functions. */
static void
ipcp_print_profile_data (FILE * f)
tree const_val;
replace_map = XCNEW (struct ipa_replace_map);
- if (dump_file)
- fprintf (dump_file, "replacing param with const\n");
const_val = build_const_val (lat, TREE_TYPE (parm_tree));
+ if (dump_file)
+ {
+ fprintf (dump_file, " replacing param ");
+ print_generic_expr (dump_file, parm_tree, 0);
+ fprintf (dump_file, " with const ");
+ print_generic_expr (dump_file, const_val, 0);
+ fprintf (dump_file, "\n");
+ }
replace_map->old_tree = parm_tree;
replace_map->new_tree = const_val;
replace_map->replace_p = true;
struct ipa_jump_func *jump_func;
struct cgraph_node *node = cs->callee, *orig;
+ if (!n_cloning_candidates)
+ return false;
+
if ((orig = ipcp_get_orig_node (node)) != NULL)
node = orig;
if (ipcp_get_orig_node (cs->caller))
count = ipa_get_param_count (orig_callee_info);
for (i = 0; i < count; i++)
{
- struct ipcp_lattice *lat = ipcp_get_ith_lattice (orig_callee_info, i);
+ struct ipcp_lattice *lat = ipcp_get_lattice (orig_callee_info, i);
if (ipcp_lat_is_const (lat))
{
jump_func = ipa_get_ith_jump_func (IPA_EDGE_REF (cs), i);
static void
ipcp_update_callgraph (void)
{
- struct cgraph_node *node, *orig_callee;
- struct cgraph_edge *cs;
+ struct cgraph_node *node;
for (node = cgraph_nodes; node; node = node->next)
- {
- /* want to fix only original nodes */
- if (!node->analyzed || ipcp_node_is_clone (node))
- continue;
- for (cs = node->callees; cs; cs = cs->next_callee)
- if (ipcp_node_is_clone (cs->callee))
+ if (node->analyzed && ipcp_node_is_clone (node))
+ {
+ bitmap args_to_skip = BITMAP_ALLOC (NULL);
+ struct cgraph_node *orig_node = ipcp_get_orig_node (node);
+ struct ipa_node_params *info = IPA_NODE_REF (orig_node);
+ int i, count = ipa_get_param_count (info);
+ struct cgraph_edge *cs, *next;
+
+ for (i = 0; i < count; i++)
+ {
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
+ tree parm_tree = ipa_get_param (info, i);
+
+ /* We can proactively remove obviously unused arguments. */
+ if (is_gimple_reg (parm_tree)
+ && !gimple_default_def (DECL_STRUCT_FUNCTION (orig_node->decl),
+ parm_tree))
+ {
+ bitmap_set_bit (args_to_skip, i);
+ continue;
+ }
+
+ if (lat->type == IPA_CONST_VALUE)
+ bitmap_set_bit (args_to_skip, i);
+ }
+ for (cs = node->callers; cs; cs = next)
{
- /* Callee is a cloned node */
- orig_callee = ipcp_get_orig_node (cs->callee);
- if (ipcp_need_redirect_p (cs))
+ next = cs->next_caller;
+ if (ipcp_node_is_clone (cs->caller) || !ipcp_need_redirect_p (cs))
{
- cgraph_redirect_edge_callee (cs, orig_callee);
- gimple_call_set_fndecl (cs->call_stmt, orig_callee->decl);
+ gimple new_stmt;
+ gimple_stmt_iterator gsi;
+
+ current_function_decl = cs->caller->decl;
+ push_cfun (DECL_STRUCT_FUNCTION (cs->caller->decl));
+
+ new_stmt = gimple_call_copy_skip_args (cs->call_stmt,
+ args_to_skip);
+ gsi = gsi_for_stmt (cs->call_stmt);
+ gsi_replace (&gsi, new_stmt, true);
+ cgraph_set_call_stmt (cs, new_stmt);
+ pop_cfun ();
+ current_function_decl = NULL;
+ }
+ else
+ {
+ cgraph_redirect_edge_callee (cs, orig_node);
+ gimple_call_set_fndecl (cs->call_stmt, orig_node->decl);
}
}
- }
+ }
}
/* Update all cfg basic blocks in NODE according to SCALE. */
}
}
-/* Maximal count found in program. */
-static gcov_type max_count;
-bitmap dead_nodes;
-
-/* Return true if original clone needs to be preserved. */
-static bool
-ipcp_need_original_clone_p (struct cgraph_node *node)
+/* If NODE was cloned, how much would program grow? */
+static long
+ipcp_estimate_growth (struct cgraph_node *node)
{
- struct cgraph_edge *e;
+ struct cgraph_edge *cs;
+ int redirectable_node_callers = 0;
+ int removable_args = 0;
+ bool need_original = node->needed;
+ struct ipa_node_params *info;
+ int i, count;
+ int growth;
- if (node->needed)
- return true;
- for (e = node->callers; e; e = e->next_caller)
- if (!bitmap_bit_p (dead_nodes, e->caller->uid)
- && ipcp_need_redirect_p (e))
- return true;
+ for (cs = node->callers; cs != NULL; cs = cs->next_caller)
+ if (cs->caller == node || !ipcp_need_redirect_p (cs))
+ redirectable_node_callers++;
+ else
+ need_original = true;
+
+ /* If we will be able to fully replace orignal node, we never increase
+ program size. */
+ if (!need_original)
+ return 0;
+
+ info = IPA_NODE_REF (node);
+ count = ipa_get_param_count (info);
+ for (i = 0; i < count; i++)
+ {
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
+ tree parm_tree = ipa_get_param (info, i);
- return false;
+ /* We can proactively remove obviously unused arguments. */
+ if (is_gimple_reg (parm_tree)
+ && !gimple_default_def (DECL_STRUCT_FUNCTION (node->decl),
+ parm_tree))
+ removable_args++;
+
+ if (lat->type == IPA_CONST_VALUE)
+ removable_args++;
+ }
+
+ /* We make just very simple estimate of savings for removal of operand from
+ call site. Precise cost is dificult to get, as our size metric counts
+ constants and moves as free. Generally we are looking for cases that
+ small function is called very many times. */
+ growth = node->local.inline_summary.self_insns
+ - removable_args * redirectable_node_callers;
+ if (growth < 0)
+ return 0;
+ return growth;
}
+
/* Estimate cost of cloning NODE. */
static long
ipcp_estimate_cloning_cost (struct cgraph_node *node)
struct cgraph_edge *e;
int cost;
- /* When we don't need original clone; we should always propagate. */
- if (!ipcp_need_original_clone_p (node))
+ cost = ipcp_estimate_growth (node) * 1000;
+ if (!cost)
{
if (dump_file)
- fprintf (dump_file, "Function %s can be fully propagated\n",
- cgraph_node_name (node));
+ fprintf (dump_file, "Versioning of %s will save code size\n",
+ cgraph_node_name (node));
return 0;
}
freq_sum += e->frequency + 1;
}
- cost = node->local.inline_summary.self_insns * 1000;
if (max_count)
cost /= count_sum * 1000 / max_count + 1;
else
for (i = 0; i < count; i++)
{
- struct ipcp_lattice *lat = ipcp_get_ith_lattice (info, i);
- tree parm_tree = ipa_get_ith_param (info, i);
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
+ tree parm_tree = ipa_get_param (info, i);
if (ipcp_lat_is_insertable (lat)
/* Do not count obviously unused arguments. */
&& (!is_gimple_reg (parm_tree)
int i;
VEC (cgraph_edge_p, heap) * redirect_callers;
varray_type replace_trees;
- struct cgraph_edge *cs;
int node_callers, count;
tree parm_tree;
struct ipa_replace_map *replace_param;
ipa_check_create_node_params ();
ipa_check_create_edge_args ();
+ if (dump_file)
+ fprintf (dump_file, "\nIPA insert stage:\n\n");
dead_nodes = BITMAP_ALLOC (NULL);
{
struct ipa_node_params *info;
/* Propagation of the constant is forbidden in certain conditions. */
- if (!node->analyzed || ipcp_node_not_modifiable_p (node))
+ if (!node->analyzed || !ipcp_node_modifiable_p (node))
continue;
info = IPA_NODE_REF (node);
if (ipa_is_called_with_var_arguments (info))
{
struct ipa_node_params *info;
int growth = 0;
+ bitmap args_to_skip;
+ struct cgraph_edge *cs;
node = (struct cgraph_node *)fibheap_extract_min (heap);
node->aux = NULL;
fprintf (dump_file, "considering function %s\n",
cgraph_node_name (node));
- if (ipcp_need_original_clone_p (node))
- growth = node->local.inline_summary.self_insns;
- else
- bitmap_set_bit (dead_nodes, node->uid);
+ growth = ipcp_estimate_growth (node);
if (new_insns + growth > max_new_insns)
break;
new_insns += growth;
+ /* Look if original function becomes dead after clonning. */
+ for (cs = node->callers; cs != NULL; cs = cs->next_caller)
+ if (cs->caller == node || ipcp_need_redirect_p (cs))
+ break;
+ if (!cs && !node->needed)
+ bitmap_set_bit (dead_nodes, node->uid);
+
info = IPA_NODE_REF (node);
count = ipa_get_param_count (info);
VARRAY_GENERIC_PTR_INIT (replace_trees, ipcp_const_param_count (node),
"replace_trees");
+ args_to_skip = BITMAP_ALLOC (NULL);
for (i = 0; i < count; i++)
{
- struct ipcp_lattice *lat = ipcp_get_ith_lattice (info, i);
- parm_tree = ipa_get_ith_param (info, i);
-
- if (lat->type == IPA_CONST_VALUE
- /* Do not count obviously unused arguments. */
- && (!is_gimple_reg (parm_tree)
- || gimple_default_def (DECL_STRUCT_FUNCTION (node->decl),
- parm_tree)))
+ struct ipcp_lattice *lat = ipcp_get_lattice (info, i);
+ parm_tree = ipa_get_param (info, i);
+
+ /* We can proactively remove obviously unused arguments. */
+ if (is_gimple_reg (parm_tree)
+ && !gimple_default_def (DECL_STRUCT_FUNCTION (node->decl),
+ parm_tree))
+ {
+ bitmap_set_bit (args_to_skip, i);
+ continue;
+ }
+
+ if (lat->type == IPA_CONST_VALUE)
{
replace_param =
ipcp_create_replace_map (parm_tree, lat);
VARRAY_PUSH_GENERIC_PTR (replace_trees, replace_param);
+ bitmap_set_bit (args_to_skip, i);
}
}
/* Redirecting all the callers of the node to the
new versioned node. */
node1 =
- cgraph_function_versioning (node, redirect_callers, replace_trees);
+ cgraph_function_versioning (node, redirect_callers, replace_trees,
+ args_to_skip);
+ BITMAP_FREE (args_to_skip);
VEC_free (cgraph_edge_p, heap, redirect_callers);
VARRAY_CLEAR (replace_trees);
if (node1 == NULL)
static unsigned int
ipcp_driver (void)
{
- /* 2. Do the interprocedural propagation. */
- ipcp_iterate_stage ();
+ cgraph_remove_unreachable_nodes (true,dump_file);
if (dump_file)
{
- fprintf (dump_file, "\nIPA structures after propagation:\n");
- ipcp_print_all_structures (dump_file);
- fprintf (dump_file, "\nProfiling info before insert stage:\n");
- ipcp_print_profile_data (dump_file);
+ fprintf (dump_file, "\nIPA structures before propagation:\n");
+ if (dump_flags & TDF_DETAILS)
+ ipa_print_all_params (dump_file);
+ ipa_print_all_jump_functions (dump_file);
}
+ /* 2. Do the interprocedural propagation. */
+ ipcp_iterate_stage ();
/* 3. Insert the constants found to the functions. */
ipcp_insert_stage ();
- if (dump_file)
+ if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "\nProfiling info after insert stage:\n");
ipcp_print_profile_data (dump_file);
free_all_ipa_structures_after_ipa_cp ();
if (dump_file)
fprintf (dump_file, "\nIPA constant propagation end\n");
- cgraph_remove_unreachable_nodes (true, NULL);
return 0;
}
/* 1. Call the init stage to initialize
the ipa_node_params and ipa_edge_args structures. */
ipcp_init_stage ();
- if (dump_file)
- {
- fprintf (dump_file, "\nIPA structures before propagation:\n");
- ipcp_print_all_structures (dump_file);
- }
}
/* Gate for IPCP optimization. */
PROP_trees, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
- TODO_dump_cgraph | TODO_dump_func /* todo_flags_finish */
+ TODO_dump_cgraph | TODO_dump_func |
+ TODO_remove_functions /* todo_flags_finish */
},
ipcp_generate_summary, /* generate_summary */
NULL, /* write_summary */