* cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.

	* cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
	callees.
	* cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
	* ipa-inline.c (update_noncloned_frequencies): New function.
	(cgraph_clone_inlined_nodes): Use it.
	* tree-inline.c (copy_bb): Fix frequency scaling; output
	diagnostic on frequency mismatches to dump file.
	(initialize_cfun): Do not scale frequency; fix count scaling;
	initialize entry and exit block frequencies; copy profile
	info.
	(copy_cfg_body): Use frequency_scale as argument;
	fix count scaling.
	(copy_body): Use frequency_scale as argument.
	(expand_call_inline): Compute frequency scale and output diagnostic
	to dump file.
	(delete_unreachable_blocks_update_callgrah): Remove checking that
	has to be done after edge redirection.
	(tree_function_versioning): Update initialize_cfun and copy_body call.

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@154205 138bc75d-0d04-0410-961f-82ee72b054a4

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 9796fa0..bee2ebc 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,5 +1,27 @@
 2009-11-14  Jan Hubicka  <jh@suse.cz>
 
 2009-11-14  Jan Hubicka  <jh@suse.cz>
 

+       * cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.
+       * cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
+       callees.
+       * cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
+       * ipa-inline.c (update_noncloned_frequencies): New function.
+       (cgraph_clone_inlined_nodes): Use it.
+       * tree-inline.c (copy_bb): Fix frequency scaling; output
+       diagnostic on frequency mismatches to dump file.
+       (initialize_cfun): Do not scale frequency; fix count scaling;
+       initialize entry and exit block frequencies; copy profile
+       info.
+       (copy_cfg_body): Use frequency_scale as argument;
+       fix count scaling.
+       (copy_body): Use frequency_scale as argument.
+       (expand_call_inline): Compute frequency scale and output diagnostic
+       to dump file.
+       (delete_unreachable_blocks_update_callgrah): Remove checking that
+       has to be done after edge redirection.
+       (tree_function_versioning): Update initialize_cfun and copy_body call.
+
+2009-11-14  Jan Hubicka  <jh@suse.cz>
+

        * cgraph.c (cgraph_release_function_body): Update use of
        ipa_transforms_to_apply.
        (cgraph_remove_node): Remove ipa_transforms_to_apply.
        * cgraph.c (cgraph_release_function_body): Update use of
        ipa_transforms_to_apply.
        (cgraph_remove_node): Remove ipa_transforms_to_apply.

diff --git a/gcc/cgraph.c b/gcc/cgraph.c
index 840cf29..f2d7609 100644 (file)
--- a/gcc/cgraph.c
+++ b/gcc/cgraph.c
@@ -1641,8 +1641,12 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
 {
   struct cgraph_edge *new_edge;
   gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
 {
   struct cgraph_edge *new_edge;
   gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;

-  gcov_type freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+  gcov_type freq;

 
 

+  /* We do not want to ignore loop nest after frequency drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;

   if (freq > CGRAPH_FREQ_MAX)
     freq = CGRAPH_FREQ_MAX;
   new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,
   if (freq > CGRAPH_FREQ_MAX)
     freq = CGRAPH_FREQ_MAX;
   new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,

diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c
index 8423c36..0c3bff2 100644 (file)
--- a/gcc/cgraphbuild.c
+++ b/gcc/cgraphbuild.c
@@ -109,7 +109,8 @@ reset_inline_failed (struct cgraph_node *node)
 int
 compute_call_stmt_bb_frequency (tree decl, basic_block bb)
 {
 int
 compute_call_stmt_bb_frequency (tree decl, basic_block bb)
 {

-  int entry_freq = ENTRY_BLOCK_PTR->frequency;
+  int entry_freq = ENTRY_BLOCK_PTR_FOR_FUNCTION
+                    (DECL_STRUCT_FUNCTION (decl))->frequency;

   int freq = bb->frequency;
 
   if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT)
   int freq = bb->frequency;
 
   if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT)

diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c
index 08088dd..e9d8311 100644 (file)
--- a/gcc/ipa-inline.c
+++ b/gcc/ipa-inline.c
@@ -207,6 +207,29 @@ cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to,
   return size;
 }
 
   return size;
 }
 

+/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
+   by NEST.  */
+
+static void
+update_noncloned_frequencies (struct cgraph_node *node,
+                             int freq_scale, int nest)
+{
+  struct cgraph_edge *e;
+
+  /* We do not want to ignore high loop nest after freq drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  for (e = node->callees; e; e = e->next_callee)
+    {
+      e->loop_nest += nest;
+      e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+      if (e->frequency > CGRAPH_FREQ_MAX)
+        e->frequency = CGRAPH_FREQ_MAX;
+      if (!e->inline_failed)
+        update_noncloned_frequencies (e->callee, freq_scale, nest);
+    }
+}
+

 /* E is expected to be an edge being inlined.  Clone destination node of
    the edge and redirect it to the new clone.
    DUPLICATE is used for bookkeeping on whether we are actually creating new
 /* E is expected to be an edge being inlined.  Clone destination node of
    the edge and redirect it to the new clone.
    DUPLICATE is used for bookkeeping on whether we are actually creating new


@@ -234,6 +257,7 @@ cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
            }
          duplicate = false;
          e->callee->local.externally_visible = false;
            }
          duplicate = false;
          e->callee->local.externally_visible = false;

+          update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);

        }
       else
        {
        }
       else
        {

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 629ccfb..34a8e9d 100644 (file)
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -1472,6 +1472,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
   basic_block copy_basic_block;
   tree decl;
   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
   basic_block copy_basic_block;
   tree decl;

+  gcov_type freq;

 
   /* create_basic_block() will append every new block to
      basic_block_info automatically.  */
 
   /* create_basic_block() will append every new block to
      basic_block_info automatically.  */


@@ -1481,11 +1482,12 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 
   /* We are going to rebuild frequencies from scratch.  These values
      have just small importance to drive canonicalize_loop_headers.  */
 
   /* We are going to rebuild frequencies from scratch.  These values
      have just small importance to drive canonicalize_loop_headers.  */

-  copy_basic_block->frequency = ((gcov_type)bb->frequency
-                                * frequency_scale / REG_BR_PROB_BASE);
+  freq = ((gcov_type)bb->frequency * frequency_scale / REG_BR_PROB_BASE);

 
 

-  if (copy_basic_block->frequency > BB_FREQ_MAX)
-    copy_basic_block->frequency = BB_FREQ_MAX;
+  /* We recompute frequencies after inlining, so this is quite safe.  */
+  if (freq > BB_FREQ_MAX)
+    freq = BB_FREQ_MAX;
+  copy_basic_block->frequency = freq;

 
   copy_gsi = gsi_start_bb (copy_basic_block);
 
 
   copy_gsi = gsi_start_bb (copy_basic_block);
 


@@ -1631,10 +1633,34 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
                case CB_CGE_DUPLICATE:
                  edge = cgraph_edge (id->src_node, orig_stmt);
                  if (edge)
                case CB_CGE_DUPLICATE:
                  edge = cgraph_edge (id->src_node, orig_stmt);
                  if (edge)

-                   edge = cgraph_clone_edge (edge, id->dst_node, stmt,
-                                             gimple_uid (stmt),
-                                             REG_BR_PROB_BASE, 1,
-                                             edge->frequency, true);
+                   {
+                     int edge_freq = edge->frequency;
+                     edge = cgraph_clone_edge (edge, id->dst_node, stmt,
+                                               gimple_uid (stmt),
+                                               REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
+                                               edge->frequency, true);
+                     /* We could also just rescale the frequency, but
+                        doing so would introduce roundoff errors and make
+                        verifier unhappy.  */
+                     edge->frequency 
+                       = compute_call_stmt_bb_frequency (id->dst_node->decl,
+                                                         copy_basic_block);
+                     if (dump_file
+                         && profile_status_for_function (cfun) != PROFILE_ABSENT
+                         && (edge_freq > edge->frequency + 10
+                             || edge_freq < edge->frequency - 10))
+                       {
+                         fprintf (dump_file, "Edge frequency estimated by "
+                                  "cgraph %i diverge from inliner's estimate %i\n",
+                                  edge_freq,
+                                  edge->frequency);
+                         fprintf (dump_file,
+                                  "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
+                                  bb->index,
+                                  bb->frequency,
+                                  copy_basic_block->frequency);
+                       }
+                   }

                  break;
 
                case CB_CGE_MOVE_CLONES:
                  break;
 
                case CB_CGE_MOVE_CLONES:


@@ -1674,7 +1700,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
                  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
                    cgraph_create_edge_including_clones
                      (id->dst_node, dest, stmt, bb->count,
                  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
                    cgraph_create_edge_including_clones
                      (id->dst_node, dest, stmt, bb->count,

-                      compute_call_stmt_bb_frequency (id->dst_node->decl, bb),
+                      compute_call_stmt_bb_frequency (id->dst_node->decl, 
+                                                      copy_basic_block),

                       bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
                  else
                    cgraph_create_edge (id->dst_node, dest, stmt,
                       bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
                  else
                    cgraph_create_edge (id->dst_node, dest, stmt,


@@ -1948,24 +1975,16 @@ remap_decl_1 (tree decl, void *data)
    NEW_FNDECL to be build.  CALLEE_FNDECL is the original */
 
 static void
    NEW_FNDECL to be build.  CALLEE_FNDECL is the original */
 
 static void

-initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
-                int frequency)
+initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count)

 {
   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
 {
   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);

-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;

 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else

-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-                      /
-                      ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;

 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();


@@ -1998,18 +2017,17 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
 
   init_empty_tree_cfg ();
 
 
   init_empty_tree_cfg ();
 

+  profile_status_for_function (cfun) = profile_status_for_function (src_cfun);

   ENTRY_BLOCK_PTR->count =
     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
   ENTRY_BLOCK_PTR->count =
     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);

-  ENTRY_BLOCK_PTR->frequency =
-    (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+  ENTRY_BLOCK_PTR->frequency
+    = ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;

   EXIT_BLOCK_PTR->count =
     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
   EXIT_BLOCK_PTR->frequency =
   EXIT_BLOCK_PTR->count =
     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
   EXIT_BLOCK_PTR->frequency =

-    (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+    EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;

   if (src_cfun->eh)
     init_eh_for_function ();
 
   if (src_cfun->eh)
     init_eh_for_function ();
 


@@ -2026,7 +2044,7 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
    another function.  Walks FN via CFG, returns new fndecl.  */
 
 static tree
    another function.  Walks FN via CFG, returns new fndecl.  */
 
 static tree

-copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
+copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,

               basic_block entry_block_map, basic_block exit_block_map)
 {
   tree callee_fndecl = id->src_fn;
               basic_block entry_block_map, basic_block exit_block_map)
 {
   tree callee_fndecl = id->src_fn;


@@ -2035,21 +2053,14 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
   struct function *cfun_to_copy;
   basic_block bb;
   tree new_fndecl = NULL;
   struct function *cfun_to_copy;
   basic_block bb;
   tree new_fndecl = NULL;

-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;

   int last;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
   int last;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else

-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-                      /
-                      ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;

 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();


@@ -2204,7 +2215,7 @@ copy_tree_body (copy_body_data *id)
    another function.  */
 
 static tree
    another function.  */
 
 static tree

-copy_body (copy_body_data *id, gcov_type count, int frequency,
+copy_body (copy_body_data *id, gcov_type count, int frequency_scale,

           basic_block entry_block_map, basic_block exit_block_map)
 {
   tree fndecl = id->src_fn;
           basic_block entry_block_map, basic_block exit_block_map)
 {
   tree fndecl = id->src_fn;


@@ -2212,7 +2223,7 @@ copy_body (copy_body_data *id, gcov_type count, int frequency,
 
   /* If this body has a CFG, walk CFG and copy.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl)));
 
   /* If this body has a CFG, walk CFG and copy.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl)));

-  body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map);
+  body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map);

   copy_debug_stmts (id);
 
   return body;
   copy_debug_stmts (id);
 
   return body;


@@ -3732,12 +3743,23 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id)
                                       cfun->local_decls);
     }
 
                                       cfun->local_decls);
     }
 

+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Inlining ");
+      print_generic_expr (dump_file, id->src_fn, 0); 
+      fprintf (dump_file, " to ");
+      print_generic_expr (dump_file, id->dst_fn, 0); 
+      fprintf (dump_file, " with frequency %i\n", cg_edge->frequency);
+    }
+

   /* This is it.  Duplicate the callee body.  Assume callee is
      pre-gimplified.  Note that we must not alter the caller
      function in any way before this point, as this CALL_EXPR may be
      a self-referential call; if we're calling ourselves, we need to
      duplicate our body before altering anything.  */
   /* This is it.  Duplicate the callee body.  Assume callee is
      pre-gimplified.  Note that we must not alter the caller
      function in any way before this point, as this CALL_EXPR may be
      a self-referential call; if we're calling ourselves, we need to
      duplicate our body before altering anything.  */

-  copy_body (id, bb->count, bb->frequency, bb, return_block);
+  copy_body (id, bb->count,
+            cg_edge->frequency * REG_BR_PROB_BASE / CGRAPH_FREQ_BASE,
+            bb, return_block);

 
   /* Reset the escaped and callused solutions.  */
   if (cfun->gimple_df)
 
   /* Reset the escaped and callused solutions.  */
   if (cfun->gimple_df)


@@ -4732,30 +4754,6 @@ delete_unreachable_blocks_update_callgraph (copy_body_data *id)
 
   if (changed)
     tidy_fallthru_edges ();
 
   if (changed)
     tidy_fallthru_edges ();

-#ifdef ENABLE_CHECKING0
-  verify_cgraph_node (id->dst_node);
-  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
-      && id->dst_node->clones)
-    {
-      struct cgraph_node *node;
-      for (node = id->dst_node->clones; node != id->dst_node;)
-       {
-         verify_cgraph_node (node);
-          
-         if (node->clones)
-           node = node->clones;
-         else if (node->next_sibling_clone)
-           node = node->next_sibling_clone;
-         else
-           {
-             while (node != id->dst_node && !node->next_sibling_clone)
-               node = node->clone_of;
-             if (node != id->dst_node)
-               node = node->next_sibling_clone;
-           }
-       }
-     }
-#endif

   return changed;
 }
 
   return changed;
 }
 


@@ -4876,8 +4874,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
   old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION
     (DECL_STRUCT_FUNCTION (old_decl));
   initialize_cfun (new_decl, old_decl,
   old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION
     (DECL_STRUCT_FUNCTION (old_decl));
   initialize_cfun (new_decl, old_decl,

-                  old_entry_block->count,
-                  old_entry_block->frequency);
+                  old_entry_block->count);

   push_cfun (DECL_STRUCT_FUNCTION (new_decl));
   
   /* Copy the function's static chain.  */
   push_cfun (DECL_STRUCT_FUNCTION (new_decl));
   
   /* Copy the function's static chain.  */


@@ -4947,7 +4944,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
       }
   
   /* Copy the Function's body.  */
       }
   
   /* Copy the Function's body.  */

-  copy_body (&id, old_entry_block->count, old_entry_block->frequency,
+  copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE,

             ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR);
   
   if (DECL_RESULT (old_decl) != NULL_TREE)
             ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR);
   
   if (DECL_RESULT (old_decl) != NULL_TREE)