/* Function splitting pass
- Copyright (C) 2010
+ Copyright (C) 2010, 2011, 2012
Free Software Foundation, Inc.
Contributed by Jan Hubicka <jh@suse.cz>
}
When func becomes inlinable and when cheap_test is often true, inlining func,
- but not fund.part leads to performance imrovement similar as inlining
+ but not fund.part leads to performance improvement similar as inlining
original func while the code size growth is smaller.
The pass is organized in three stages:
#include "fibheap.h"
#include "params.h"
#include "gimple-pretty-print.h"
+#include "ipa-inline.h"
/* Per basic block info. */
/* Size of the partitions. */
unsigned int header_time, header_size, split_time, split_size;
- /* SSA names that need to be passed into spit funciton. */
+ /* SSA names that need to be passed into spit function. */
bitmap ssa_names_to_pass;
/* Basic block where we split (that will become entry point of new function. */
/* Basic blocks we are splitting away. */
bitmap split_bbs;
+
+ /* True when return value is computed on split part and thus it needs
+ to be returned. */
+ bool split_part_set_retval;
};
/* Best split point found. */
struct split_point best_split_point;
-/* Callback for walk_stmt_load_store_addr_ops. If T is non-ssa automatic
+/* Set of basic blocks that are not allowed to dominate a split point. */
+
+static bitmap forbidden_dominators;
+
+static tree find_retval (basic_block return_bb);
+
+/* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
variable, check it if it is present in bitmap passed via DATA. */
static bool
-test_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t,
- void *data ATTRIBUTE_UNUSED)
+test_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data)
{
t = get_base_address (t);
- if (t && !is_gimple_reg (t)
- && ((TREE_CODE (t) == VAR_DECL
+ if (!t || is_gimple_reg (t))
+ return false;
+
+ if (TREE_CODE (t) == PARM_DECL
+ || (TREE_CODE (t) == VAR_DECL
&& auto_var_in_fn_p (t, current_function_decl))
- || (TREE_CODE (t) == RESULT_DECL)
- || (TREE_CODE (t) == PARM_DECL)))
+ || TREE_CODE (t) == RESULT_DECL
+ || TREE_CODE (t) == LABEL_DECL)
return bitmap_bit_p ((bitmap)data, DECL_UID (t));
+
+ /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
+ to pretend that the value pointed to is actual result decl. */
+ if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
+ && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
+ && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
+ && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ return
+ bitmap_bit_p ((bitmap)data,
+ DECL_UID (DECL_RESULT (current_function_decl)));
+
return false;
}
dump_split_point (FILE * file, struct split_point *current)
{
fprintf (file,
- "Split point at BB %i header time:%i header size: %i"
- " split time: %i split size: %i\n bbs: ",
+ "Split point at BB %i\n"
+ " header time: %i header size: %i\n"
+ " split time: %i split size: %i\n bbs: ",
current->entry_bb->index, current->header_time,
current->header_size, current->split_time, current->split_size);
dump_bitmap (file, current->split_bbs);
dump_bitmap (file, current->ssa_names_to_pass);
}
+/* Look for all BBs in header that might lead to the split part and verify
+ that they are not defining any non-SSA var used by the split part.
+ Parameters are the same as for consider_split. */
+
+static bool
+verify_non_ssa_vars (struct split_point *current, bitmap non_ssa_vars,
+ basic_block return_bb)
+{
+ bitmap seen = BITMAP_ALLOC (NULL);
+ VEC (basic_block,heap) *worklist = NULL;
+ edge e;
+ edge_iterator ei;
+ bool ok = true;
+
+ FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
+ if (e->src != ENTRY_BLOCK_PTR
+ && !bitmap_bit_p (current->split_bbs, e->src->index))
+ {
+ VEC_safe_push (basic_block, heap, worklist, e->src);
+ bitmap_set_bit (seen, e->src->index);
+ }
+
+ while (!VEC_empty (basic_block, worklist))
+ {
+ gimple_stmt_iterator bsi;
+ basic_block bb = VEC_pop (basic_block, worklist);
+
+ FOR_EACH_EDGE (e, ei, bb->preds)
+ if (e->src != ENTRY_BLOCK_PTR
+ && bitmap_set_bit (seen, e->src->index))
+ {
+ gcc_checking_assert (!bitmap_bit_p (current->split_bbs,
+ e->src->index));
+ VEC_safe_push (basic_block, heap, worklist, e->src);
+ }
+ for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ {
+ gimple stmt = gsi_stmt (bsi);
+ if (is_gimple_debug (stmt))
+ continue;
+ if (walk_stmt_load_store_addr_ops
+ (stmt, non_ssa_vars, test_nonssa_use, test_nonssa_use,
+ test_nonssa_use))
+ {
+ ok = false;
+ goto done;
+ }
+ if (gimple_code (stmt) == GIMPLE_LABEL
+ && test_nonssa_use (stmt, gimple_label_label (stmt),
+ non_ssa_vars))
+ {
+ ok = false;
+ goto done;
+ }
+ }
+ for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ {
+ if (walk_stmt_load_store_addr_ops
+ (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use,
+ test_nonssa_use))
+ {
+ ok = false;
+ goto done;
+ }
+ }
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ if (e->dest != return_bb)
+ continue;
+ for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
+ {
+ gimple stmt = gsi_stmt (bsi);
+ tree op = gimple_phi_arg_def (stmt, e->dest_idx);
+
+ if (!is_gimple_reg (gimple_phi_result (stmt)))
+ continue;
+ if (TREE_CODE (op) != SSA_NAME
+ && test_nonssa_use (stmt, op, non_ssa_vars))
+ {
+ ok = false;
+ goto done;
+ }
+ }
+ }
+ }
+done:
+ BITMAP_FREE (seen);
+ VEC_free (basic_block, heap, worklist);
+ return ok;
+}
+
+/* If STMT is a call, check the callee against a list of forbidden
+ predicate functions. If a match is found, look for uses of the
+ call result in condition statements that compare against zero.
+ For each such use, find the block targeted by the condition
+ statement for the nonzero result, and set the bit for this block
+ in the forbidden dominators bitmap. The purpose of this is to avoid
+ selecting a split point where we are likely to lose the chance
+ to optimize away an unused function call. */
+
+static void
+check_forbidden_calls (gimple stmt)
+{
+ imm_use_iterator use_iter;
+ use_operand_p use_p;
+ tree lhs;
+
+ /* At the moment, __builtin_constant_p is the only forbidden
+ predicate function call (see PR49642). */
+ if (!gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P))
+ return;
+
+ lhs = gimple_call_lhs (stmt);
+
+ if (!lhs || TREE_CODE (lhs) != SSA_NAME)
+ return;
+
+ FOR_EACH_IMM_USE_FAST (use_p, use_iter, lhs)
+ {
+ tree op1;
+ basic_block use_bb, forbidden_bb;
+ enum tree_code code;
+ edge true_edge, false_edge;
+ gimple use_stmt = USE_STMT (use_p);
+
+ if (gimple_code (use_stmt) != GIMPLE_COND)
+ continue;
+
+ /* Assuming canonical form for GIMPLE_COND here, with constant
+ in second position. */
+ op1 = gimple_cond_rhs (use_stmt);
+ code = gimple_cond_code (use_stmt);
+ use_bb = gimple_bb (use_stmt);
+
+ extract_true_false_edges_from_block (use_bb, &true_edge, &false_edge);
+
+ /* We're only interested in comparisons that distinguish
+ unambiguously from zero. */
+ if (!integer_zerop (op1) || code == LE_EXPR || code == GE_EXPR)
+ continue;
+
+ if (code == EQ_EXPR)
+ forbidden_bb = false_edge->dest;
+ else
+ forbidden_bb = true_edge->dest;
+
+ bitmap_set_bit (forbidden_dominators, forbidden_bb->index);
+ }
+}
+
+/* If BB is dominated by any block in the forbidden dominators set,
+ return TRUE; else FALSE. */
+
+static bool
+dominated_by_forbidden (basic_block bb)
+{
+ unsigned dom_bb;
+ bitmap_iterator bi;
+
+ EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators, 1, dom_bb, bi)
+ {
+ if (dominated_by_p (CDI_DOMINATORS, bb, BASIC_BLOCK (dom_bb)))
+ return true;
+ }
+
+ return false;
+}
+
/* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa
variables used and RETURN_BB is return basic block.
See if we can split function here. */
edge_iterator ei;
gimple_stmt_iterator bsi;
unsigned int i;
- int incomming_freq = 0;
+ int incoming_freq = 0;
+ tree retval;
if (dump_file && (dump_flags & TDF_DETAILS))
dump_split_point (dump_file, current);
FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
if (!bitmap_bit_p (current->split_bbs, e->src->index))
- incomming_freq += EDGE_FREQUENCY (e);
+ incoming_freq += EDGE_FREQUENCY (e);
/* Do not split when we would end up calling function anyway. */
- if (incomming_freq
+ if (incoming_freq
>= (ENTRY_BLOCK_PTR->frequency
* PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
- " Refused: incomming frequency is too large.\n");
+ " Refused: incoming frequency is too large.\n");
return;
}
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Refused: header empty\n");
- gcc_unreachable ();
return;
}
- /* Verify that PHI args on entry are either virutal or all their operands
- incomming from header are the same. */
+ /* Verify that PHI args on entry are either virtual or all their operands
+ incoming from header are the same. */
for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
gimple stmt = gsi_stmt (bsi);
call overhead. */
call_overhead = eni_size_weights.call_cost;
for (parm = DECL_ARGUMENTS (current_function_decl); parm;
- parm = TREE_CHAIN (parm))
+ parm = DECL_CHAIN (parm))
{
if (!is_gimple_reg (parm))
{
/* When there are non-ssa vars used in the split region, see if they
are used in the header region. If so, reject the split.
FIXME: we can use nested function support to access both. */
- if (!bitmap_empty_p (non_ssa_vars))
+ if (!bitmap_empty_p (non_ssa_vars)
+ && !verify_non_ssa_vars (current, non_ssa_vars, return_bb))
{
- basic_block bb;
- FOR_EACH_BB (bb)
- {
- gimple_stmt_iterator bsi;
- if (!bitmap_bit_p (current->split_bbs, bb->index))
- continue;
- for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
- {
- if (is_gimple_debug (gsi_stmt (bsi)))
- continue;
- if (walk_stmt_load_store_addr_ops
- (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use,
- test_nonssa_use, test_nonssa_use))
- {
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- " Refused: split part has non-ssa uses\n");
- return;
- }
- }
- for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
- {
- if (walk_stmt_load_store_addr_ops
- (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use,
- test_nonssa_use, test_nonssa_use))
- {
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- " Refused: split part has non-ssa uses\n");
- return;
- }
- }
- FOR_EACH_EDGE (e, ei, bb->succs)
- {
- if (e->dest != return_bb)
- continue;
- for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi);
- gsi_next (&bsi))
- {
- gimple stmt = gsi_stmt (bsi);
- tree op = gimple_phi_arg_def (stmt, e->dest_idx);
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ " Refused: split part has non-ssa uses\n");
+ return;
+ }
- if (!is_gimple_reg (gimple_phi_result (stmt)))
- continue;
- if (TREE_CODE (op) != SSA_NAME
- && test_nonssa_use (stmt, op, non_ssa_vars))
- {
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- " Refused: split part has non-ssa uses\n");
- return;
- }
- }
- }
- }
+ /* If the split point is dominated by a forbidden block, reject
+ the split. */
+ if (!bitmap_empty_p (forbidden_dominators)
+ && dominated_by_forbidden (current->entry_bb))
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ " Refused: split point dominated by forbidden block\n");
return;
}
+
+ /* See if retval used by return bb is computed by header or split part.
+ When it is computed by split part, we need to produce return statement
+ in the split part and add code to header to pass it around.
+
+ This is bit tricky to test:
+ 1) When there is no return_bb or no return value, we always pass
+ value around.
+ 2) Invariants are always computed by caller.
+ 3) For SSA we need to look if defining statement is in header or split part
+ 4) For non-SSA we need to look where the var is computed. */
+ retval = find_retval (return_bb);
+ if (!retval)
+ current->split_part_set_retval = true;
+ else if (is_gimple_min_invariant (retval))
+ current->split_part_set_retval = false;
+ /* Special case is value returned by reference we record as if it was non-ssa
+ set to result_decl. */
+ else if (TREE_CODE (retval) == SSA_NAME
+ && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL
+ && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ current->split_part_set_retval
+ = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval)));
+ else if (TREE_CODE (retval) == SSA_NAME)
+ current->split_part_set_retval
+ = (!SSA_NAME_IS_DEFAULT_DEF (retval)
+ && (bitmap_bit_p (current->split_bbs,
+ gimple_bb (SSA_NAME_DEF_STMT (retval))->index)
+ || gimple_bb (SSA_NAME_DEF_STMT (retval)) == return_bb));
+ else if (TREE_CODE (retval) == PARM_DECL)
+ current->split_part_set_retval = false;
+ else if (TREE_CODE (retval) == VAR_DECL
+ || TREE_CODE (retval) == RESULT_DECL)
+ current->split_part_set_retval
+ = bitmap_bit_p (non_ssa_vars, DECL_UID (retval));
+ else
+ current->split_part_set_retval = true;
+
+ /* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
+ for the return value. If there are other PHIs, give up. */
+ if (return_bb != EXIT_BLOCK_PTR)
+ {
+ gimple_stmt_iterator psi;
+
+ for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi))
+ if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi)))
+ && !(retval
+ && current->split_part_set_retval
+ && TREE_CODE (retval) == SSA_NAME
+ && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
+ && SSA_NAME_DEF_STMT (retval) == gsi_stmt (psi)))
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ " Refused: return bb has extra PHIs\n");
+ return;
+ }
+ }
+
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Accepted!\n");
}
}
-/* Return basic block containing RETURN statement, or EXIT_BLOCK_PTR if none
- found.
+/* Return basic block containing RETURN statement. We allow basic blocks
+ of the form:
+ <retval> = tmp_var;
+ return <retval>
+ but return_bb can not be more complex than this.
+ If nothing is found, return EXIT_BLOCK_PTR.
+
When there are multiple RETURN statement, chose one with return value,
since that one is more likely shared by multiple code paths.
+
+ Return BB is special, because for function splitting it is the only
+ basic block that is duplicated in between header and split part of the
+ function.
+
TODO: We might support multiple return blocks. */
static basic_block
find_return_bb (void)
{
edge e;
- edge_iterator ei;
basic_block return_bb = EXIT_BLOCK_PTR;
+ gimple_stmt_iterator bsi;
+ bool found_return = false;
+ tree retval = NULL_TREE;
+
+ if (!single_pred_p (EXIT_BLOCK_PTR))
+ return return_bb;
+
+ e = single_pred_edge (EXIT_BLOCK_PTR);
+ for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi))
+ {
+ gimple stmt = gsi_stmt (bsi);
+ if (gimple_code (stmt) == GIMPLE_LABEL
+ || is_gimple_debug (stmt)
+ || gimple_clobber_p (stmt))
+ ;
+ else if (gimple_code (stmt) == GIMPLE_ASSIGN
+ && found_return
+ && gimple_assign_single_p (stmt)
+ && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt),
+ current_function_decl)
+ || is_gimple_min_invariant (gimple_assign_rhs1 (stmt)))
+ && retval == gimple_assign_lhs (stmt))
+ ;
+ else if (gimple_code (stmt) == GIMPLE_RETURN)
+ {
+ found_return = true;
+ retval = gimple_return_retval (stmt);
+ }
+ else
+ break;
+ }
+ if (gsi_end_p (bsi) && found_return)
+ return_bb = e->src;
- if (EDGE_COUNT (EXIT_BLOCK_PTR->preds) == 1)
- FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
- {
- gimple_stmt_iterator bsi;
- bool found_return = false;
- tree retval = NULL_TREE;
-
- for (bsi = gsi_start_bb (e->src); !gsi_end_p (bsi); gsi_next (&bsi))
- if (gimple_code (gsi_stmt (bsi)) != GIMPLE_RETURN
- && gimple_code (gsi_stmt (bsi)) != GIMPLE_LABEL
- && !is_gimple_debug (gsi_stmt (bsi)))
- break;
- else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
- {
- found_return = true;
- retval = gimple_return_retval (gsi_stmt (bsi));
- }
- if (gsi_end_p (bsi) && found_return)
- {
- if (retval)
- return e->src;
- else
- return_bb = e->src;
- }
- }
return return_bb;
}
-/* Callback for walk_stmt_load_store_addr_ops. If T is non-ssa automatic
- variable, mark it as used in bitmap passed via DATA.
+/* Given return basic block RETURN_BB, see where return value is really
+ stored. */
+static tree
+find_retval (basic_block return_bb)
+{
+ gimple_stmt_iterator bsi;
+ for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
+ return gimple_return_retval (gsi_stmt (bsi));
+ else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
+ && !gimple_clobber_p (gsi_stmt (bsi)))
+ return gimple_assign_rhs1 (gsi_stmt (bsi));
+ return NULL;
+}
+
+/* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
+ variable, mark it as used in bitmap passed via DATA.
Return true when access to T prevents splitting the function. */
static bool
-mark_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t,
- void *data ATTRIBUTE_UNUSED)
+mark_nonssa_use (gimple stmt ATTRIBUTE_UNUSED, tree t, void *data)
{
t = get_base_address (t);
if (TREE_CODE (t) == PARM_DECL)
{
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Can not split use of non-ssa function parameter.\n");
+ fprintf (dump_file,
+ "Cannot split: use of non-ssa function parameter.\n");
return true;
}
- if ((TREE_CODE (t) == VAR_DECL && auto_var_in_fn_p (t, current_function_decl))
- || (TREE_CODE (t) == RESULT_DECL))
+ if ((TREE_CODE (t) == VAR_DECL
+ && auto_var_in_fn_p (t, current_function_decl))
+ || TREE_CODE (t) == RESULT_DECL
+ || TREE_CODE (t) == LABEL_DECL)
bitmap_set_bit ((bitmap)data, DECL_UID (t));
+
+ /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
+ to pretend that the value pointed to is actual result decl. */
+ if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
+ && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
+ && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
+ && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ return
+ bitmap_bit_p ((bitmap)data,
+ DECL_UID (DECL_RESULT (current_function_decl)));
+
return false;
}
if (is_gimple_debug (stmt))
continue;
+ if (gimple_clobber_p (stmt))
+ continue;
+
/* FIXME: We can split regions containing EH. We can not however
split RESX, EH_DISPATCH and EH_POINTER referring to same region
into different partitions. This would require tracking of
EH regions and checking in consider_split_point if they
are not used elsewhere. */
- if (gimple_code (stmt) == GIMPLE_RESX
- && stmt_can_throw_external (stmt))
+ if (gimple_code (stmt) == GIMPLE_RESX)
{
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Can not split external resx.\n");
+ fprintf (dump_file, "Cannot split: resx.\n");
can_split = false;
}
if (gimple_code (stmt) == GIMPLE_EH_DISPATCH)
{
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Can not split eh dispatch.\n");
+ fprintf (dump_file, "Cannot split: eh dispatch.\n");
can_split = false;
}
way to store builtin_stack_save result in non-SSA variable
since all calls to those are compiler generated. */
case BUILT_IN_APPLY:
+ case BUILT_IN_APPLY_ARGS:
case BUILT_IN_VA_START:
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Can not split builtin_apply and va_start.\n");
+ fprintf (dump_file,
+ "Cannot split: builtin_apply and va_start.\n");
can_split = false;
break;
case BUILT_IN_EH_POINTER:
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Can not split builtin_eh_pointer.\n");
+ fprintf (dump_file, "Cannot split: builtin_eh_pointer.\n");
can_split = false;
break;
default:
mark_nonssa_use,
mark_nonssa_use);
}
- /* Record also uses comming from PHI operand in return BB. */
+ /* Record also uses coming from PHI operand in return BB. */
FOR_EACH_EDGE (e, ei, bb->succs)
if (e->dest == return_bb)
{
- bool found_phi = false;
for (bsi = gsi_start_phis (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
gimple stmt = gsi_stmt (bsi);
continue;
if (!is_gimple_reg (gimple_phi_result (stmt)))
continue;
- found_phi = true;
if (TREE_CODE (op) == SSA_NAME)
bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
else
can_split &= !mark_nonssa_use (stmt, op, non_ssa_vars);
}
- if (!gsi_end_p (gsi_last_bb (return_bb)))
- {
- ssa_op_iter iter;
- gimple stmt = gsi_stmt (gsi_last_bb (return_bb));
- tree op;
- if (!found_phi)
- FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
- bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
- can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars,
- mark_nonssa_use,
- mark_nonssa_use,
- mark_nonssa_use);
- }
}
return can_split;
}
bitmap bbs_visited;
/* Last examined edge in DFS walk. Since we walk unoriented graph,
- the value is up to sum of incomming and outgoing edges of BB. */
+ the value is up to sum of incoming and outgoing edges of BB. */
unsigned int edge_num;
/* Stack entry index of earliest BB reachable from current BB
- or any BB visited later in DFS valk. */
+ or any BB visited later in DFS walk. */
int earliest;
/* Overall time and size of all BBs reached from this BB in DFS walk. */
&& (intptr_t)dest->aux < entry->earliest)
entry->earliest = (intptr_t)dest->aux;
}
- /* We are done with examing the edges. pop off the value from stack and
- merge stuff we cummulate during the walk. */
+ /* We are done with examining the edges. Pop off the value from stack
+ and merge stuff we accumulate during the walk. */
else if (entry->bb != ENTRY_BLOCK_PTR)
{
stack_entry *prev = VEC_index (stack_entry, stack,
ENTRY_BLOCK_PTR->aux = NULL;
FOR_EACH_BB (bb)
bb->aux = NULL;
+ VEC_free (stack_entry, heap, stack);
BITMAP_FREE (current.ssa_names_to_pass);
}
split_function (struct split_point *split_point)
{
VEC (tree, heap) *args_to_pass = NULL;
- bitmap args_to_skip = BITMAP_ALLOC (NULL);
+ bitmap args_to_skip;
tree parm;
int num = 0;
- struct cgraph_node *node;
+ struct cgraph_node *node, *cur_node = cgraph_get_node (current_function_decl);
basic_block return_bb = find_return_bb ();
basic_block call_bb;
gimple_stmt_iterator gsi;
tree retval = NULL, real_retval = NULL;
bool split_part_return_p = false;
gimple last_stmt = NULL;
+ unsigned int i;
+ tree arg;
if (dump_file)
{
dump_split_point (dump_file, split_point);
}
+ if (cur_node->local.can_change_signature)
+ args_to_skip = BITMAP_ALLOC (NULL);
+ else
+ args_to_skip = NULL;
+
/* Collect the parameters of new function and args_to_skip bitmap. */
for (parm = DECL_ARGUMENTS (current_function_decl);
- parm; parm = TREE_CHAIN (parm), num++)
- if (!is_gimple_reg (parm)
- || !gimple_default_def (cfun, parm)
- || !bitmap_bit_p (split_point->ssa_names_to_pass,
- SSA_NAME_VERSION (gimple_default_def (cfun, parm))))
+ parm; parm = DECL_CHAIN (parm), num++)
+ if (args_to_skip
+ && (!is_gimple_reg (parm)
+ || !gimple_default_def (cfun, parm)
+ || !bitmap_bit_p (split_point->ssa_names_to_pass,
+ SSA_NAME_VERSION (gimple_default_def (cfun,
+ parm)))))
bitmap_set_bit (args_to_skip, num);
else
- VEC_safe_push (tree, heap, args_to_pass, gimple_default_def (cfun, parm));
+ {
+ /* This parm might not have been used up to now, but is going to be
+ used, hence register it. */
+ add_referenced_var (parm);
+ if (is_gimple_reg (parm))
+ {
+ arg = gimple_default_def (cfun, parm);
+ if (!arg)
+ {
+ arg = make_ssa_name (parm, gimple_build_nop ());
+ set_default_def (parm, arg);
+ }
+ }
+ else
+ arg = parm;
+
+ if (!useless_type_conversion_p (DECL_ARG_TYPE (parm), TREE_TYPE (arg)))
+ arg = fold_convert (DECL_ARG_TYPE (parm), arg);
+ VEC_safe_push (tree, heap, args_to_pass, arg);
+ }
/* See if the split function will return. */
FOR_EACH_EDGE (e, ei, return_bb->preds)
if (e)
split_part_return_p = true;
- /* If we return, we will need the return block. */
- if (return_bb != EXIT_BLOCK_PTR && split_part_return_p)
+ /* Add return block to what will become the split function.
+ We do not return; no return block is needed. */
+ if (!split_part_return_p)
+ ;
+ /* We have no return block, so nothing is needed. */
+ else if (return_bb == EXIT_BLOCK_PTR)
+ ;
+ /* When we do not want to return value, we need to construct
+ new return block with empty return statement.
+ FIXME: Once we are able to change return type, we should change function
+ to return void instead of just outputting function with undefined return
+ value. For structures this affects quality of codegen. */
+ else if (!split_point->split_part_set_retval
+ && find_retval (return_bb))
+ {
+ bool redirected = true;
+ basic_block new_return_bb = create_basic_block (NULL, 0, return_bb);
+ gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb);
+ gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT);
+ while (redirected)
+ {
+ redirected = false;
+ FOR_EACH_EDGE (e, ei, return_bb->preds)
+ if (bitmap_bit_p (split_point->split_bbs, e->src->index))
+ {
+ new_return_bb->count += e->count;
+ new_return_bb->frequency += EDGE_FREQUENCY (e);
+ redirect_edge_and_branch (e, new_return_bb);
+ redirected = true;
+ break;
+ }
+ }
+ e = make_edge (new_return_bb, EXIT_BLOCK_PTR, 0);
+ e->probability = REG_BR_PROB_BASE;
+ e->count = new_return_bb->count;
+ bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
+ }
+ /* When we pass around the value, use existing return block. */
+ else
bitmap_set_bit (split_point->split_bbs, return_bb->index);
+ /* If RETURN_BB has virtual operand PHIs, they must be removed and the
+ virtual operand marked for renaming as we change the CFG in a way that
+ tree-inline is not able to compensate for.
+
+ Note this can happen whether or not we have a return value. If we have
+ a return value, then RETURN_BB may have PHIs for real operands too. */
+ if (return_bb != EXIT_BLOCK_PTR)
+ {
+ bool phi_p = false;
+ for (gsi = gsi_start_phis (return_bb); !gsi_end_p (gsi);)
+ {
+ gimple stmt = gsi_stmt (gsi);
+ if (is_gimple_reg (gimple_phi_result (stmt)))
+ {
+ gsi_next (&gsi);
+ continue;
+ }
+ mark_virtual_phi_result_for_renaming (stmt);
+ remove_phi_node (&gsi, true);
+ phi_p = true;
+ }
+ /* In reality we have to rename the reaching definition of the
+ virtual operand at return_bb as we will eventually release it
+ when we remove the code region we outlined.
+ So we have to rename all immediate virtual uses of that region
+ if we didn't see a PHI definition yet. */
+ /* ??? In real reality we want to set the reaching vdef of the
+ entry of the SESE region as the vuse of the call and the reaching
+ vdef of the exit of the SESE region as the vdef of the call. */
+ if (!phi_p)
+ for (gsi = gsi_start_bb (return_bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple stmt = gsi_stmt (gsi);
+ if (gimple_vuse (stmt))
+ {
+ gimple_set_vuse (stmt, NULL_TREE);
+ update_stmt (stmt);
+ }
+ if (gimple_vdef (stmt))
+ break;
+ }
+ }
+
/* Now create the actual clone. */
rebuild_cgraph_edges ();
- node = cgraph_function_versioning (cgraph_node (current_function_decl),
- NULL, NULL,
- args_to_skip,
+ node = cgraph_function_versioning (cur_node, NULL, NULL, args_to_skip,
+ !split_part_return_p,
split_point->split_bbs,
- split_point->entry_bb, "_part");
+ split_point->entry_bb, "part");
/* For usual cloning it is enough to clear builtin only when signature
changes. For partial inlining we however can not expect the part
of builtin implementation to have same semantic as the whole. */
DECL_BUILT_IN_CLASS (node->decl) = NOT_BUILT_IN;
DECL_FUNCTION_CODE (node->decl) = (enum built_in_function) 0;
}
- cgraph_node_remove_callees (cgraph_node (current_function_decl));
+ cgraph_node_remove_callees (cur_node);
if (!split_part_return_p)
TREE_THIS_VOLATILE (node->decl) = 1;
if (dump_file)
/* Produce the call statement. */
gsi = gsi_last_bb (call_bb);
+ FOR_EACH_VEC_ELT (tree, args_to_pass, i, arg)
+ if (!is_gimple_val (arg))
+ {
+ arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE,
+ false, GSI_CONTINUE_LINKING);
+ VEC_replace (tree, args_to_pass, i, arg);
+ }
call = gimple_build_call_vec (node->decl, args_to_pass);
gimple_set_block (call, DECL_INITIAL (current_function_decl));
+ /* We avoid address being taken on any variable used by split part,
+ so return slot optimization is always possible. Moreover this is
+ required to make DECL_BY_REFERENCE work. */
+ if (aggregate_value_p (DECL_RESULT (current_function_decl),
+ TREE_TYPE (current_function_decl)))
+ gimple_call_set_return_slot_opt (call, true);
+
/* Update return value. This is bit tricky. When we do not return,
do nothing. When we return we might need to update return_bb
or produce a new return statement. */
return_bb == EXIT_BLOCK_PTR ? 0 : EDGE_FALLTHRU);
e->count = call_bb->count;
e->probability = REG_BR_PROB_BASE;
+
+ /* If there is return basic block, see what value we need to store
+ return value into and put call just before it. */
if (return_bb != EXIT_BLOCK_PTR)
{
- gimple return_stmt = gsi_stmt (gsi_last_bb (return_bb));
- gcc_assert (gimple_code (return_stmt) == GIMPLE_RETURN);
+ real_retval = retval = find_retval (return_bb);
- if ((real_retval = retval = gimple_return_retval (return_stmt))
- && !is_gimple_min_invariant (retval)
- && (TREE_CODE (retval) != SSA_NAME
- || !SSA_NAME_IS_DEFAULT_DEF (retval)))
+ if (real_retval && split_point->split_part_set_retval)
{
gimple_stmt_iterator psi;
- /* See if there is PHI definind return value. */
- for (psi = gsi_start_phis (return_bb);
- !gsi_end_p (psi); gsi_next (&psi))
- if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi))))
- break;
-
- /* When we have PHI, update PHI. When there is no PHI,
- update the return statement itself. */
- if (TREE_CODE (retval) == SSA_NAME)
+ /* See if we need new SSA_NAME for the result.
+ When DECL_BY_REFERENCE is true, retval is actually pointer to
+ return value and it is constant in whole function. */
+ if (TREE_CODE (retval) == SSA_NAME
+ && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
{
retval = make_ssa_name (SSA_NAME_VAR (retval), call);
+
+ /* See if there is PHI defining return value. */
+ for (psi = gsi_start_phis (return_bb);
+ !gsi_end_p (psi); gsi_next (&psi))
+ if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi))))
+ break;
+
+ /* When there is PHI, just update its value. */
if (TREE_CODE (retval) == SSA_NAME
&& !gsi_end_p (psi))
add_phi_arg (gsi_stmt (psi), retval, e, UNKNOWN_LOCATION);
- else if (TREE_CODE (retval) == SSA_NAME)
+ /* Otherwise update the return BB itself.
+ find_return_bb allows at most one assignment to return value,
+ so update first statement. */
+ else
{
- gimple_return_set_retval (return_stmt, retval);
- update_stmt (return_stmt);
+ gimple_stmt_iterator bsi;
+ for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
+ if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
+ {
+ gimple_return_set_retval (gsi_stmt (bsi), retval);
+ break;
+ }
+ else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
+ && !gimple_clobber_p (gsi_stmt (bsi)))
+ {
+ gimple_assign_set_rhs1 (gsi_stmt (bsi), retval);
+ break;
+ }
+ update_stmt (gsi_stmt (bsi));
}
}
- gimple_call_set_lhs (call, retval);
+ if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ {
+ gimple_call_set_lhs (call, build_simple_mem_ref (retval));
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ }
+ else
+ {
+ tree restype;
+ restype = TREE_TYPE (DECL_RESULT (current_function_decl));
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ if (!useless_type_conversion_p (TREE_TYPE (retval), restype))
+ {
+ gimple cpy;
+ tree tem = create_tmp_reg (restype, NULL);
+ tem = make_ssa_name (tem, call);
+ cpy = gimple_build_assign_with_ops (NOP_EXPR, retval,
+ tem, NULL_TREE);
+ gsi_insert_after (&gsi, cpy, GSI_NEW_STMT);
+ retval = tem;
+ }
+ gimple_call_set_lhs (call, retval);
+ update_stmt (call);
+ }
}
- gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ else
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
}
+ /* We don't use return block (there is either no return in function or
+ multiple of them). So create new basic block with return statement.
+ */
else
{
gimple ret;
- if (!VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
+ if (split_point->split_part_set_retval
+ && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
{
retval = DECL_RESULT (current_function_decl);
&& !DECL_BY_REFERENCE (retval))
retval = create_tmp_reg (TREE_TYPE (retval), NULL);
if (is_gimple_reg (retval))
- retval = make_ssa_name (retval, call);
- gimple_call_set_lhs (call, retval);
+ {
+ /* When returning by reference, there is only one SSA name
+ assigned to RESULT_DECL (that is pointer to return value).
+ Look it up or create new one if it is missing. */
+ if (DECL_BY_REFERENCE (retval))
+ {
+ tree retval_name;
+ if ((retval_name = gimple_default_def (cfun, retval))
+ != NULL)
+ retval = retval_name;
+ else
+ {
+ retval_name = make_ssa_name (retval,
+ gimple_build_nop ());
+ set_default_def (retval, retval_name);
+ retval = retval_name;
+ }
+ }
+ /* Otherwise produce new SSA name for return value. */
+ else
+ retval = make_ssa_name (retval, call);
+ }
+ if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ gimple_call_set_lhs (call, build_simple_mem_ref (retval));
+ else
+ gimple_call_set_lhs (call, retval);
}
gsi_insert_after (&gsi, call, GSI_NEW_STMT);
ret = gimple_build_return (retval);
}
free_dominance_info (CDI_DOMINATORS);
free_dominance_info (CDI_POST_DOMINATORS);
- compute_inline_parameters (node);
+ compute_inline_parameters (node, true);
}
/* Execute function splitting pass. */
basic_block bb;
int overall_time = 0, overall_size = 0;
int todo = 0;
- struct cgraph_node *node = cgraph_node (current_function_decl);
+ struct cgraph_node *node = cgraph_get_node (current_function_decl);
- if (flags_from_decl_or_type (current_function_decl) & ECF_NORETURN)
+ if (flags_from_decl_or_type (current_function_decl)
+ & (ECF_NORETURN|ECF_MALLOC))
{
if (dump_file)
- fprintf (dump_file, "Not splitting: noreturn function.\n");
+ fprintf (dump_file, "Not splitting: noreturn/malloc function.\n");
return 0;
}
if (MAIN_NAME_P (DECL_NAME (current_function_decl)))
}
/* This can be relaxed; function might become inlinable after splitting
away the uninlinable part. */
- if (!node->local.inlinable)
+ if (!inline_summary (node)->inlinable)
{
if (dump_file)
fprintf (dump_file, "Not splitting: not inlinable.\n");
return 0;
}
- if (node->local.disregard_inline_limits)
+ if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
{
if (dump_file)
- fprintf (dump_file, "Not splitting: disregading inline limits.\n");
+ fprintf (dump_file, "Not splitting: disregarding inline limits.\n");
return 0;
}
/* This can be relaxed; most of versioning tests actually prevents
fprintf (dump_file, "Not splitting: nested function.\n");
return 0;
}
- /* FIXME: Should be easy to support. */
- if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
- {
- if (dump_file)
- fprintf (dump_file, "Not splitting: returns value by reference.\n");
- return 0;
- }
/* See if it makes sense to try to split.
It makes sense to split if we inline, that is if we have direct calls to
handle or direct calls are possibly going to appear as result of indirect
- inlining or LTO.
+ inlining or LTO. Also handle -fprofile-generate as LTO to allow non-LTO
+ training for LTO -fprofile-use build.
+
Note that we are not completely conservative about disqualifying functions
called once. It is possible that the caller is called more then once and
then inlining would still benefit. */
if ((!node->callers || !node->callers->next_caller)
&& !node->address_taken
- && ((!flag_lto && !flag_whopr) || !node->local.externally_visible))
+ && (!flag_lto || !node->local.externally_visible))
{
if (dump_file)
fprintf (dump_file, "Not splitting: not called directly "
return 0;
}
+ /* Initialize bitmap to track forbidden calls. */
+ forbidden_dominators = BITMAP_ALLOC (NULL);
+ calculate_dominance_info (CDI_DOMINATORS);
+
/* Compute local info about basic blocks and determine function size/time. */
VEC_safe_grow_cleared (bb_info, heap, bb_info_vec, last_basic_block + 1);
memset (&best_split_point, 0, sizeof (best_split_point));
this_time = estimate_num_insns (stmt, &eni_time_weights) * freq;
size += this_size;
time += this_time;
+ check_forbidden_calls (stmt);
if (dump_file && (dump_flags & TDF_DETAILS))
{
BITMAP_FREE (best_split_point.split_bbs);
todo = TODO_update_ssa | TODO_cleanup_cfg;
}
+ BITMAP_FREE (forbidden_dominators);
VEC_free (bb_info, heap, bb_info_vec);
bb_info_vec = NULL;
return todo;
}
+/* Gate function splitting pass. When doing profile feedback, we want
+ to execute the pass after profiling is read. So disable one in
+ early optimization. */
+
static bool
gate_split_functions (void)
{
- return flag_partial_inlining;
+ return (flag_partial_inlining
+ && !profile_arc_flag && !flag_branch_probabilities);
}
struct gimple_opt_pass pass_split_functions =
0, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
- TODO_dump_func /* todo_flags_finish */
+ TODO_verify_all /* todo_flags_finish */
+ }
+};
+
+/* Gate feedback driven function splitting pass.
+ We don't need to split when profiling at all, we are producing
+ lousy code anyway. */
+
+static bool
+gate_feedback_split_functions (void)
+{
+ return (flag_partial_inlining
+ && flag_branch_probabilities);
+}
+
+/* Execute function splitting pass. */
+
+static unsigned int
+execute_feedback_split_functions (void)
+{
+ unsigned int retval = execute_split_functions ();
+ if (retval)
+ retval |= TODO_rebuild_cgraph_edges;
+ return retval;
+}
+
+struct gimple_opt_pass pass_feedback_split_functions =
+{
+ {
+ GIMPLE_PASS,
+ "feedback_fnsplit", /* name */
+ gate_feedback_split_functions, /* gate */
+ execute_feedback_split_functions, /* execute */
+ NULL, /* sub */
+ NULL, /* next */
+ 0, /* static_pass_number */
+ TV_IPA_FNSPLIT, /* tv_id */
+ PROP_cfg, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ TODO_verify_all /* todo_flags_finish */
}
};
OSDN Git Service
