X-Git-Url: http://git.sourceforge.jp/view?a=blobdiff_plain;f=gcc%2Ftree-inline.c;h=ae773f63b58eb67a17ab4b45e92562dcb756eb26;hb=22bb664ca0021f4e540cdfa56bc1d4238a7c1356;hp=7ef58c3f40c316540bfc2b47f43d344cb36a7a3e;hpb=9e9bac20aa32b9bb41ded1bbe0451f67ad99e0a5;p=pf3gnuchains%2Fgcc-fork.git diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index 7ef58c3f40c..ae773f63b58 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -1,6 +1,6 @@ /* Tree inlining. - Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 - Free Software Foundation, Inc. + Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, + 2012 Free Software Foundation, Inc. Contributed by Alexandre Oliva This file is part of GCC. @@ -23,16 +23,13 @@ along with GCC; see the file COPYING3. If not see #include "system.h" #include "coretypes.h" #include "tm.h" -#include "toplev.h" +#include "diagnostic-core.h" #include "tree.h" #include "tree-inline.h" -#include "rtl.h" -#include "expr.h" #include "flags.h" #include "params.h" #include "input.h" #include "insn-config.h" -#include "varray.h" #include "hashtab.h" #include "langhooks.h" #include "basic-block.h" @@ -42,9 +39,8 @@ along with GCC; see the file COPYING3. If not see #include "tree-mudflap.h" #include "tree-flow.h" #include "function.h" -#include "ggc.h" #include "tree-flow.h" -#include "diagnostic.h" +#include "tree-pretty-print.h" #include "except.h" #include "debug.h" #include "pointer-set.h" @@ -54,6 +50,8 @@ along with GCC; see the file COPYING3. If not see #include "target.h" #include "integrate.h" +#include "rtl.h" /* FIXME: For asm_str_count. */ + /* I'm not real happy about this, but we need to handle gimple and non-gimple trees. */ #include "gimple.h" @@ -65,7 +63,7 @@ along with GCC; see the file COPYING3. If not see MODIFY_EXPRs that store to a dedicated returned-value variable. The duplicated eh_region info of the copy will later be appended to the info for the caller; the eh_region info in copied throwing - statements and RESX_EXPRs is adjusted accordingly. + statements and RESX statements are adjusted accordingly. Cloning: (only in C++) We have one body for a con/de/structor, and multiple function decls, each with a unique parameter list. @@ -103,10 +101,6 @@ along with GCC; see the file COPYING3. If not see calls? */ -/* Weights that estimate_num_insns uses for heuristics in inlining. */ - -eni_weights eni_inlining_weights; - /* Weights that estimate_num_insns uses to estimate the size of the produced code. */ @@ -119,21 +113,20 @@ eni_weights eni_time_weights; /* Prototypes. */ -static tree declare_return_variable (copy_body_data *, tree, tree, tree *); -static bool inlinable_function_p (tree); +static tree declare_return_variable (copy_body_data *, tree, tree, basic_block); static void remap_block (tree *, copy_body_data *); -static tree remap_decls (tree, copy_body_data *); static void copy_bind_expr (tree *, int *, copy_body_data *); static tree mark_local_for_remap_r (tree *, int *, void *); static void unsave_expr_1 (tree); static tree unsave_r (tree *, int *, void *); static void declare_inline_vars (tree, tree); static void remap_save_expr (tree *, void *, int *); -static void add_lexical_block (tree current_block, tree new_block); +static void prepend_lexical_block (tree current_block, tree new_block); static tree copy_decl_to_var (tree, copy_body_data *); static tree copy_result_decl_to_var (tree, copy_body_data *); static tree copy_decl_maybe_to_var (tree, copy_body_data *); static gimple remap_gimple_stmt (gimple, copy_body_data *); +static bool delete_unreachable_blocks_update_callgraph (copy_body_data *id); /* Insert a tree->tree mapping for ID. Despite the name suggests that the trees should be variables, it is used for more than that. */ @@ -149,6 +142,36 @@ insert_decl_map (copy_body_data *id, tree key, tree value) *pointer_map_insert (id->decl_map, value) = value; } +/* Insert a tree->tree mapping for ID. This is only used for + variables. */ + +static void +insert_debug_decl_map (copy_body_data *id, tree key, tree value) +{ + if (!gimple_in_ssa_p (id->src_cfun)) + return; + + if (!MAY_HAVE_DEBUG_STMTS) + return; + + if (!target_for_debug_bind (key)) + return; + + gcc_assert (TREE_CODE (key) == PARM_DECL); + gcc_assert (TREE_CODE (value) == VAR_DECL); + + if (!id->debug_map) + id->debug_map = pointer_map_create (); + + *pointer_map_insert (id->debug_map, key) = value; +} + +/* If nonzero, we're remapping the contents of inlined debug + statements. If negative, an error has occurred, such as a + reference to a variable that isn't available in the inlined + context. */ +static int processing_debug_stmt = 0; + /* Construct new SSA name for old NAME. ID is the inline context. */ static tree @@ -163,12 +186,45 @@ remap_ssa_name (tree name, copy_body_data *id) if (n) return unshare_expr (*n); + if (processing_debug_stmt) + { + if (TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL + && SSA_NAME_IS_DEFAULT_DEF (name) + && id->entry_bb == NULL + && single_succ_p (ENTRY_BLOCK_PTR)) + { + tree vexpr = make_node (DEBUG_EXPR_DECL); + gimple def_temp; + gimple_stmt_iterator gsi; + tree val = SSA_NAME_VAR (name); + + n = (tree *) pointer_map_contains (id->decl_map, val); + if (n != NULL) + val = *n; + if (TREE_CODE (val) != PARM_DECL) + { + processing_debug_stmt = -1; + return name; + } + def_temp = gimple_build_debug_source_bind (vexpr, val, NULL); + DECL_ARTIFICIAL (vexpr) = 1; + TREE_TYPE (vexpr) = TREE_TYPE (name); + DECL_MODE (vexpr) = DECL_MODE (SSA_NAME_VAR (name)); + gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR)); + gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT); + return vexpr; + } + + processing_debug_stmt = -1; + return name; + } + /* Do not set DEF_STMT yet as statement is not copied yet. We do that in copy_bb. */ new_tree = remap_decl (SSA_NAME_VAR (name), id); /* We might've substituted constant or another SSA_NAME for - the variable. + the variable. Replace the SSA name representing RESULT_DECL by variable during inlining: this saves us from need to introduce PHI node in a case @@ -177,11 +233,21 @@ remap_ssa_name (tree name, copy_body_data *id) && (TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL || !id->transform_return_to_modify)) { + struct ptr_info_def *pi; new_tree = make_ssa_name (new_tree, NULL); insert_decl_map (id, name, new_tree); SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name); TREE_TYPE (new_tree) = TREE_TYPE (SSA_NAME_VAR (new_tree)); + /* At least IPA points-to info can be directly transferred. */ + if (id->src_cfun->gimple_df + && id->src_cfun->gimple_df->ipa_pta + && (pi = SSA_NAME_PTR_INFO (name)) + && !pi->pt.anything) + { + struct ptr_info_def *new_pi = get_ptr_info (new_tree); + new_pi->pt = pi->pt; + } if (gimple_nop_p (SSA_NAME_DEF_STMT (name))) { /* By inlining function having uninitialized variable, we might @@ -195,16 +261,16 @@ remap_ssa_name (tree name, copy_body_data *id) regions of the CFG, but this is expensive to test. */ if (id->entry_bb && is_gimple_reg (SSA_NAME_VAR (name)) + && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name) && TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL && (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR, 0)->dest || EDGE_COUNT (id->entry_bb->preds) != 1)) { gimple_stmt_iterator gsi = gsi_last_bb (id->entry_bb); gimple init_stmt; - - init_stmt = gimple_build_assign (new_tree, - fold_convert (TREE_TYPE (new_tree), - integer_zero_node)); + tree zero = build_zero_cst (TREE_TYPE (new_tree)); + + init_stmt = gimple_build_assign (new_tree, zero); gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT); SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0; } @@ -228,22 +294,26 @@ tree remap_decl (tree decl, copy_body_data *id) { tree *n; - tree fn; /* We only remap local variables in the current function. */ - fn = id->src_fn; /* See if we have remapped this declaration. */ n = (tree *) pointer_map_contains (id->decl_map, decl); + if (!n && processing_debug_stmt) + { + processing_debug_stmt = -1; + return decl; + } + /* If we didn't already have an equivalent for this declaration, create one now. */ if (!n) { /* Make a copy of the variable or label. */ tree t = id->copy_decl (decl, id); - + /* Remember it, so that if we encounter this local entity again we can reuse this copy. Do this early because remap_type may need this decl for TYPE_STUB_DECL. */ @@ -269,27 +339,24 @@ remap_decl (tree decl, copy_body_data *id) walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL); } - if (cfun && gimple_in_ssa_p (cfun) - && (TREE_CODE (t) == VAR_DECL - || TREE_CODE (t) == RESULT_DECL || TREE_CODE (t) == PARM_DECL)) - { - tree def = gimple_default_def (id->src_cfun, decl); - get_var_ann (t); - if (TREE_CODE (decl) != PARM_DECL && def) - { - tree map = remap_ssa_name (def, id); - /* Watch out RESULT_DECLs whose SSA names map directly - to them. */ - if (TREE_CODE (map) == SSA_NAME - && gimple_nop_p (SSA_NAME_DEF_STMT (map))) - set_default_def (t, map); - } - add_referenced_var (t); - } + if ((TREE_CODE (t) == VAR_DECL + || TREE_CODE (t) == RESULT_DECL + || TREE_CODE (t) == PARM_DECL) + && id->src_fn && DECL_STRUCT_FUNCTION (id->src_fn) + && gimple_referenced_vars (DECL_STRUCT_FUNCTION (id->src_fn)) + /* We don't want to mark as referenced VAR_DECLs that were + not marked as such in the src function. */ + && (TREE_CODE (decl) != VAR_DECL + || referenced_var_lookup (DECL_STRUCT_FUNCTION (id->src_fn), + DECL_UID (decl)))) + add_referenced_var (t); return t; } - return unshare_expr (*n); + if (id->do_not_unshare) + return *n; + else + return unshare_expr (*n); } static tree @@ -305,6 +372,10 @@ remap_type_1 (tree type, copy_body_data *id) new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id), TYPE_MODE (type), TYPE_REF_CAN_ALIAS_ALL (type)); + if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type)) + new_tree = build_type_attribute_qual_variant (new_tree, + TYPE_ATTRIBUTES (type), + TYPE_QUALS (type)); insert_decl_map (id, type, new_tree); return new_tree; } @@ -313,6 +384,10 @@ remap_type_1 (tree type, copy_body_data *id) new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id), TYPE_MODE (type), TYPE_REF_CAN_ALIAS_ALL (type)); + if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type)) + new_tree = build_type_attribute_qual_variant (new_tree, + TYPE_ATTRIBUTES (type), + TYPE_QUALS (type)); insert_decl_map (id, type, new_tree); return new_tree; } @@ -376,11 +451,11 @@ remap_type_1 (tree type, copy_body_data *id) { tree f, nf = NULL; - for (f = TYPE_FIELDS (new_tree); f ; f = TREE_CHAIN (f)) + for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f)) { t = remap_decl (f, id); DECL_CONTEXT (t) = new_tree; - TREE_CHAIN (t) = nf; + DECL_CHAIN (t) = nf; nf = t; } TYPE_FIELDS (new_tree) = nreverse (nf); @@ -427,26 +502,84 @@ remap_type (tree type, copy_body_data *id) return tmp; } +/* Return previously remapped type of TYPE in ID. Return NULL if TYPE + is NULL or TYPE has not been remapped before. */ + static tree -remap_decls (tree decls, copy_body_data *id) +remapped_type (tree type, copy_body_data *id) +{ + tree *node; + + if (type == NULL) + return type; + + /* See if we have remapped this type. */ + node = (tree *) pointer_map_contains (id->decl_map, type); + if (node) + return *node; + else + return NULL; +} + + /* The type only needs remapping if it's variably modified. */ +/* Decide if DECL can be put into BLOCK_NONLOCAL_VARs. */ + +static bool +can_be_nonlocal (tree decl, copy_body_data *id) +{ + /* We can not duplicate function decls. */ + if (TREE_CODE (decl) == FUNCTION_DECL) + return true; + + /* Local static vars must be non-local or we get multiple declaration + problems. */ + if (TREE_CODE (decl) == VAR_DECL + && !auto_var_in_fn_p (decl, id->src_fn)) + return true; + + /* At the moment dwarf2out can handle only these types of nodes. We + can support more later. */ + if (TREE_CODE (decl) != VAR_DECL && TREE_CODE (decl) != PARM_DECL) + return false; + + /* We must use global type. We call remapped_type instead of + remap_type since we don't want to remap this type here if it + hasn't been remapped before. */ + if (TREE_TYPE (decl) != remapped_type (TREE_TYPE (decl), id)) + return false; + + /* Wihtout SSA we can't tell if variable is used. */ + if (!gimple_in_ssa_p (cfun)) + return false; + + /* Live variables must be copied so we can attach DECL_RTL. */ + if (var_ann (decl)) + return false; + + return true; +} + +static tree +remap_decls (tree decls, VEC(tree,gc) **nonlocalized_list, copy_body_data *id) { tree old_var; tree new_decls = NULL_TREE; /* Remap its variables. */ - for (old_var = decls; old_var; old_var = TREE_CHAIN (old_var)) + for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var)) { tree new_var; - /* We cannot chain the local static declarations into the local_decls - as we can't duplicate them or break one decl rule. Go ahead - and link them into local_decls. */ - - if (!auto_var_in_fn_p (old_var, id->src_fn) - && !DECL_EXTERNAL (old_var)) + if (can_be_nonlocal (old_var, id)) { - cfun->local_decls = tree_cons (NULL_TREE, old_var, - cfun->local_decls); + if (TREE_CODE (old_var) == VAR_DECL + && ! DECL_EXTERNAL (old_var) + && (var_ann (old_var) || !gimple_in_ssa_p (cfun))) + add_local_decl (cfun, old_var); + if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE) + && !DECL_IGNORED_P (old_var) + && nonlocalized_list) + VEC_safe_push (tree, gc, *nonlocalized_list, old_var); continue; } @@ -456,13 +589,34 @@ remap_decls (tree decls, copy_body_data *id) /* If we didn't remap this variable, we can't mess with its TREE_CHAIN. If we remapped this variable to the return slot, it's already declared somewhere else, so don't declare it here. */ - if (!new_var || new_var == id->retvar) + + if (new_var == id->retvar) ; + else if (!new_var) + { + if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE) + && !DECL_IGNORED_P (old_var) + && nonlocalized_list) + VEC_safe_push (tree, gc, *nonlocalized_list, old_var); + } else { gcc_assert (DECL_P (new_var)); - TREE_CHAIN (new_var) = new_decls; + DECL_CHAIN (new_var) = new_decls; new_decls = new_var; + + /* Also copy value-expressions. */ + if (TREE_CODE (new_var) == VAR_DECL + && DECL_HAS_VALUE_EXPR_P (new_var)) + { + tree tem = DECL_VALUE_EXPR (new_var); + bool old_regimplify = id->regimplify; + id->remapping_type_depth++; + walk_tree (&tem, copy_tree_body_r, id, NULL); + id->remapping_type_depth--; + id->regimplify = old_regimplify; + SET_DECL_VALUE_EXPR (new_var, tem); + } } } @@ -477,7 +631,6 @@ remap_block (tree *block, copy_body_data *id) { tree old_block; tree new_block; - tree fn; /* Make the new block. */ old_block = *block; @@ -485,12 +638,14 @@ remap_block (tree *block, copy_body_data *id) TREE_USED (new_block) = TREE_USED (old_block); BLOCK_ABSTRACT_ORIGIN (new_block) = old_block; BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block); + BLOCK_NONLOCALIZED_VARS (new_block) + = VEC_copy (tree, gc, BLOCK_NONLOCALIZED_VARS (old_block)); *block = new_block; /* Remap its variables. */ - BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block), id); - - fn = id->dst_fn; + BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block), + &BLOCK_NONLOCALIZED_VARS (new_block), + id); if (id->transform_lang_insert_block) id->transform_lang_insert_block (new_block); @@ -512,7 +667,10 @@ remap_blocks (tree block, copy_body_data *id) remap_block (&new_tree, id); gcc_assert (new_tree != block); for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t)) - add_lexical_block (new_tree, remap_blocks (t, id)); + prepend_lexical_block (new_tree, remap_blocks (t, id)); + /* Blocks are in arbitrary order, but make things slightly prettier and do + not swap order when producing a copy. */ + BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree)); return new_tree; } @@ -525,10 +683,19 @@ copy_statement_list (tree *tp) new_tree = alloc_stmt_list (); ni = tsi_start (new_tree); oi = tsi_start (*tp); + TREE_TYPE (new_tree) = TREE_TYPE (*tp); *tp = new_tree; for (; !tsi_end_p (oi); tsi_next (&oi)) - tsi_link_after (&ni, tsi_stmt (oi), TSI_NEW_STMT); + { + tree stmt = tsi_stmt (oi); + if (TREE_CODE (stmt) == STATEMENT_LIST) + /* This copy is not redundant; tsi_link_after will smash this + STATEMENT_LIST into the end of the one we're building, and we + don't want to do that with the original. */ + copy_statement_list (&stmt); + tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING); + } } static void @@ -546,14 +713,14 @@ copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id) if (BIND_EXPR_VARS (*tp)) /* This will remap a lot of the same decls again, but this should be harmless. */ - BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), id); + BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id); } /* Create a new gimple_seq by remapping all the statements in BODY using the inlining information in ID. */ -gimple_seq +static gimple_seq remap_gimple_seq (gimple_seq body, copy_body_data *id) { gimple_stmt_iterator si; @@ -592,7 +759,7 @@ copy_gimple_bind (gimple stmt, copy_body_data *id) harmless. */ new_vars = gimple_bind_vars (stmt); if (new_vars) - new_vars = remap_decls (new_vars, id); + new_vars = remap_decls (new_vars, NULL, id); new_bind = gimple_build_bind (new_vars, new_body, new_block); @@ -632,6 +799,13 @@ remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data) gcc_assert (new_decl); /* Replace this variable with the copy. */ STRIP_TYPE_NOPS (new_decl); + /* ??? The C++ frontend uses void * pointer zero to initialize + any other type. This confuses the middle-end type verification. + As cloned bodies do not go through gimplification again the fixup + there doesn't trigger. */ + if (TREE_CODE (new_decl) == INTEGER_CST + && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl))) + new_decl = fold_convert (TREE_TYPE (*tp), new_decl); *tp = new_decl; *walk_subtrees = 0; } @@ -669,69 +843,45 @@ remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data) { /* Otherwise, just copy the node. Note that copy_tree_r already knows not to copy VAR_DECLs, etc., so this is safe. */ - if (TREE_CODE (*tp) == INDIRECT_REF) - { - /* Get rid of *& from inline substitutions that can happen when a - pointer argument is an ADDR_EXPR. */ - tree decl = TREE_OPERAND (*tp, 0); - tree *n; - n = (tree *) pointer_map_contains (id->decl_map, decl); - if (n) - { - tree type, new_tree, old; + /* We should never have TREE_BLOCK set on non-statements. */ + if (EXPR_P (*tp)) + gcc_assert (!TREE_BLOCK (*tp)); - /* If we happen to get an ADDR_EXPR in n->value, strip - it manually here as we'll eventually get ADDR_EXPRs - which lie about their types pointed to. In this case - build_fold_indirect_ref wouldn't strip the - INDIRECT_REF, but we absolutely rely on that. As - fold_indirect_ref does other useful transformations, - try that first, though. */ - type = TREE_TYPE (TREE_TYPE (*n)); - new_tree = unshare_expr (*n); - old = *tp; - *tp = gimple_fold_indirect_ref (new_tree); - if (!*tp) - { - if (TREE_CODE (new_tree) == ADDR_EXPR) - { - *tp = fold_indirect_ref_1 (type, new_tree); - /* ??? We should either assert here or build - a VIEW_CONVERT_EXPR instead of blindly leaking - incompatible types to our IL. */ - if (! *tp) - *tp = TREE_OPERAND (new_tree, 0); - } - else - { - *tp = build1 (INDIRECT_REF, type, new_tree); - TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old); - } - } - *walk_subtrees = 0; - return NULL; - } + if (TREE_CODE (*tp) == MEM_REF) + { + tree ptr = TREE_OPERAND (*tp, 0); + tree type = remap_type (TREE_TYPE (*tp), id); + tree old = *tp; + + /* We need to re-canonicalize MEM_REFs from inline substitutions + that can happen when a pointer argument is an ADDR_EXPR. + Recurse here manually to allow that. */ + walk_tree (&ptr, remap_gimple_op_r, data, NULL); + *tp = fold_build2 (MEM_REF, type, + ptr, TREE_OPERAND (*tp, 1)); + TREE_THIS_NOTRAP (*tp) = TREE_THIS_NOTRAP (old); + TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old); + TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old); + *walk_subtrees = 0; + return NULL; } /* Here is the "usual case". Copy this tree node, and then tweak some special cases. */ copy_tree_r (tp, walk_subtrees, NULL); + if (TREE_CODE (*tp) != OMP_CLAUSE) + TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id); + /* Global variables we haven't seen yet need to go into referenced vars. If not referenced from types only. */ if (gimple_in_ssa_p (cfun) && TREE_CODE (*tp) == VAR_DECL - && id->remapping_type_depth == 0) + && id->remapping_type_depth == 0 + && !processing_debug_stmt) add_referenced_var (*tp); - /* We should never have TREE_BLOCK set on non-statements. */ - if (EXPR_P (*tp)) - gcc_assert (!TREE_BLOCK (*tp)); - - if (TREE_CODE (*tp) != OMP_CLAUSE) - TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id); - if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3)) { /* The copied TARGET_EXPR has never been expanded, even if the @@ -742,21 +892,15 @@ remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data) else if (TREE_CODE (*tp) == ADDR_EXPR) { /* Variable substitution need not be simple. In particular, - the INDIRECT_REF substitution above. Make sure that + the MEM_REF substitution above. Make sure that TREE_CONSTANT and friends are up-to-date. But make sure to not improperly set TREE_BLOCK on some sub-expressions. */ int invariant = is_gimple_min_invariant (*tp); tree block = id->block; id->block = NULL_TREE; - walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL); + walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL); id->block = block; - - /* Handle the case where we substituted an INDIRECT_REF - into the operand of the ADDR_EXPR. */ - if (TREE_CODE (TREE_OPERAND (*tp, 0)) == INDIRECT_REF) - *tp = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0); - else - recompute_tree_invariant_for_addr_expr (*tp); + recompute_tree_invariant_for_addr_expr (*tp); /* If this used to be invariant, but is not any longer, then regimplification is probably needed. */ @@ -837,7 +981,8 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data) } else if (TREE_CODE (*tp) == STATEMENT_LIST) copy_statement_list (tp); - else if (TREE_CODE (*tp) == SAVE_EXPR) + else if (TREE_CODE (*tp) == SAVE_EXPR + || TREE_CODE (*tp) == TARGET_EXPR) remap_save_expr (tp, id->decl_map, walk_subtrees); else if (TREE_CODE (*tp) == LABEL_DECL && (! DECL_CONTEXT (*tp) @@ -893,7 +1038,7 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data) STRIP_TYPE_NOPS (value); if (TREE_CONSTANT (value) || TREE_READONLY (value)) { - *tp = build_empty_stmt (); + *tp = build_empty_stmt (EXPR_LOCATION (*tp)); return copy_tree_body_r (tp, walk_subtrees, data); } } @@ -917,14 +1062,18 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data) but we absolutely rely on that. As fold_indirect_ref does other useful transformations, try that first, though. */ tree type = TREE_TYPE (TREE_TYPE (*n)); - new_tree = unshare_expr (*n); + if (id->do_not_unshare) + new_tree = *n; + else + new_tree = unshare_expr (*n); old = *tp; *tp = gimple_fold_indirect_ref (new_tree); if (! *tp) { if (TREE_CODE (new_tree) == ADDR_EXPR) { - *tp = fold_indirect_ref_1 (type, new_tree); + *tp = fold_indirect_ref_1 (EXPR_LOCATION (new_tree), + type, new_tree); /* ??? We should either assert here or build a VIEW_CONVERT_EXPR instead of blindly leaking incompatible types to our IL. */ @@ -936,47 +1085,64 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data) *tp = build1 (INDIRECT_REF, type, new_tree); TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old); TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old); + TREE_READONLY (*tp) = TREE_READONLY (old); + TREE_THIS_NOTRAP (*tp) = TREE_THIS_NOTRAP (old); } } *walk_subtrees = 0; return NULL; } } + else if (TREE_CODE (*tp) == MEM_REF) + { + /* We need to re-canonicalize MEM_REFs from inline substitutions + that can happen when a pointer argument is an ADDR_EXPR. */ + tree decl = TREE_OPERAND (*tp, 0); + tree *n; + + n = (tree *) pointer_map_contains (id->decl_map, decl); + if (n) + { + tree old = *tp; + *tp = fold_build2 (MEM_REF, TREE_TYPE (*tp), + unshare_expr (*n), TREE_OPERAND (*tp, 1)); + TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old); + TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old); + *walk_subtrees = 0; + return NULL; + } + } /* Here is the "usual case". Copy this tree node, and then tweak some special cases. */ copy_tree_r (tp, walk_subtrees, NULL); /* Global variables we haven't seen yet needs to go into referenced - vars. If not referenced from types only. */ + vars. If not referenced from types or debug stmts only. */ if (gimple_in_ssa_p (cfun) && TREE_CODE (*tp) == VAR_DECL - && id->remapping_type_depth == 0) + && id->remapping_type_depth == 0 + && !processing_debug_stmt) add_referenced_var (*tp); - + /* If EXPR has block defined, map it to newly constructed block. When inlining we want EXPRs without block appear in the block - of function call. */ + of function call if we are not remapping a type. */ if (EXPR_P (*tp)) { - new_block = id->block; + new_block = id->remapping_type_depth == 0 ? id->block : NULL; if (TREE_BLOCK (*tp)) { tree *n; n = (tree *) pointer_map_contains (id->decl_map, TREE_BLOCK (*tp)); - gcc_assert (n); - new_block = *n; + gcc_assert (n || id->remapping_type_depth != 0); + if (n) + new_block = *n; } TREE_BLOCK (*tp) = new_block; } - if (TREE_CODE (*tp) == RESX_EXPR && id->eh_region_offset) - TREE_OPERAND (*tp, 0) = - build_int_cst (NULL_TREE, - id->eh_region_offset - + TREE_INT_CST_LOW (TREE_OPERAND (*tp, 0))); - if (TREE_CODE (*tp) != OMP_CLAUSE) TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id); @@ -1016,6 +1182,35 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data) return NULL_TREE; } +/* Helper for remap_gimple_stmt. Given an EH region number for the + source function, map that to the duplicate EH region number in + the destination function. */ + +static int +remap_eh_region_nr (int old_nr, copy_body_data *id) +{ + eh_region old_r, new_r; + void **slot; + + old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr); + slot = pointer_map_contains (id->eh_map, old_r); + new_r = (eh_region) *slot; + + return new_r->index; +} + +/* Similar, but operate on INTEGER_CSTs. */ + +static tree +remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id) +{ + int old_nr, new_nr; + + old_nr = tree_low_cst (old_t_nr, 0); + new_nr = remap_eh_region_nr (old_nr, id); + + return build_int_cst (integer_type_node, new_nr); +} /* Helper for copy_bb. Remap statement STMT using the inlining information in ID. Return the new statement copy. */ @@ -1026,6 +1221,7 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) gimple copy = NULL; struct walk_stmt_info wi; tree new_block; + bool skip_first = false; /* Begin by recognizing trees that we'll completely rewrite for the inlining context. Our output for these trees is completely @@ -1046,8 +1242,15 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) If RETVAL is just the result decl, the result decl has already been set (e.g. a recent "foo (&result_decl, ...)"); just toss the entire GIMPLE_RETURN. */ - if (retval && TREE_CODE (retval) != RESULT_DECL) - copy = gimple_build_assign (id->retvar, retval); + if (retval + && (TREE_CODE (retval) != RESULT_DECL + && (TREE_CODE (retval) != SSA_NAME + || TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL))) + { + copy = gimple_build_assign (id->retvar, retval); + /* id->retvar is already substituted. Skip it on later remapping. */ + skip_first = true; + } else return gimple_build_nop (); } @@ -1077,7 +1280,7 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) case GIMPLE_TRY: s1 = remap_gimple_seq (gimple_try_eval (stmt), id); s2 = remap_gimple_seq (gimple_try_cleanup (stmt), id); - copy = gimple_build_try (s1, s2, gimple_try_kind (stmt)); + copy = gimple_build_try (s1, s2, gimple_try_kind (stmt)); break; case GIMPLE_WITH_CLEANUP_EXPR: @@ -1156,6 +1359,18 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) (s1, gimple_omp_single_clauses (stmt)); break; + case GIMPLE_OMP_CRITICAL: + s1 = remap_gimple_seq (gimple_omp_body (stmt), id); + copy + = gimple_build_omp_critical (s1, gimple_omp_critical_name (stmt)); + break; + + case GIMPLE_TRANSACTION: + s1 = remap_gimple_seq (gimple_transaction_body (stmt), id); + copy = gimple_build_transaction (s1, gimple_transaction_label (stmt)); + gimple_transaction_set_subcode (copy, gimple_transaction_subcode (stmt)); + break; + default: gcc_unreachable (); } @@ -1186,8 +1401,81 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) } } + if (gimple_debug_bind_p (stmt)) + { + copy = gimple_build_debug_bind (gimple_debug_bind_get_var (stmt), + gimple_debug_bind_get_value (stmt), + stmt); + VEC_safe_push (gimple, heap, id->debug_stmts, copy); + return copy; + } + if (gimple_debug_source_bind_p (stmt)) + { + copy = gimple_build_debug_source_bind + (gimple_debug_source_bind_get_var (stmt), + gimple_debug_source_bind_get_value (stmt), stmt); + VEC_safe_push (gimple, heap, id->debug_stmts, copy); + return copy; + } + /* Create a new deep copy of the statement. */ copy = gimple_copy (stmt); + + /* Remap the region numbers for __builtin_eh_{pointer,filter}, + RESX and EH_DISPATCH. */ + if (id->eh_map) + switch (gimple_code (copy)) + { + case GIMPLE_CALL: + { + tree r, fndecl = gimple_call_fndecl (copy); + if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) + switch (DECL_FUNCTION_CODE (fndecl)) + { + case BUILT_IN_EH_COPY_VALUES: + r = gimple_call_arg (copy, 1); + r = remap_eh_region_tree_nr (r, id); + gimple_call_set_arg (copy, 1, r); + /* FALLTHRU */ + + case BUILT_IN_EH_POINTER: + case BUILT_IN_EH_FILTER: + r = gimple_call_arg (copy, 0); + r = remap_eh_region_tree_nr (r, id); + gimple_call_set_arg (copy, 0, r); + break; + + default: + break; + } + + /* Reset alias info if we didn't apply measures to + keep it valid over inlining by setting DECL_PT_UID. */ + if (!id->src_cfun->gimple_df + || !id->src_cfun->gimple_df->ipa_pta) + gimple_call_reset_alias_info (copy); + } + break; + + case GIMPLE_RESX: + { + int r = gimple_resx_region (copy); + r = remap_eh_region_nr (r, id); + gimple_resx_set_region (copy, r); + } + break; + + case GIMPLE_EH_DISPATCH: + { + int r = gimple_eh_dispatch_region (copy); + r = remap_eh_region_nr (r, id); + gimple_eh_dispatch_set_region (copy, r); + } + break; + + default: + break; + } } /* If STMT has a block defined, map it to the newly constructed @@ -1204,17 +1492,25 @@ remap_gimple_stmt (gimple stmt, copy_body_data *id) gimple_set_block (copy, new_block); + if (gimple_debug_bind_p (copy) || gimple_debug_source_bind_p (copy)) + return copy; + /* Remap all the operands in COPY. */ memset (&wi, 0, sizeof (wi)); wi.info = id; - walk_gimple_op (copy, remap_gimple_op_r, &wi); + if (skip_first) + walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL); + else + walk_gimple_op (copy, remap_gimple_op_r, &wi); - /* We have to handle EH region remapping of GIMPLE_RESX specially because - the region number is not an operand. */ - if (gimple_code (stmt) == GIMPLE_RESX && id->eh_region_offset) + /* Clear the copied virtual operands. We are not remapping them here + but are going to recreate them from scratch. */ + if (gimple_has_mem_ops (copy)) { - gimple_resx_set_region (copy, gimple_resx_region (stmt) + id->eh_region_offset); + gimple_set_vdef (copy, NULL_TREE); + gimple_set_vuse (copy, NULL_TREE); } + return copy; } @@ -1226,23 +1522,31 @@ static basic_block copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, gcov_type count_scale) { - gimple_stmt_iterator gsi, copy_gsi; + gimple_stmt_iterator gsi, copy_gsi, seq_gsi; basic_block copy_basic_block; tree decl; + gcov_type freq; + basic_block prev; + + /* Search for previous copied basic block. */ + prev = bb->prev_bb; + while (!prev->aux) + prev = prev->prev_bb; /* create_basic_block() will append every new block to basic_block_info automatically. */ copy_basic_block = create_basic_block (NULL, (void *) 0, - (basic_block) bb->prev_bb->aux); + (basic_block) prev->aux); copy_basic_block->count = bb->count * count_scale / REG_BR_PROB_BASE; /* We are going to rebuild frequencies from scratch. These values have just small importance to drive canonicalize_loop_headers. */ - copy_basic_block->frequency = ((gcov_type)bb->frequency - * frequency_scale / REG_BR_PROB_BASE); + freq = ((gcov_type)bb->frequency * frequency_scale / REG_BR_PROB_BASE); - if (copy_basic_block->frequency > BB_FREQ_MAX) - copy_basic_block->frequency = BB_FREQ_MAX; + /* We recompute frequencies after inlining, so this is quite safe. */ + if (freq > BB_FREQ_MAX) + freq = BB_FREQ_MAX; + copy_basic_block->frequency = freq; copy_gsi = gsi_start_bb (copy_basic_block); @@ -1257,6 +1561,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, continue; gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun, orig_stmt); + seq_gsi = copy_gsi; /* With return slot optimization we can end up with non-gimple (foo *)&this->m, fix that here. */ @@ -1265,21 +1570,34 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, && !is_gimple_val (gimple_assign_rhs1 (stmt))) { tree new_rhs; - new_rhs = force_gimple_operand_gsi (©_gsi, - gimple_assign_rhs1 (stmt), - true, NULL, true, GSI_SAME_STMT); + new_rhs = force_gimple_operand_gsi (&seq_gsi, + gimple_assign_rhs1 (stmt), + true, NULL, false, + GSI_CONTINUE_LINKING); gimple_assign_set_rhs1 (stmt, new_rhs); + id->regimplify = false; } - else if (id->regimplify) - gimple_regimplify_operands (stmt, ©_gsi); - gsi_insert_after (©_gsi, stmt, GSI_NEW_STMT); + gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT); + + if (id->regimplify) + gimple_regimplify_operands (stmt, &seq_gsi); + + /* If copy_basic_block has been empty at the start of this iteration, + call gsi_start_bb again to get at the newly added statements. */ + if (gsi_end_p (copy_gsi)) + copy_gsi = gsi_start_bb (copy_basic_block); + else + gsi_next (©_gsi); /* Process the new statement. The call to gimple_regimplify_operands possibly turned the statement into multiple statements, we need to process all of them. */ - while (!gsi_end_p (copy_gsi)) + do { + tree fn; + + stmt = gsi_stmt (copy_gsi); if (is_gimple_call (stmt) && gimple_call_va_arg_pack_p (stmt) && id->gimple_call) @@ -1292,7 +1610,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, size_t nargs = gimple_call_num_args (id->gimple_call); size_t n; - for (p = DECL_ARGUMENTS (id->src_fn); p; p = TREE_CHAIN (p)) + for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p)) nargs--; /* Create the new array of arguments. */ @@ -1339,7 +1657,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, tree count, p; gimple new_stmt; - for (p = DECL_ARGUMENTS (id->src_fn); p; p = TREE_CHAIN (p)) + for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p)) nargs--; count = build_int_cst (integer_type_node, nargs); @@ -1366,80 +1684,111 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, callgraph edges and update or duplicate them. */ if (is_gimple_call (stmt)) { - struct cgraph_node *node; struct cgraph_edge *edge; + int flags; switch (id->transform_call_graph_edges) { - case CB_CGE_DUPLICATE: - edge = cgraph_edge (id->src_node, orig_stmt); - if (edge) - cgraph_clone_edge (edge, id->dst_node, stmt, - REG_BR_PROB_BASE, 1, - edge->frequency, true); - break; - - case CB_CGE_MOVE_CLONES: - for (node = id->dst_node->next_clone; - node; - node = node->next_clone) - { - edge = cgraph_edge (node, orig_stmt); - if (edge) - cgraph_set_call_stmt (edge, stmt); - } - /* FALLTHRU */ - - case CB_CGE_MOVE: - edge = cgraph_edge (id->dst_node, orig_stmt); - if (edge) - cgraph_set_call_stmt (edge, stmt); - break; + case CB_CGE_DUPLICATE: + edge = cgraph_edge (id->src_node, orig_stmt); + if (edge) + { + int edge_freq = edge->frequency; + edge = cgraph_clone_edge (edge, id->dst_node, stmt, + gimple_uid (stmt), + REG_BR_PROB_BASE, CGRAPH_FREQ_BASE, + true); + /* We could also just rescale the frequency, but + doing so would introduce roundoff errors and make + verifier unhappy. */ + edge->frequency + = compute_call_stmt_bb_frequency (id->dst_node->decl, + copy_basic_block); + if (dump_file + && profile_status_for_function (cfun) != PROFILE_ABSENT + && (edge_freq > edge->frequency + 10 + || edge_freq < edge->frequency - 10)) + { + fprintf (dump_file, "Edge frequency estimated by " + "cgraph %i diverge from inliner's estimate %i\n", + edge_freq, + edge->frequency); + fprintf (dump_file, + "Orig bb: %i, orig bb freq %i, new bb freq %i\n", + bb->index, + bb->frequency, + copy_basic_block->frequency); + } + stmt = cgraph_redirect_edge_call_stmt_to_callee (edge); + } + break; + + case CB_CGE_MOVE_CLONES: + cgraph_set_call_stmt_including_clones (id->dst_node, + orig_stmt, stmt); + edge = cgraph_edge (id->dst_node, stmt); + break; + + case CB_CGE_MOVE: + edge = cgraph_edge (id->dst_node, orig_stmt); + if (edge) + cgraph_set_call_stmt (edge, stmt); + break; + + default: + gcc_unreachable (); + } - default: - gcc_unreachable (); + /* Constant propagation on argument done during inlining + may create new direct call. Produce an edge for it. */ + if ((!edge + || (edge->indirect_inlining_edge + && id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)) + && id->dst_node->analyzed + && (fn = gimple_call_fndecl (stmt)) != NULL) + { + struct cgraph_node *dest = cgraph_get_node (fn); + + /* We have missing edge in the callgraph. This can happen + when previous inlining turned an indirect call into a + direct call by constant propagating arguments or we are + producing dead clone (for further cloning). In all + other cases we hit a bug (incorrect node sharing is the + most common reason for missing edges). */ + gcc_assert (dest->needed || !dest->analyzed + || dest->address_taken + || !id->src_node->analyzed + || !id->dst_node->analyzed); + if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES) + cgraph_create_edge_including_clones + (id->dst_node, dest, orig_stmt, stmt, bb->count, + compute_call_stmt_bb_frequency (id->dst_node->decl, + copy_basic_block), + CIF_ORIGINALLY_INDIRECT_CALL); + else + cgraph_create_edge (id->dst_node, dest, stmt, + bb->count, + compute_call_stmt_bb_frequency + (id->dst_node->decl, copy_basic_block))->inline_failed + = CIF_ORIGINALLY_INDIRECT_CALL; + if (dump_file) + { + fprintf (dump_file, "Created new direct edge to %s\n", + cgraph_node_name (dest)); + } } - } - /* If you think we can abort here, you are wrong. - There is no region 0 in gimple. */ - gcc_assert (lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) != 0); - - if (stmt_could_throw_p (stmt) - /* When we are cloning for inlining, we are supposed to - construct a clone that calls precisely the same functions - as original. However IPA optimizers might've proved - earlier some function calls as non-trapping that might - render some basic blocks dead that might become - unreachable. - - We can't update SSA with unreachable blocks in CFG and thus - we prevent the scenario by preserving even the "dead" eh - edges until the point they are later removed by - fixup_cfg pass. */ - || (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES - && lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) > 0)) - { - int region = lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt); - - /* Add an entry for the copied tree in the EH hashtable. - When cloning or versioning, use the hashtable in - cfun, and just copy the EH number. When inlining, use the - hashtable in the caller, and adjust the region number. */ - if (region > 0) - add_stmt_to_eh_region (stmt, region + id->eh_region_offset); - - /* If this tree doesn't have a region associated with it, - and there is a "current region," - then associate this tree with the current region - and add edges associated with this region. */ - if (lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) <= 0 - && id->eh_region > 0 - && stmt_could_throw_p (stmt)) - add_stmt_to_eh_region (stmt, id->eh_region); + flags = gimple_call_flags (stmt); + if (flags & ECF_MAY_BE_ALLOCA) + cfun->calls_alloca = true; + if (flags & ECF_RETURNS_TWICE) + cfun->calls_setjmp = true; } - if (gimple_in_ssa_p (cfun)) + maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt, + id->eh_map, id->eh_lp_nr); + + if (gimple_in_ssa_p (cfun) && !is_gimple_debug (stmt)) { ssa_op_iter i; tree def; @@ -1452,6 +1801,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, gsi_next (©_gsi); } + while (!gsi_end_p (copy_gsi)); copy_gsi = gsi_last_bb (copy_basic_block); } @@ -1491,8 +1841,6 @@ update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb, gimple phi; gimple_stmt_iterator si; - gcc_assert (e->flags & EDGE_ABNORMAL); - if (!nonlocal_goto) gcc_assert (e->flags & EDGE_EH); @@ -1508,7 +1856,8 @@ update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb, /* There shouldn't be any PHI nodes in the ENTRY_BLOCK. */ gcc_assert (!e->dest->aux); - gcc_assert (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi))); + gcc_assert ((e->flags & EDGE_EH) + || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi))); if (!is_gimple_reg (PHI_RESULT (phi))) { @@ -1530,9 +1879,10 @@ update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb, /* Copy edges from BB into its copy constructed earlier, scale profile accordingly. Edges will be taken care of later. Assume aux - pointers to point to the copies of each BB. */ + pointers to point to the copies of each BB. Return true if any + debug stmts are left after a statement that must end the basic block. */ -static void +static bool copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) { basic_block new_bb = (basic_block) bb->aux; @@ -1540,6 +1890,7 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) edge old_edge; gimple_stmt_iterator si; int flags; + bool need_debug_cleanup = false; /* Use the indices from the original blocks to create edges for the new ones. */ @@ -1560,7 +1911,7 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) } if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK) - return; + return false; for (si = gsi_start_bb (new_bb); !gsi_end_p (si);) { @@ -1568,9 +1919,12 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) bool can_throw, nonlocal_goto; copy_stmt = gsi_stmt (si); - update_stmt (copy_stmt); - if (gimple_in_ssa_p (cfun)) - mark_symbols_for_renaming (copy_stmt); + if (!is_gimple_debug (copy_stmt)) + { + update_stmt (copy_stmt); + if (gimple_in_ssa_p (cfun)) + mark_symbols_for_renaming (copy_stmt); + } /* Do this before the possible split_block. */ gsi_next (&si); @@ -1592,6 +1946,13 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) if (can_throw || nonlocal_goto) { if (!gsi_end_p (si)) + { + while (!gsi_end_p (si) && is_gimple_debug (gsi_stmt (si))) + gsi_next (&si); + if (gsi_end_p (si)) + need_debug_cleanup = true; + } + if (!gsi_end_p (si)) /* Note that bb's predecessor edges aren't necessarily right at this point; split_block doesn't care. */ { @@ -1603,7 +1964,9 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) } } - if (can_throw) + if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH) + make_eh_dispatch_edges (copy_stmt); + else if (can_throw) make_eh_edges (copy_stmt); if (nonlocal_goto) @@ -1614,6 +1977,7 @@ copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb) update_ssa_across_abnormal_edges (gimple_bb (copy_stmt), ret_bb, can_throw, nonlocal_goto); } + return need_debug_cleanup; } /* Copy the PHIs. All blocks and edges are copied, some blocks @@ -1628,12 +1992,13 @@ copy_phis_for_bb (basic_block bb, copy_body_data *id) edge_iterator ei; gimple phi; gimple_stmt_iterator si; + edge new_edge; + bool inserted = false; for (si = gsi_start (phi_nodes (bb)); !gsi_end_p (si); gsi_next (&si)) { tree res, new_res; gimple new_phi; - edge new_edge; phi = gsi_stmt (si); res = PHI_RESULT (phi); @@ -1645,11 +2010,22 @@ copy_phis_for_bb (basic_block bb, copy_body_data *id) = new_phi = create_phi_node (new_res, new_bb); FOR_EACH_EDGE (new_edge, ei, new_bb->preds) { - edge const old_edge - = find_edge ((basic_block) new_edge->src->aux, bb); - tree arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge); - tree new_arg = arg; + edge old_edge = find_edge ((basic_block) new_edge->src->aux, bb); + tree arg; + tree new_arg; tree block = id->block; + edge_iterator ei2; + + /* When doing partial cloning, we allow PHIs on the entry block + as long as all the arguments are the same. Find any input + edge to see argument to copy. */ + if (!old_edge) + FOR_EACH_EDGE (old_edge, ei2, bb->preds) + if (!old_edge->src->aux) + break; + + arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge); + new_arg = arg; id->block = NULL_TREE; walk_tree (&new_arg, copy_tree_body_r, id, NULL); id->block = block; @@ -1662,12 +2038,19 @@ copy_phis_for_bb (basic_block bb, copy_body_data *id) { gimple_seq stmts = NULL; new_arg = force_gimple_operand (new_arg, &stmts, true, NULL); - gsi_insert_seq_on_edge_immediate (new_edge, stmts); + gsi_insert_seq_on_edge (new_edge, stmts); + inserted = true; } - add_phi_arg (new_phi, new_arg, new_edge); + add_phi_arg (new_phi, new_arg, new_edge, + gimple_phi_arg_location_from_edge (phi, old_edge)); } } } + + /* Commit the delayed edge insertions. */ + if (inserted) + FOR_EACH_EDGE (new_edge, ei, new_bb->preds) + gsi_commit_one_edge_insert (new_edge, NULL); } @@ -1683,51 +2066,59 @@ remap_decl_1 (tree decl, void *data) NEW_FNDECL to be build. CALLEE_FNDECL is the original */ static void -initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count, - int frequency) +initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count) { - struct function *new_cfun - = (struct function *) ggc_alloc_cleared (sizeof (struct function)); struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl); - gcov_type count_scale, frequency_scale; + gcov_type count_scale; if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count) count_scale = (REG_BR_PROB_BASE * count / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count); else - count_scale = 1; - - if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency) - frequency_scale = (REG_BR_PROB_BASE * frequency - / - ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency); - else - frequency_scale = count_scale; + count_scale = REG_BR_PROB_BASE; /* Register specific tree functions. */ gimple_register_cfg_hooks (); - *new_cfun = *DECL_STRUCT_FUNCTION (callee_fndecl); - new_cfun->funcdef_no = get_next_funcdef_no (); - VALUE_HISTOGRAMS (new_cfun) = NULL; - new_cfun->local_decls = NULL; - new_cfun->cfg = NULL; - new_cfun->decl = new_fndecl /*= copy_node (callee_fndecl)*/; - DECL_STRUCT_FUNCTION (new_fndecl) = new_cfun; - push_cfun (new_cfun); + + /* Get clean struct function. */ + push_struct_function (new_fndecl); + + /* We will rebuild these, so just sanity check that they are empty. */ + gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL); + gcc_assert (cfun->local_decls == NULL); + gcc_assert (cfun->cfg == NULL); + gcc_assert (cfun->decl == new_fndecl); + + /* Copy items we preserve during cloning. */ + cfun->static_chain_decl = src_cfun->static_chain_decl; + cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area; + cfun->function_end_locus = src_cfun->function_end_locus; + cfun->curr_properties = src_cfun->curr_properties; + cfun->last_verified = src_cfun->last_verified; + cfun->va_list_gpr_size = src_cfun->va_list_gpr_size; + cfun->va_list_fpr_size = src_cfun->va_list_fpr_size; + cfun->has_nonlocal_label = src_cfun->has_nonlocal_label; + cfun->stdarg = src_cfun->stdarg; + cfun->after_inlining = src_cfun->after_inlining; + cfun->can_throw_non_call_exceptions + = src_cfun->can_throw_non_call_exceptions; + cfun->returns_struct = src_cfun->returns_struct; + cfun->returns_pcc_struct = src_cfun->returns_pcc_struct; + cfun->after_tree_profile = src_cfun->after_tree_profile; + init_empty_tree_cfg (); + profile_status_for_function (cfun) = profile_status_for_function (src_cfun); ENTRY_BLOCK_PTR->count = (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale / REG_BR_PROB_BASE); - ENTRY_BLOCK_PTR->frequency = - (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency * - frequency_scale / REG_BR_PROB_BASE); + ENTRY_BLOCK_PTR->frequency + = ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency; EXIT_BLOCK_PTR->count = (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale / REG_BR_PROB_BASE); EXIT_BLOCK_PTR->frequency = - (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency * - frequency_scale / REG_BR_PROB_BASE); + EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency; if (src_cfun->eh) init_eh_for_function (); @@ -1740,12 +2131,81 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count, pop_cfun (); } +/* Helper function for copy_cfg_body. Move debug stmts from the end + of NEW_BB to the beginning of successor basic blocks when needed. If the + successor has multiple predecessors, reset them, otherwise keep + their value. */ + +static void +maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb) +{ + edge e; + edge_iterator ei; + gimple_stmt_iterator si = gsi_last_nondebug_bb (new_bb); + + if (gsi_end_p (si) + || gsi_one_before_end_p (si) + || !(stmt_can_throw_internal (gsi_stmt (si)) + || stmt_can_make_abnormal_goto (gsi_stmt (si)))) + return; + + FOR_EACH_EDGE (e, ei, new_bb->succs) + { + gimple_stmt_iterator ssi = gsi_last_bb (new_bb); + gimple_stmt_iterator dsi = gsi_after_labels (e->dest); + while (is_gimple_debug (gsi_stmt (ssi))) + { + gimple stmt = gsi_stmt (ssi), new_stmt; + tree var; + tree value; + + /* For the last edge move the debug stmts instead of copying + them. */ + if (ei_one_before_end_p (ei)) + { + si = ssi; + gsi_prev (&ssi); + if (!single_pred_p (e->dest) && gimple_debug_bind_p (stmt)) + gimple_debug_bind_reset_value (stmt); + gsi_remove (&si, false); + gsi_insert_before (&dsi, stmt, GSI_SAME_STMT); + continue; + } + + if (gimple_debug_bind_p (stmt)) + { + var = gimple_debug_bind_get_var (stmt); + if (single_pred_p (e->dest)) + { + value = gimple_debug_bind_get_value (stmt); + value = unshare_expr (value); + } + else + value = NULL_TREE; + new_stmt = gimple_build_debug_bind (var, value, stmt); + } + else if (gimple_debug_source_bind_p (stmt)) + { + var = gimple_debug_source_bind_get_var (stmt); + value = gimple_debug_source_bind_get_value (stmt); + new_stmt = gimple_build_debug_source_bind (var, value, stmt); + } + else + gcc_unreachable (); + gsi_insert_before (&dsi, new_stmt, GSI_SAME_STMT); + VEC_safe_push (gimple, heap, id->debug_stmts, new_stmt); + gsi_prev (&ssi); + } + } +} + /* Make a copy of the body of FN so that it can be inserted inline in another function. Walks FN via CFG, returns new fndecl. */ static tree -copy_cfg_body (copy_body_data * id, gcov_type count, int frequency, - basic_block entry_block_map, basic_block exit_block_map) +copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale, + basic_block entry_block_map, basic_block exit_block_map, + bitmap blocks_to_copy, basic_block new_entry) { tree callee_fndecl = id->src_fn; /* Original cfun for the callee, doesn't change. */ @@ -1753,25 +2213,43 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency, struct function *cfun_to_copy; basic_block bb; tree new_fndecl = NULL; - gcov_type count_scale, frequency_scale; + bool need_debug_cleanup = false; + gcov_type count_scale; int last; + int incoming_frequency = 0; + gcov_type incoming_count = 0; if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count) count_scale = (REG_BR_PROB_BASE * count / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count); else - count_scale = 1; - - if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency) - frequency_scale = (REG_BR_PROB_BASE * frequency - / - ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency); - else - frequency_scale = count_scale; + count_scale = REG_BR_PROB_BASE; /* Register specific tree functions. */ gimple_register_cfg_hooks (); + /* If we are inlining just region of the function, make sure to connect new entry + to ENTRY_BLOCK_PTR. Since new entry can be part of loop, we must compute + frequency and probability of ENTRY_BLOCK_PTR based on the frequencies and + probabilities of edges incoming from nonduplicated region. */ + if (new_entry) + { + edge e; + edge_iterator ei; + + FOR_EACH_EDGE (e, ei, new_entry->preds) + if (!e->src->aux) + { + incoming_frequency += EDGE_FREQUENCY (e); + incoming_count += e->count; + } + incoming_count = incoming_count * count_scale / REG_BR_PROB_BASE; + incoming_frequency + = incoming_frequency * frequency_scale / REG_BR_PROB_BASE; + ENTRY_BLOCK_PTR->count = incoming_count; + ENTRY_BLOCK_PTR->frequency = incoming_frequency; + } + /* Must have a CFG here at this point. */ gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (callee_fndecl))); @@ -1785,56 +2263,193 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency, /* Duplicate any exception-handling regions. */ if (cfun->eh) - { - id->eh_region_offset - = duplicate_eh_regions (cfun_to_copy, remap_decl_1, id, - 0, id->eh_region); - } + id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr, + remap_decl_1, id); /* Use aux pointers to map the original blocks to copy. */ FOR_EACH_BB_FN (bb, cfun_to_copy) - { - basic_block new_bb = copy_bb (id, bb, frequency_scale, count_scale); - bb->aux = new_bb; - new_bb->aux = bb; - } + if (!blocks_to_copy || bitmap_bit_p (blocks_to_copy, bb->index)) + { + basic_block new_bb = copy_bb (id, bb, frequency_scale, count_scale); + bb->aux = new_bb; + new_bb->aux = bb; + } last = last_basic_block; /* Now that we've duplicated the blocks, duplicate their edges. */ FOR_ALL_BB_FN (bb, cfun_to_copy) - copy_edges_for_bb (bb, count_scale, exit_block_map); + if (!blocks_to_copy + || (bb->index > 0 && bitmap_bit_p (blocks_to_copy, bb->index))) + need_debug_cleanup |= copy_edges_for_bb (bb, count_scale, exit_block_map); + + if (new_entry) + { + edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU); + e->probability = REG_BR_PROB_BASE; + e->count = incoming_count; + } if (gimple_in_ssa_p (cfun)) FOR_ALL_BB_FN (bb, cfun_to_copy) - copy_phis_for_bb (bb, id); + if (!blocks_to_copy + || (bb->index > 0 && bitmap_bit_p (blocks_to_copy, bb->index))) + copy_phis_for_bb (bb, id); FOR_ALL_BB_FN (bb, cfun_to_copy) - { - ((basic_block)bb->aux)->aux = NULL; - bb->aux = NULL; - } + if (bb->aux) + { + if (need_debug_cleanup + && bb->index != ENTRY_BLOCK + && bb->index != EXIT_BLOCK) + maybe_move_debug_stmts_to_successors (id, (basic_block) bb->aux); + ((basic_block)bb->aux)->aux = NULL; + bb->aux = NULL; + } /* Zero out AUX fields of newly created block during EH edge insertion. */ for (; last < last_basic_block; last++) - BASIC_BLOCK (last)->aux = NULL; + { + if (need_debug_cleanup) + maybe_move_debug_stmts_to_successors (id, BASIC_BLOCK (last)); + BASIC_BLOCK (last)->aux = NULL; + } entry_block_map->aux = NULL; exit_block_map->aux = NULL; + if (id->eh_map) + { + pointer_map_destroy (id->eh_map); + id->eh_map = NULL; + } + return new_fndecl; } +/* Copy the debug STMT using ID. We deal with these statements in a + special way: if any variable in their VALUE expression wasn't + remapped yet, we won't remap it, because that would get decl uids + out of sync, causing codegen differences between -g and -g0. If + this arises, we drop the VALUE expression altogether. */ + +static void +copy_debug_stmt (gimple stmt, copy_body_data *id) +{ + tree t, *n; + struct walk_stmt_info wi; + + t = id->block; + if (gimple_block (stmt)) + { + n = (tree *) pointer_map_contains (id->decl_map, gimple_block (stmt)); + if (n) + t = *n; + } + gimple_set_block (stmt, t); + + /* Remap all the operands in COPY. */ + memset (&wi, 0, sizeof (wi)); + wi.info = id; + + processing_debug_stmt = 1; + + if (gimple_debug_source_bind_p (stmt)) + t = gimple_debug_source_bind_get_var (stmt); + else + t = gimple_debug_bind_get_var (stmt); + + if (TREE_CODE (t) == PARM_DECL && id->debug_map + && (n = (tree *) pointer_map_contains (id->debug_map, t))) + { + gcc_assert (TREE_CODE (*n) == VAR_DECL); + t = *n; + } + else if (TREE_CODE (t) == VAR_DECL + && !TREE_STATIC (t) + && gimple_in_ssa_p (cfun) + && !pointer_map_contains (id->decl_map, t) + && !var_ann (t)) + /* T is a non-localized variable. */; + else + walk_tree (&t, remap_gimple_op_r, &wi, NULL); + + if (gimple_debug_bind_p (stmt)) + { + gimple_debug_bind_set_var (stmt, t); + + if (gimple_debug_bind_has_value_p (stmt)) + walk_tree (gimple_debug_bind_get_value_ptr (stmt), + remap_gimple_op_r, &wi, NULL); + + /* Punt if any decl couldn't be remapped. */ + if (processing_debug_stmt < 0) + gimple_debug_bind_reset_value (stmt); + } + else if (gimple_debug_source_bind_p (stmt)) + { + gimple_debug_source_bind_set_var (stmt, t); + walk_tree (gimple_debug_source_bind_get_value_ptr (stmt), + remap_gimple_op_r, &wi, NULL); + } + + processing_debug_stmt = 0; + + update_stmt (stmt); + if (gimple_in_ssa_p (cfun)) + mark_symbols_for_renaming (stmt); +} + +/* Process deferred debug stmts. In order to give values better odds + of being successfully remapped, we delay the processing of debug + stmts until all other stmts that might require remapping are + processed. */ + +static void +copy_debug_stmts (copy_body_data *id) +{ + size_t i; + gimple stmt; + + if (!id->debug_stmts) + return; + + FOR_EACH_VEC_ELT (gimple, id->debug_stmts, i, stmt) + copy_debug_stmt (stmt, id); + + VEC_free (gimple, heap, id->debug_stmts); +} + +/* Make a copy of the body of SRC_FN so that it can be inserted inline in + another function. */ + +static tree +copy_tree_body (copy_body_data *id) +{ + tree fndecl = id->src_fn; + tree body = DECL_SAVED_TREE (fndecl); + + walk_tree (&body, copy_tree_body_r, id, NULL); + + return body; +} + +/* Make a copy of the body of FN so that it can be inserted inline in + another function. */ + static tree -copy_body (copy_body_data *id, gcov_type count, int frequency, - basic_block entry_block_map, basic_block exit_block_map) +copy_body (copy_body_data *id, gcov_type count, int frequency_scale, + basic_block entry_block_map, basic_block exit_block_map, + bitmap blocks_to_copy, basic_block new_entry) { tree fndecl = id->src_fn; tree body; /* If this body has a CFG, walk CFG and copy. */ gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl))); - body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map); + body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map, + blocks_to_copy, new_entry); + copy_debug_stmts (id); return body; } @@ -1855,43 +2470,91 @@ self_inlining_addr_expr (tree value, tree fn) return var && auto_var_in_fn_p (var, fn); } -static void -insert_init_stmt (basic_block bb, gimple init_stmt) +/* Append to BB a debug annotation that binds VAR to VALUE, inheriting + lexical block and line number information from base_stmt, if given, + or from the last stmt of the block otherwise. */ + +static gimple +insert_init_debug_bind (copy_body_data *id, + basic_block bb, tree var, tree value, + gimple base_stmt) { - gimple_stmt_iterator si = gsi_last_bb (bb); - gimple_stmt_iterator i; - gimple_seq seq = gimple_seq_alloc (); - struct gimplify_ctx gctx; + gimple note; + gimple_stmt_iterator gsi; + tree tracked_var; - push_gimplify_context (&gctx); + if (!gimple_in_ssa_p (id->src_cfun)) + return NULL; - i = gsi_start (seq); - gimple_regimplify_operands (init_stmt, &i); + if (!MAY_HAVE_DEBUG_STMTS) + return NULL; - if (gimple_in_ssa_p (cfun) - && init_stmt - && !gimple_seq_empty_p (seq)) + tracked_var = target_for_debug_bind (var); + if (!tracked_var) + return NULL; + + if (bb) { - /* The replacement can expose previously unreferenced - variables. */ - for (i = gsi_start (seq); !gsi_end_p (i); gsi_next (&i)) - find_new_referenced_vars (gsi_stmt (i)); + gsi = gsi_last_bb (bb); + if (!base_stmt && !gsi_end_p (gsi)) + base_stmt = gsi_stmt (gsi); + } - /* Insert the gimplified sequence needed for INIT_STMT - after SI. INIT_STMT will be inserted after SEQ. */ - gsi_insert_seq_after (&si, seq, GSI_NEW_STMT); - } + note = gimple_build_debug_bind (tracked_var, value, base_stmt); - pop_gimplify_context (NULL); + if (bb) + { + if (!gsi_end_p (gsi)) + gsi_insert_after (&gsi, note, GSI_SAME_STMT); + else + gsi_insert_before (&gsi, note, GSI_SAME_STMT); + } + return note; +} + +static void +insert_init_stmt (copy_body_data *id, basic_block bb, gimple init_stmt) +{ /* If VAR represents a zero-sized variable, it's possible that the assignment statement may result in no gimple statements. */ if (init_stmt) - gsi_insert_after (&si, init_stmt, GSI_NEW_STMT); + { + gimple_stmt_iterator si = gsi_last_bb (bb); + + /* We can end up with init statements that store to a non-register + from a rhs with a conversion. Handle that here by forcing the + rhs into a temporary. gimple_regimplify_operands is not + prepared to do this for us. */ + if (!is_gimple_debug (init_stmt) + && !is_gimple_reg (gimple_assign_lhs (init_stmt)) + && is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt))) + && gimple_assign_rhs_class (init_stmt) == GIMPLE_UNARY_RHS) + { + tree rhs = build1 (gimple_assign_rhs_code (init_stmt), + gimple_expr_type (init_stmt), + gimple_assign_rhs1 (init_stmt)); + rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false, + GSI_NEW_STMT); + gimple_assign_set_rhs_code (init_stmt, TREE_CODE (rhs)); + gimple_assign_set_rhs1 (init_stmt, rhs); + } + gsi_insert_after (&si, init_stmt, GSI_NEW_STMT); + gimple_regimplify_operands (init_stmt, &si); + mark_symbols_for_renaming (init_stmt); - if (gimple_in_ssa_p (cfun)) - for (;!gsi_end_p (si); gsi_next (&si)) - mark_symbols_for_renaming (gsi_stmt (si)); + if (!is_gimple_debug (init_stmt) && MAY_HAVE_DEBUG_STMTS) + { + tree var, def = gimple_assign_lhs (init_stmt); + + if (TREE_CODE (def) == SSA_NAME) + var = SSA_NAME_VAR (def); + else + var = def; + + insert_init_debug_bind (id, bb, var, def, init_stmt); + } + } } /* Initialize parameter P with VALUE. If needed, produce init statement @@ -1911,19 +2574,43 @@ setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn, && value != error_mark_node && !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value))) { + /* If we can match up types by promotion/demotion do so. */ if (fold_convertible_p (TREE_TYPE (p), value)) - rhs = fold_build1 (NOP_EXPR, TREE_TYPE (p), value); + rhs = fold_convert (TREE_TYPE (p), value); else - /* ??? For valid (GIMPLE) programs we should not end up here. - Still if something has gone wrong and we end up with truly - mismatched types here, fall back to using a VIEW_CONVERT_EXPR - to not leak invalid GIMPLE to the following passes. */ - rhs = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (p), value); + { + /* ??? For valid programs we should not end up here. + Still if we end up with truly mismatched types here, fall back + to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid + GIMPLE to the following passes. */ + if (!is_gimple_reg_type (TREE_TYPE (value)) + || TYPE_SIZE (TREE_TYPE (p)) == TYPE_SIZE (TREE_TYPE (value))) + rhs = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (p), value); + else + rhs = build_zero_cst (TREE_TYPE (p)); + } } + /* Make an equivalent VAR_DECL. Note that we must NOT remap the type + here since the type of this decl must be visible to the calling + function. */ + var = copy_decl_to_var (p, id); + + /* We're actually using the newly-created var. */ + if (gimple_in_ssa_p (cfun) && TREE_CODE (var) == VAR_DECL) + add_referenced_var (var); + + /* Declare this new variable. */ + DECL_CHAIN (var) = *vars; + *vars = var; + + /* Make gimplifier happy about this variable. */ + DECL_SEEN_IN_BIND_EXPR_P (var) = 1; + /* If the parameter is never assigned to, has no SSA_NAMEs created, - we may not need to create a new variable here at all. Instead, we may - be able to just use the argument value. */ + we would not need to create a new variable here at all, if it + weren't for debug info. Still, we can just use the argument + value. */ if (TREE_READONLY (p) && !TREE_ADDRESSABLE (p) && value && !TREE_SIDE_EFFECTS (value) @@ -1944,32 +2631,16 @@ setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn, && ! self_inlining_addr_expr (value, fn)) { insert_decl_map (id, p, value); - return NULL; + insert_debug_decl_map (id, p, var); + return insert_init_debug_bind (id, bb, var, value, NULL); } } - /* Make an equivalent VAR_DECL. Note that we must NOT remap the type - here since the type of this decl must be visible to the calling - function. */ - var = copy_decl_to_var (p, id); - if (gimple_in_ssa_p (cfun) && TREE_CODE (var) == VAR_DECL) - { - get_var_ann (var); - add_referenced_var (var); - } - /* Register the VAR_DECL as the equivalent for the PARM_DECL; that way, when the PARM_DECL is encountered, it will be automatically replaced by the VAR_DECL. */ insert_decl_map (id, p, var); - /* Declare this new variable. */ - TREE_CHAIN (var) = *vars; - *vars = var; - - /* Make gimplifier happy about this variable. */ - DECL_SEEN_IN_BIND_EXPR_P (var) = 1; - /* Even if P was TREE_READONLY, the new VAR should not be. In the original code, we would have constructed a temporary, and then the function body would have never @@ -1989,28 +2660,27 @@ setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn, We need to construct map for the variable anyway as it might be used in different SSA names when parameter is set in function. - FIXME: This usually kills the last connection in between inlined - function parameter and the actual value in debug info. Can we do - better here? If we just inserted the statement, copy propagation - would kill it anyway as it always did in older versions of GCC. - - We might want to introduce a notion that single SSA_NAME might - represent multiple variables for purposes of debugging. */ + Do replacement at -O0 for const arguments replaced by constant. + This is important for builtin_constant_p and other construct requiring + constant argument to be visible in inlined function body. */ if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p) + && (optimize + || (TREE_READONLY (p) + && is_gimple_min_invariant (rhs))) && (TREE_CODE (rhs) == SSA_NAME || is_gimple_min_invariant (rhs)) && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def)) { insert_decl_map (id, def, rhs); - return NULL; + return insert_init_debug_bind (id, bb, var, rhs, NULL); } /* If the value of argument is never used, don't care about initializing it. */ - if (gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p)) + if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p)) { gcc_assert (!value || !TREE_SIDE_EFFECTS (value)); - return NULL; + return insert_init_debug_bind (id, bb, var, rhs, NULL); } /* Initialize this VAR_DECL from the equivalent argument. Convert @@ -2020,7 +2690,7 @@ setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn, if (rhs == error_mark_node) { insert_decl_map (id, p, var); - return NULL; + return insert_init_debug_bind (id, bb, var, rhs, NULL); } STRIP_USELESS_TYPE_CONVERSION (rhs); @@ -2038,7 +2708,7 @@ setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn, init_stmt = gimple_build_assign (var, rhs); if (bb && init_stmt) - insert_init_stmt (bb, init_stmt); + insert_init_stmt (id, bb, init_stmt); } return init_stmt; } @@ -2061,12 +2731,39 @@ initialize_inlined_parameters (copy_body_data *id, gimple stmt, /* Loop through the parameter declarations, replacing each with an equivalent VAR_DECL, appropriately initialized. */ - for (p = parms, i = 0; p; p = TREE_CHAIN (p), i++) + for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++) { tree val; val = i < gimple_call_num_args (stmt) ? gimple_call_arg (stmt, i) : NULL; setup_one_parameter (id, p, val, fn, bb, &vars); } + /* After remapping parameters remap their types. This has to be done + in a second loop over all parameters to appropriately remap + variable sized arrays when the size is specified in a + parameter following the array. */ + for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++) + { + tree *varp = (tree *) pointer_map_contains (id->decl_map, p); + if (varp + && TREE_CODE (*varp) == VAR_DECL) + { + tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p) + ? gimple_default_def (id->src_cfun, p) : NULL); + tree var = *varp; + TREE_TYPE (var) = remap_type (TREE_TYPE (var), id); + /* Also remap the default definition if it was remapped + to the default definition of the parameter replacement + by the parameter setup. */ + if (def) + { + tree *defp = (tree *) pointer_map_contains (id->decl_map, def); + if (defp + && TREE_CODE (*defp) == SSA_NAME + && SSA_NAME_VAR (*defp) == var) + TREE_TYPE (*defp) = TREE_TYPE (var); + } + } + } /* Initialize the static chain. */ p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl; @@ -2092,28 +2789,29 @@ initialize_inlined_parameters (copy_body_data *id, gimple stmt, is set only for CALL_EXPR_RETURN_SLOT_OPT. MODIFY_DEST, if non-null, was the LHS of the MODIFY_EXPR to which this call is the RHS. - The return value is a (possibly null) value that is the result of the - function as seen by the callee. *USE_P is a (possibly null) value that - holds the result as seen by the caller. */ + The return value is a (possibly null) value that holds the result + as seen by the caller. */ static tree declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest, - tree *use_p) + basic_block entry_bb) { tree callee = id->src_fn; - tree caller = id->dst_fn; tree result = DECL_RESULT (callee); tree callee_type = TREE_TYPE (result); - tree caller_type = TREE_TYPE (TREE_TYPE (callee)); + tree caller_type; tree var, use; - /* We don't need to do anything for functions that don't return - anything. */ - if (!result || VOID_TYPE_P (callee_type)) - { - *use_p = NULL_TREE; - return NULL_TREE; - } + /* Handle type-mismatches in the function declaration return type + vs. the call expression. */ + if (modify_dest) + caller_type = TREE_TYPE (modify_dest); + else + caller_type = TREE_TYPE (TREE_TYPE (callee)); + + /* We don't need to do anything for functions that don't return anything. */ + if (VOID_TYPE_P (callee_type)) + return NULL_TREE; /* If there was a return slot, then the return value is the dereferenced address of that object. */ @@ -2128,31 +2826,12 @@ declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest, STRIP_USELESS_TYPE_CONVERSION (return_slot_addr); /* We are going to construct *&return_slot and we can't do that - for variables believed to be not addressable. + for variables believed to be not addressable. FIXME: This check possibly can match, because values returned via return slot optimization are not believed to have address taken by alias analysis. */ gcc_assert (TREE_CODE (return_slot) != SSA_NAME); - if (gimple_in_ssa_p (cfun)) - { - HOST_WIDE_INT bitsize; - HOST_WIDE_INT bitpos; - tree offset; - enum machine_mode mode; - int unsignedp; - int volatilep; - tree base; - base = get_inner_reference (return_slot, &bitsize, &bitpos, - &offset, - &mode, &unsignedp, &volatilep, - false); - if (TREE_CODE (base) == INDIRECT_REF) - base = TREE_OPERAND (base, 0); - if (TREE_CODE (base) == SSA_NAME) - base = SSA_NAME_VAR (base); - mark_sym_for_renaming (base); - } var = return_slot_addr; } else @@ -2225,15 +2904,9 @@ declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest, var = copy_result_decl_to_var (result, id); if (gimple_in_ssa_p (cfun)) - { - get_var_ann (var); - add_referenced_var (var); - } + add_referenced_var (var); DECL_SEEN_IN_BIND_EXPR_P (var) = 1; - DECL_STRUCT_FUNCTION (caller)->local_decls - = tree_cons (NULL_TREE, var, - DECL_STRUCT_FUNCTION (caller)->local_decls); /* Do not have the rest of GCC warn about this variable as it should not be visible to the user. */ @@ -2245,85 +2918,135 @@ declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest, promoted, convert it back to the expected type. */ use = var; if (!useless_type_conversion_p (caller_type, TREE_TYPE (var))) - use = fold_convert (caller_type, var); - + { + /* If we can match up types by promotion/demotion do so. */ + if (fold_convertible_p (caller_type, var)) + use = fold_convert (caller_type, var); + else + { + /* ??? For valid programs we should not end up here. + Still if we end up with truly mismatched types here, fall back + to using a MEM_REF to not leak invalid GIMPLE to the following + passes. */ + /* Prevent var from being written into SSA form. */ + if (TREE_CODE (TREE_TYPE (var)) == VECTOR_TYPE + || TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE) + DECL_GIMPLE_REG_P (var) = false; + else if (is_gimple_reg_type (TREE_TYPE (var))) + TREE_ADDRESSABLE (var) = true; + use = fold_build2 (MEM_REF, caller_type, + build_fold_addr_expr (var), + build_int_cst (ptr_type_node, 0)); + } + } + STRIP_USELESS_TYPE_CONVERSION (use); if (DECL_BY_REFERENCE (result)) - var = build_fold_addr_expr (var); + { + TREE_ADDRESSABLE (var) = 1; + var = build_fold_addr_expr (var); + } done: /* Register the VAR_DECL as the equivalent for the RESULT_DECL; that way, when the RESULT_DECL is encountered, it will be - automatically replaced by the VAR_DECL. */ - insert_decl_map (id, result, var); + automatically replaced by the VAR_DECL. + + When returning by reference, ensure that RESULT_DECL remaps to + gimple_val. */ + if (DECL_BY_REFERENCE (result) + && !is_gimple_val (var)) + { + tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr"); + if (gimple_in_ssa_p (id->src_cfun)) + add_referenced_var (temp); + insert_decl_map (id, result, temp); + /* When RESULT_DECL is in SSA form, we need to use it's default_def + SSA_NAME. */ + if (gimple_in_ssa_p (id->src_cfun) && gimple_default_def (id->src_cfun, result)) + temp = remap_ssa_name (gimple_default_def (id->src_cfun, result), id); + insert_init_stmt (id, entry_bb, gimple_build_assign (temp, var)); + } + else + insert_decl_map (id, result, var); /* Remember this so we can ignore it in remap_decls. */ id->retvar = var; - *use_p = use; - return var; + return use; } -/* Returns nonzero if a function can be inlined as a tree. */ +/* Callback through walk_tree. Determine if a DECL_INITIAL makes reference + to a local label. */ -bool -tree_inlinable_function_p (tree fn) +static tree +has_label_address_in_static_1 (tree *nodep, int *walk_subtrees, void *fnp) { - bool ret = inlinable_function_p (fn); + tree node = *nodep; + tree fn = (tree) fnp; + + if (TREE_CODE (node) == LABEL_DECL && DECL_CONTEXT (node) == fn) + return node; - if (getenv ("TUPLES_INLINE")) - fprintf (stderr, "Function %s is %sinlinable\n", get_name (fn), - ret ? "" : "not "); + if (TYPE_P (node)) + *walk_subtrees = 0; - return ret; + return NULL_TREE; } -static const char *inline_forbidden_reason; - -/* A callback for walk_gimple_seq to handle tree operands. Returns - NULL_TREE if a function can be inlined, otherwise sets the reason - why not and returns a tree representing the offending operand. */ +/* Determine if the function can be copied. If so return NULL. If + not return a string describng the reason for failure. */ -static tree -inline_forbidden_p_op (tree *nodep, int *walk_subtrees ATTRIBUTE_UNUSED, - void *fnp ATTRIBUTE_UNUSED) +static const char * +copy_forbidden (struct function *fun, tree fndecl) { - tree node = *nodep; - tree t; - - if (TREE_CODE (node) == RECORD_TYPE || TREE_CODE (node) == UNION_TYPE) + const char *reason = fun->cannot_be_copied_reason; + tree decl; + unsigned ix; + + /* Only examine the function once. */ + if (fun->cannot_be_copied_set) + return reason; + + /* We cannot copy a function that receives a non-local goto + because we cannot remap the destination label used in the + function that is performing the non-local goto. */ + /* ??? Actually, this should be possible, if we work at it. + No doubt there's just a handful of places that simply + assume it doesn't happen and don't substitute properly. */ + if (fun->has_nonlocal_label) { - /* We cannot inline a function of the form - - void F (int i) { struct S { int ar[i]; } s; } - - Attempting to do so produces a catch-22. - If walk_tree examines the TYPE_FIELDS chain of RECORD_TYPE/ - UNION_TYPE nodes, then it goes into infinite recursion on a - structure containing a pointer to its own type. If it doesn't, - then the type node for S doesn't get adjusted properly when - F is inlined. - - ??? This is likely no longer true, but it's too late in the 4.0 - cycle to try to find out. This should be checked for 4.1. */ - for (t = TYPE_FIELDS (node); t; t = TREE_CHAIN (t)) - if (variably_modified_type_p (TREE_TYPE (t), NULL)) - { - inline_forbidden_reason - = G_("function %q+F can never be inlined " - "because it uses variable sized variables"); - return node; - } + reason = G_("function %q+F can never be copied " + "because it receives a non-local goto"); + goto fail; } - return NULL_TREE; + FOR_EACH_LOCAL_DECL (fun, ix, decl) + if (TREE_CODE (decl) == VAR_DECL + && TREE_STATIC (decl) + && !DECL_EXTERNAL (decl) + && DECL_INITIAL (decl) + && walk_tree_without_duplicates (&DECL_INITIAL (decl), + has_label_address_in_static_1, + fndecl)) + { + reason = G_("function %q+F can never be copied because it saves " + "address of local label in a static variable"); + goto fail; + } + + fail: + fun->cannot_be_copied_reason = reason; + fun->cannot_be_copied_set = true; + return reason; } -/* A callback for walk_gimple_seq to handle statements. Returns - non-NULL iff a function can not be inlined. Also sets the reason - why. */ +static const char *inline_forbidden_reason; + +/* A callback for walk_gimple_seq to handle statements. Returns non-null + iff a function can not be inlined. Also sets the reason why. */ static tree inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, @@ -2340,8 +3063,11 @@ inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, this may change program's memory overhead drastically when the function using alloca is called in loop. In GCC present in SPEC2000 inlining into schedule_block cause it to require 2GB of - RAM instead of 256MB. */ + RAM instead of 256MB. Don't do so for alloca calls emitted for + VLA objects as those can't cause unbounded growth (they're always + wrapped inside stack_save/stack_restore regions. */ if (gimple_alloca_call_p (stmt) + && !gimple_call_alloca_for_var_p (stmt) && !lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn))) { inline_forbidden_reason @@ -2424,56 +3150,19 @@ inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, instantiations, which causes unexpected behavior. */ if (TREE_CODE (t) != LABEL_DECL) { - inline_forbidden_reason - = G_("function %q+F can never be inlined " - "because it contains a computed goto"); - *handled_ops_p = true; - return t; - } - break; - - case GIMPLE_LABEL: - t = gimple_label_label (stmt); - if (DECL_NONLOCAL (t)) - { - /* We cannot inline a function that receives a non-local goto - because we cannot remap the destination label used in the - function that is performing the non-local goto. */ - inline_forbidden_reason - = G_("function %q+F can never be inlined " - "because it receives a non-local goto"); - *handled_ops_p = true; - return t; - } - break; - - default: - break; - } - - *handled_ops_p = false; - return NULL_TREE; -} - - -static tree -inline_forbidden_p_2 (tree *nodep, int *walk_subtrees, - void *fnp) -{ - tree node = *nodep; - tree fn = (tree) fnp; + inline_forbidden_reason + = G_("function %q+F can never be inlined " + "because it contains a computed goto"); + *handled_ops_p = true; + return t; + } + break; - if (TREE_CODE (node) == LABEL_DECL && DECL_CONTEXT (node) == fn) - { - inline_forbidden_reason - = G_("function %q+F can never be inlined " - "because it saves address of local label in a static variable"); - return node; + default: + break; } - if (TYPE_P (node)) - *walk_subtrees = 0; - + *handled_ops_p = false; return NULL_TREE; } @@ -2483,14 +3172,19 @@ inline_forbidden_p_2 (tree *nodep, int *walk_subtrees, static bool inline_forbidden_p (tree fndecl) { - location_t saved_loc = input_location; struct function *fun = DECL_STRUCT_FUNCTION (fndecl); - tree step; struct walk_stmt_info wi; struct pointer_set_t *visited_nodes; basic_block bb; bool forbidden_p = false; + /* First check for shared reasons not to copy the code. */ + inline_forbidden_reason = copy_forbidden (fun, fndecl); + if (inline_forbidden_reason != NULL) + return true; + + /* Next, walk the statements of the function looking for + constraucts we can't handle, or are non-optimal for inlining. */ visited_nodes = pointer_set_create (); memset (&wi, 0, sizeof (wi)); wi.info = (void *) fndecl; @@ -2500,41 +3194,21 @@ inline_forbidden_p (tree fndecl) { gimple ret; gimple_seq seq = bb_seq (bb); - ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, - inline_forbidden_p_op, &wi); + ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi); forbidden_p = (ret != NULL); if (forbidden_p) - goto egress; - } - - for (step = fun->local_decls; step; step = TREE_CHAIN (step)) - { - tree decl = TREE_VALUE (step); - if (TREE_CODE (decl) == VAR_DECL - && TREE_STATIC (decl) - && !DECL_EXTERNAL (decl) - && DECL_INITIAL (decl)) - { - tree ret; - ret = walk_tree_without_duplicates (&DECL_INITIAL (decl), - inline_forbidden_p_2, fndecl); - forbidden_p = (ret != NULL); - if (forbidden_p) - goto egress; - } + break; } -egress: pointer_set_destroy (visited_nodes); - input_location = saved_loc; return forbidden_p; } /* Returns nonzero if FN is a function that does not have any fundamental inline blocking properties. */ -static bool -inlinable_function_p (tree fn) +bool +tree_inlinable_function_p (tree fn) { bool inlinable = true; bool do_warning; @@ -2548,6 +3222,7 @@ inlinable_function_p (tree fn) /* We only warn for functions declared `inline' by the user. */ do_warning = (warn_inline && DECL_DECLARED_INLINE_P (fn) + && !DECL_NO_INLINE_WARNING_P (fn) && !DECL_IN_SYSTEM_HEADER (fn)); always_inline = lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)); @@ -2561,12 +3236,6 @@ inlinable_function_p (tree fn) inlinable = false; } - /* Don't auto-inline anything that might not be bound within - this unit of translation. */ - else if (!DECL_DECLARED_INLINE_P (fn) - && DECL_REPLACEABLE_P (fn)) - inlinable = false; - else if (!function_attribute_inlinable_p (fn)) { if (do_warning) @@ -2575,12 +3244,6 @@ inlinable_function_p (tree fn) inlinable = false; } - /* If we don't have the function body available, we can't inline it. - However, this should not be recorded since we also get here for - forward declared inline functions. Therefore, return at once. */ - if (!gimple_body (fn)) - return false; - else if (inline_forbidden_p (fn)) { /* See if we should warn about uninlinable functions. Previously, @@ -2591,7 +3254,7 @@ inlinable_function_p (tree fn) As a bonus we can now give more details about the reason why a function is not inlinable. */ if (always_inline) - sorry (inline_forbidden_reason, fn); + error (inline_forbidden_reason, fn); else if (do_warning) warning (OPT_Winline, inline_forbidden_reason, fn); @@ -2612,9 +3275,21 @@ estimate_move_cost (tree type) { HOST_WIDE_INT size; + gcc_assert (!VOID_TYPE_P (type)); + + if (TREE_CODE (type) == VECTOR_TYPE) + { + enum machine_mode inner = TYPE_MODE (TREE_TYPE (type)); + enum machine_mode simd + = targetm.vectorize.preferred_simd_mode (inner); + int simd_mode_size = GET_MODE_SIZE (simd); + return ((GET_MODE_SIZE (TYPE_MODE (type)) + simd_mode_size - 1) + / simd_mode_size); + } + size = int_size_in_bytes (type); - if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO) + if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (!optimize_size)) /* Cost of a memcpy call, 3 arguments and the call. */ return 4; else @@ -2624,7 +3299,8 @@ estimate_move_cost (tree type) /* Returns cost of operation CODE, according to WEIGHTS */ static int -estimate_operator_cost (enum tree_code code, eni_weights *weights) +estimate_operator_cost (enum tree_code code, eni_weights *weights, + tree op1 ATTRIBUTE_UNUSED, tree op2) { switch (code) { @@ -2634,18 +3310,22 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights) CASE_CONVERT: case COMPLEX_EXPR: case PAREN_EXPR: + case VIEW_CONVERT_EXPR: return 0; /* Assign cost of 1 to usual operations. ??? We may consider mapping RTL costs to this. */ case COND_EXPR: case VEC_COND_EXPR: + case VEC_PERM_EXPR: case PLUS_EXPR: case POINTER_PLUS_EXPR: case MINUS_EXPR: case MULT_EXPR: + case FMA_EXPR: + case ADDR_SPACE_CONVERT_EXPR: case FIXED_CONVERT_EXPR: case FIX_TRUNC_EXPR: @@ -2705,6 +3385,9 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights) case WIDEN_SUM_EXPR: case WIDEN_MULT_EXPR: case DOT_PROD_EXPR: + case WIDEN_MULT_PLUS_EXPR: + case WIDEN_MULT_MINUS_EXPR: + case WIDEN_LSHIFT_EXPR: case VEC_WIDEN_MULT_HI_EXPR: case VEC_WIDEN_MULT_LO_EXPR: @@ -2715,10 +3398,8 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights) case VEC_PACK_TRUNC_EXPR: case VEC_PACK_SAT_EXPR: case VEC_PACK_FIX_TRUNC_EXPR: - case VEC_EXTRACT_EVEN_EXPR: - case VEC_EXTRACT_ODD_EXPR: - case VEC_INTERLEAVE_HIGH_EXPR: - case VEC_INTERLEAVE_LOW_EXPR: + case VEC_WIDEN_LSHIFT_HI_EXPR: + case VEC_WIDEN_LSHIFT_LO_EXPR: return 1; @@ -2734,7 +3415,9 @@ estimate_operator_cost (enum tree_code code, eni_weights *weights) case FLOOR_MOD_EXPR: case ROUND_MOD_EXPR: case RDIV_EXPR: - return weights->div_mod_cost; + if (TREE_CODE (op2) != INTEGER_CST) + return weights->div_mod_cost; + return 1; default: /* We expect a copy assignment with no operator. */ @@ -2771,6 +3454,7 @@ estimate_num_insns (gimple stmt, eni_weights *weights) unsigned cost, i; enum gimple_code code = gimple_code (stmt); tree lhs; + tree rhs; switch (code) { @@ -2793,102 +3477,118 @@ estimate_num_insns (gimple stmt, eni_weights *weights) likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost of moving something into "a", which we compute using the function estimate_move_cost. */ + if (gimple_clobber_p (stmt)) + return 0; /* ={v} {CLOBBER} stmt expands to nothing. */ + lhs = gimple_assign_lhs (stmt); + rhs = gimple_assign_rhs1 (stmt); + if (is_gimple_reg (lhs)) cost = 0; else cost = estimate_move_cost (TREE_TYPE (lhs)); - cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights); + if (!is_gimple_reg (rhs) && !is_gimple_min_invariant (rhs)) + cost += estimate_move_cost (TREE_TYPE (rhs)); + + cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights, + gimple_assign_rhs1 (stmt), + get_gimple_rhs_class (gimple_assign_rhs_code (stmt)) + == GIMPLE_BINARY_RHS + ? gimple_assign_rhs2 (stmt) : NULL); break; case GIMPLE_COND: - cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights); + cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights, + gimple_op (stmt, 0), + gimple_op (stmt, 1)); break; case GIMPLE_SWITCH: /* Take into account cost of the switch + guess 2 conditional jumps for - each case label. + each case label. TODO: once the switch expansion logic is sufficiently separated, we can do better job on estimating cost of the switch. */ - cost = gimple_switch_num_labels (stmt) * 2; + if (weights->time_based) + cost = floor_log2 (gimple_switch_num_labels (stmt)) * 2; + else + cost = gimple_switch_num_labels (stmt) * 2; break; case GIMPLE_CALL: { tree decl = gimple_call_fndecl (stmt); - tree addr = gimple_call_fn (stmt); - tree funtype = TREE_TYPE (addr); - - if (POINTER_TYPE_P (funtype)) - funtype = TREE_TYPE (funtype); - - if (decl && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_MD) - cost = weights->target_builtin_call_cost; - else - cost = weights->call_cost; - - if (decl && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL) - switch (DECL_FUNCTION_CODE (decl)) - { - case BUILT_IN_CONSTANT_P: - return 0; - case BUILT_IN_EXPECT: - cost = 0; - break; - - /* Prefetch instruction is not expensive. */ - case BUILT_IN_PREFETCH: - cost = weights->target_builtin_call_cost; - break; - - default: - break; - } - - if (decl) - funtype = TREE_TYPE (decl); - - /* Our cost must be kept in sync with - cgraph_estimate_size_after_inlining that does use function - declaration to figure out the arguments. */ - if (decl && DECL_ARGUMENTS (decl)) - { - tree arg; - for (arg = DECL_ARGUMENTS (decl); arg; arg = TREE_CHAIN (arg)) - cost += estimate_move_cost (TREE_TYPE (arg)); - } - else if (funtype && prototype_p (funtype)) - { - tree t; - for (t = TYPE_ARG_TYPES (funtype); t; t = TREE_CHAIN (t)) - cost += estimate_move_cost (TREE_VALUE (t)); - } - else + struct cgraph_node *node = NULL; + + /* Do not special case builtins where we see the body. + This just confuse inliner. */ + if (!decl || !(node = cgraph_get_node (decl)) || node->analyzed) + ; + /* For buitins that are likely expanded to nothing or + inlined do not account operand costs. */ + else if (is_simple_builtin (decl)) + return 0; + else if (is_inexpensive_builtin (decl)) + return weights->target_builtin_call_cost; + else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL) { - for (i = 0; i < gimple_call_num_args (stmt); i++) + /* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so + specialize the cheap expansion we do here. + ??? This asks for a more general solution. */ + switch (DECL_FUNCTION_CODE (decl)) { - tree arg = gimple_call_arg (stmt, i); - cost += estimate_move_cost (TREE_TYPE (arg)); + case BUILT_IN_POW: + case BUILT_IN_POWF: + case BUILT_IN_POWL: + if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST + && REAL_VALUES_EQUAL + (TREE_REAL_CST (gimple_call_arg (stmt, 1)), dconst2)) + return estimate_operator_cost (MULT_EXPR, weights, + gimple_call_arg (stmt, 0), + gimple_call_arg (stmt, 0)); + break; + + default: + break; } } + cost = node ? weights->call_cost : weights->indirect_call_cost; + if (gimple_call_lhs (stmt)) + cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt))); + for (i = 0; i < gimple_call_num_args (stmt); i++) + { + tree arg = gimple_call_arg (stmt, i); + cost += estimate_move_cost (TREE_TYPE (arg)); + } break; } + case GIMPLE_RETURN: + return weights->return_cost; + case GIMPLE_GOTO: case GIMPLE_LABEL: case GIMPLE_NOP: case GIMPLE_PHI: - case GIMPLE_RETURN: - case GIMPLE_CHANGE_DYNAMIC_TYPE: case GIMPLE_PREDICT: + case GIMPLE_DEBUG: return 0; case GIMPLE_ASM: + return asm_str_count (gimple_asm_string (stmt)); + case GIMPLE_RESX: - return 1; + /* This is either going to be an external function call with one + argument, or two register copy statements plus a goto. */ + return 2; + + case GIMPLE_EH_DISPATCH: + /* ??? This is going to turn into a switch statement. Ideally + we'd have a look at the eh region and estimate the number of + edges involved. */ + return 10; case GIMPLE_BIND: return estimate_num_insns_seq (gimple_bind_body (stmt), weights); @@ -2931,6 +3631,11 @@ estimate_num_insns (gimple stmt, eni_weights *weights) return (weights->omp_cost + estimate_num_insns_seq (gimple_omp_body (stmt), weights)); + case GIMPLE_TRANSACTION: + return (weights->tm_cost + + estimate_num_insns_seq (gimple_transaction_body (stmt), + weights)); + default: gcc_unreachable (); } @@ -2966,24 +3671,27 @@ estimate_num_insns_fn (tree fndecl, eni_weights *weights) void init_inline_once (void) { - eni_inlining_weights.call_cost = PARAM_VALUE (PARAM_INLINE_CALL_COST); - eni_inlining_weights.target_builtin_call_cost = 1; - eni_inlining_weights.div_mod_cost = 10; - eni_inlining_weights.omp_cost = 40; - eni_size_weights.call_cost = 1; + eni_size_weights.indirect_call_cost = 3; eni_size_weights.target_builtin_call_cost = 1; eni_size_weights.div_mod_cost = 1; eni_size_weights.omp_cost = 40; + eni_size_weights.tm_cost = 10; + eni_size_weights.time_based = false; + eni_size_weights.return_cost = 1; /* Estimating time for call is difficult, since we have no idea what the called function does. In the current uses of eni_time_weights, underestimating the cost does less harm than overestimating it, so we choose a rather small value here. */ eni_time_weights.call_cost = 10; - eni_time_weights.target_builtin_call_cost = 10; + eni_time_weights.indirect_call_cost = 15; + eni_time_weights.target_builtin_call_cost = 1; eni_time_weights.div_mod_cost = 10; eni_time_weights.omp_cost = 40; + eni_time_weights.tm_cost = 40; + eni_time_weights.time_based = true; + eni_time_weights.return_cost = 2; } /* Estimate the number of instructions in a gimple_seq. */ @@ -3003,31 +3711,48 @@ count_insns_seq (gimple_seq seq, eni_weights *weights) /* Install new lexical TREE_BLOCK underneath 'current_block'. */ static void -add_lexical_block (tree current_block, tree new_block) +prepend_lexical_block (tree current_block, tree new_block) { - tree *blk_p; - - /* Walk to the last sub-block. */ - for (blk_p = &BLOCK_SUBBLOCKS (current_block); - *blk_p; - blk_p = &BLOCK_CHAIN (*blk_p)) - ; - *blk_p = new_block; + BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block); + BLOCK_SUBBLOCKS (current_block) = new_block; BLOCK_SUPERCONTEXT (new_block) = current_block; } -/* Fetch callee declaration from the call graph edge going from NODE and - associated with STMR call statement. Return NULL_TREE if not found. */ -static tree -get_indirect_callee_fndecl (struct cgraph_node *node, gimple stmt) +/* Add local variables from CALLEE to CALLER. */ + +static inline void +add_local_variables (struct function *callee, struct function *caller, + copy_body_data *id, bool check_var_ann) { - struct cgraph_edge *cs; + tree var; + unsigned ix; - cs = cgraph_edge (node, stmt); - if (cs) - return cs->callee->decl; + FOR_EACH_LOCAL_DECL (callee, ix, var) + if (TREE_STATIC (var) && !TREE_ASM_WRITTEN (var)) + { + if (!check_var_ann + || (var_ann (var) && add_referenced_var (var))) + add_local_decl (caller, var); + } + else if (!can_be_nonlocal (var, id)) + { + tree new_var = remap_decl (var, id); - return NULL_TREE; + /* Remap debug-expressions. */ + if (TREE_CODE (new_var) == VAR_DECL + && DECL_DEBUG_EXPR_IS_FROM (new_var) + && new_var != var) + { + tree tem = DECL_DEBUG_EXPR (var); + bool old_regimplify = id->regimplify; + id->remapping_type_depth++; + walk_tree (&tem, copy_tree_body_r, id, NULL); + id->remapping_type_depth--; + id->regimplify = old_regimplify; + SET_DECL_DEBUG_EXPR (new_var, tem); + } + add_local_decl (caller, new_var); + } } /* If STMT is a GIMPLE_CALL, replace it with its inline expansion. */ @@ -3035,21 +3760,19 @@ get_indirect_callee_fndecl (struct cgraph_node *node, gimple stmt) static bool expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) { - tree retvar, use_retvar; + tree use_retvar; tree fn; - struct pointer_map_t *st; + struct pointer_map_t *st, *dst; tree return_slot; tree modify_dest; location_t saved_location; struct cgraph_edge *cg_edge; - const char *reason; + cgraph_inline_failed_t reason; basic_block return_block; edge e; gimple_stmt_iterator gsi, stmt_gsi; bool successfully_inlined = FALSE; bool purge_dead_abnormal_edges; - tree t_step; - tree var; /* Set input_location here so we get the right instantiation context if we call instantiate_decl from inlinable_function_p. */ @@ -3061,18 +3784,14 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) if (gimple_code (stmt) != GIMPLE_CALL) goto egress; + cg_edge = cgraph_edge (id->dst_node, stmt); + gcc_checking_assert (cg_edge); /* First, see if we can figure out what function is being called. If we cannot, then there is no hope of inlining the function. */ - fn = gimple_call_fndecl (stmt); - if (!fn) - { - fn = get_indirect_callee_fndecl (id->dst_node, stmt); - if (!fn) - goto egress; - } - - /* Turn forward declarations into real ones. */ - fn = cgraph_node (fn)->decl; + if (cg_edge->indirect_unknown_callee) + goto egress; + fn = cg_edge->callee->decl; + gcc_checking_assert (fn); /* If FN is a declaration of a function in a nested scope that was globally declared inline, we don't set its DECL_INITIAL. @@ -3083,65 +3802,41 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) gimple_body. */ if (!DECL_INITIAL (fn) && DECL_ABSTRACT_ORIGIN (fn) - && gimple_body (DECL_ABSTRACT_ORIGIN (fn))) + && gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn))) fn = DECL_ABSTRACT_ORIGIN (fn); - /* Objective C and fortran still calls tree_rest_of_compilation directly. - Kill this check once this is fixed. */ - if (!id->dst_node->analyzed) - goto egress; - - cg_edge = cgraph_edge (id->dst_node, stmt); - - /* Constant propagation on argument done during previous inlining - may create new direct call. Produce an edge for it. */ - if (!cg_edge) - { - struct cgraph_node *dest = cgraph_node (fn); - - /* We have missing edge in the callgraph. This can happen in one case - where previous inlining turned indirect call into direct call by - constant propagating arguments. In all other cases we hit a bug - (incorrect node sharing is most common reason for missing edges. */ - gcc_assert (dest->needed); - cgraph_create_edge (id->dst_node, dest, stmt, - bb->count, CGRAPH_FREQ_BASE, - bb->loop_depth)->inline_failed - = N_("originally indirect function call not considered for inlining"); - if (dump_file) - { - fprintf (dump_file, "Created new direct edge to %s", - cgraph_node_name (dest)); - } - goto egress; - } - - /* Don't try to inline functions that are not well-suited to - inlining. */ + /* Don't try to inline functions that are not well-suited to inlining. */ if (!cgraph_inline_p (cg_edge, &reason)) { /* If this call was originally indirect, we do not want to emit any inlining related warnings or sorry messages because there are no guarantees regarding those. */ - if (cg_edge->indirect_call) + if (cg_edge->indirect_inlining_edge) goto egress; if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)) /* Avoid warnings during early inline pass. */ - && cgraph_global_info_ready) + && cgraph_global_info_ready + /* PR 20090218-1_0.c. Body can be provided by another module. */ + && (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto)) { - sorry ("inlining failed in call to %q+F: %s", fn, reason); - sorry ("called from here"); + error ("inlining failed in call to always_inline %q+F: %s", fn, + cgraph_inline_failed_string (reason)); + error ("called from here"); } - else if (warn_inline && DECL_DECLARED_INLINE_P (fn) + else if (warn_inline + && DECL_DECLARED_INLINE_P (fn) + && !DECL_NO_INLINE_WARNING_P (fn) && !DECL_IN_SYSTEM_HEADER (fn) - && strlen (reason) + && reason != CIF_UNSPECIFIED && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fn)) + /* Do not warn about not inlined recursive calls. */ + && !cgraph_edge_recursive_p (cg_edge) /* Avoid warnings during early inline pass. */ && cgraph_global_info_ready) { warning (OPT_Winline, "inlining failed in call to %q+F: %s", - fn, reason); + fn, _(cgraph_inline_failed_string (reason))); warning (OPT_Winline, "called from here"); } goto egress; @@ -3154,7 +3849,12 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) #endif /* We will be inlining this callee. */ - id->eh_region = lookup_stmt_eh_region (stmt); + id->eh_lp_nr = lookup_stmt_eh_lp (stmt); + + /* Update the callers EH personality. */ + if (DECL_FUNCTION_PERSONALITY (cg_edge->callee->decl)) + DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl) + = DECL_FUNCTION_PERSONALITY (cg_edge->callee->decl); /* Split the block holding the GIMPLE_CALL. */ e = split_block (bb, stmt); @@ -3193,12 +3893,14 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) id->block = make_node (BLOCK); BLOCK_ABSTRACT_ORIGIN (id->block) = fn; BLOCK_SOURCE_LOCATION (id->block) = input_location; - add_lexical_block (gimple_block (stmt), id->block); + prepend_lexical_block (gimple_block (stmt), id->block); /* Local declarations will be replaced by their equivalents in this map. */ st = id->decl_map; id->decl_map = pointer_map_create (); + dst = id->debug_map; + id->debug_map = NULL; /* Record the function we are about to inline. */ id->src_fn = fn; @@ -3209,10 +3911,17 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) gcc_assert (!id->src_cfun->after_inlining); id->entry_bb = bb; + if (lookup_attribute ("cold", DECL_ATTRIBUTES (fn))) + { + gimple_stmt_iterator si = gsi_last_bb (bb); + gsi_insert_after (&si, gimple_build_predict (PRED_COLD_FUNCTION, + NOT_TAKEN), + GSI_NEW_STMT); + } initialize_inlined_parameters (id, stmt, fn, bb); if (DECL_INITIAL (fn)) - add_lexical_block (id->block, remap_blocks (DECL_INITIAL (fn), id)); + prepend_lexical_block (id->block, remap_blocks (DECL_INITIAL (fn), id)); /* Return statements in the function body will be replaced by jumps to the RET_LABEL. */ @@ -3255,13 +3964,18 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) } /* Declare the return variable for the function. */ - retvar = declare_return_variable (id, return_slot, modify_dest, &use_retvar); + use_retvar = declare_return_variable (id, return_slot, modify_dest, bb); - if (DECL_IS_OPERATOR_NEW (fn)) + /* Add local vars in this inlined callee to caller. */ + add_local_variables (id->src_cfun, cfun, id, true); + + if (dump_file && (dump_flags & TDF_DETAILS)) { - gcc_assert (TREE_CODE (retvar) == VAR_DECL - && POINTER_TYPE_P (TREE_TYPE (retvar))); - DECL_NO_TBAA_P (retvar) = 1; + fprintf (dump_file, "Inlining "); + print_generic_expr (dump_file, id->src_fn, 0); + fprintf (dump_file, " to "); + print_generic_expr (dump_file, id->dst_fn, 0); + fprintf (dump_file, " with frequency %i\n", cg_edge->frequency); } /* This is it. Duplicate the callee body. Assume callee is @@ -3269,25 +3983,26 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) function in any way before this point, as this CALL_EXPR may be a self-referential call; if we're calling ourselves, we need to duplicate our body before altering anything. */ - copy_body (id, bb->count, bb->frequency, bb, return_block); + copy_body (id, bb->count, + cg_edge->frequency * REG_BR_PROB_BASE / CGRAPH_FREQ_BASE, + bb, return_block, NULL, NULL); - /* Add local vars in this inlined callee to caller. */ - t_step = id->src_cfun->local_decls; - for (; t_step; t_step = TREE_CHAIN (t_step)) - { - var = TREE_VALUE (t_step); - if (TREE_STATIC (var) && !TREE_ASM_WRITTEN (var)) - cfun->local_decls = tree_cons (NULL_TREE, var, - cfun->local_decls); - else - cfun->local_decls = tree_cons (NULL_TREE, remap_decl (var, id), - cfun->local_decls); - } + /* Reset the escaped solution. */ + if (cfun->gimple_df) + pt_solution_reset (&cfun->gimple_df->escaped); /* Clean up. */ + if (id->debug_map) + { + pointer_map_destroy (id->debug_map); + id->debug_map = dst; + } pointer_map_destroy (id->decl_map); id->decl_map = st; + /* Unlink the calls virtual operands before replacing it. */ + unlink_stmt_vdef (stmt); + /* If the inlined function returns a result that we care about, substitute the GIMPLE_CALL with an assignment of the return variable to the LHS of the call. That is, if STMT was @@ -3298,10 +4013,7 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) stmt = gimple_build_assign (gimple_call_lhs (stmt), use_retvar); gsi_replace (&stmt_gsi, stmt, false); if (gimple_in_ssa_p (cfun)) - { - update_stmt (stmt); - mark_symbols_for_renaming (stmt); - } + mark_symbols_for_renaming (stmt); maybe_clean_or_replace_eh_stmt (old_stmt, stmt); } else @@ -3321,7 +4033,6 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) undefined via a move. */ stmt = gimple_build_assign (gimple_call_lhs (stmt), def); gsi_replace (&stmt_gsi, stmt, true); - update_stmt (stmt); } else { @@ -3336,7 +4047,10 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) } if (purge_dead_abnormal_edges) - gimple_purge_dead_abnormal_call_edges (return_block); + { + gimple_purge_dead_eh_edges (return_block); + gimple_purge_dead_abnormal_call_edges (return_block); + } /* If the value of the new expression is ignored, that's OK. We don't warn about this for CALL_EXPRs, so we shouldn't warn about @@ -3367,9 +4081,7 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) /* Expand call statements reachable from STMT_P. We can only have CALL_EXPRs as the "toplevel" tree code or nested - in a MODIFY_EXPR. See tree-gimple.c:get_call_expr_in(). We can - unfortunately not use that function here because we need a pointer - to the CALL_EXPR, not the tree itself. */ + in a MODIFY_EXPR. */ static bool gimple_expand_calls_inline (basic_block bb, copy_body_data *id) @@ -3406,16 +4118,68 @@ fold_marked_statements (int first, struct pointer_set_t *statements) if (pointer_set_contains (statements, gsi_stmt (gsi))) { gimple old_stmt = gsi_stmt (gsi); + tree old_decl = is_gimple_call (old_stmt) ? gimple_call_fndecl (old_stmt) : 0; - if (fold_stmt (&gsi)) + if (old_decl && DECL_BUILT_IN (old_decl)) + { + /* Folding builtins can create multiple instructions, + we need to look at all of them. */ + gimple_stmt_iterator i2 = gsi; + gsi_prev (&i2); + if (fold_stmt (&gsi)) + { + gimple new_stmt; + /* If a builtin at the end of a bb folded into nothing, + the following loop won't work. */ + if (gsi_end_p (gsi)) + { + cgraph_update_edges_for_call_stmt (old_stmt, + old_decl, NULL); + break; + } + if (gsi_end_p (i2)) + i2 = gsi_start_bb (BASIC_BLOCK (first)); + else + gsi_next (&i2); + while (1) + { + new_stmt = gsi_stmt (i2); + update_stmt (new_stmt); + cgraph_update_edges_for_call_stmt (old_stmt, old_decl, + new_stmt); + + if (new_stmt == gsi_stmt (gsi)) + { + /* It is okay to check only for the very last + of these statements. If it is a throwing + statement nothing will change. If it isn't + this can remove EH edges. If that weren't + correct then because some intermediate stmts + throw, but not the last one. That would mean + we'd have to split the block, which we can't + here and we'd loose anyway. And as builtins + probably never throw, this all + is mood anyway. */ + if (maybe_clean_or_replace_eh_stmt (old_stmt, + new_stmt)) + gimple_purge_dead_eh_edges (BASIC_BLOCK (first)); + break; + } + gsi_next (&i2); + } + } + } + else if (fold_stmt (&gsi)) { /* Re-read the statement from GSI as fold_stmt() may have changed it. */ gimple new_stmt = gsi_stmt (gsi); update_stmt (new_stmt); - if (is_gimple_call (old_stmt)) - cgraph_update_edges_for_call_stmt (old_stmt, new_stmt); + if (is_gimple_call (old_stmt) + || is_gimple_call (new_stmt)) + cgraph_update_edges_for_call_stmt (old_stmt, old_decl, + new_stmt); if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt)) gimple_purge_dead_eh_edges (BASIC_BLOCK (first)); @@ -3445,29 +4209,20 @@ unsigned int optimize_inline_calls (tree fn) { copy_body_data id; - tree prev_fn; basic_block bb; int last = n_basic_blocks; struct gimplify_ctx gctx; - - /* There is no point in performing inlining if errors have already - occurred -- and we might crash if we try to inline invalid - code. */ - if (errorcount || sorrycount) - return 0; + bool inlined_p = false; /* Clear out ID. */ memset (&id, 0, sizeof (id)); - id.src_node = id.dst_node = cgraph_node (fn); + id.src_node = id.dst_node = cgraph_get_node (fn); + gcc_assert (id.dst_node->analyzed); id.dst_fn = fn; /* Or any functions that aren't finished yet. */ - prev_fn = NULL_TREE; if (current_function_decl) - { - id.dst_fn = current_function_decl; - prev_fn = current_function_decl; - } + id.dst_fn = current_function_decl; id.copy_decl = copy_decl_maybe_to_var; id.transform_call_graph_edges = CB_CGE_DUPLICATE; @@ -3492,7 +4247,7 @@ optimize_inline_calls (tree fn) follow it; we'll trudge through them, processing their CALL_EXPRs along the way. */ FOR_EACH_BB (bb) - gimple_expand_calls_inline (bb, &id); + inlined_p |= gimple_expand_calls_inline (bb, &id); pop_gimplify_context (NULL); @@ -3507,21 +4262,24 @@ optimize_inline_calls (tree fn) gcc_assert (e->inline_failed); } #endif - - /* Fold the statements before compacting/renumbering the basic blocks. */ + + /* Fold queued statements. */ fold_marked_statements (last, id.statements_to_fold); pointer_set_destroy (id.statements_to_fold); - - /* Renumber the (code) basic_blocks consecutively. */ - compact_blocks (); + + gcc_assert (!id.debug_stmts); + + /* If we didn't inline into the function there is nothing to do. */ + if (!inlined_p) + return 0; + /* Renumber the lexical scoping (non-code) blocks consecutively. */ number_blocks (fn); - /* We are not going to maintain the cgraph edges up to date. - Kill it so it won't confuse us. */ - cgraph_node_remove_callees (id.dst_node); - - fold_cond_expr_cond (); + delete_unreachable_blocks_update_callgraph (&id); +#ifdef ENABLE_CHECKING + verify_cgraph_node (id.dst_node); +#endif /* It would be nice to check SSA/CFG/statement consistency here, but it is not possible yet - the IPA passes might make various functions to not @@ -3530,6 +4288,7 @@ optimize_inline_calls (tree fn) return (TODO_update_ssa | TODO_cleanup_cfg | (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0) + | (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0) | (profile_status != PROFILE_ABSENT ? TODO_rebuild_frequencies : 0)); } @@ -3552,7 +4311,8 @@ copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED) here. */ tree chain = NULL_TREE, new_tree; - chain = TREE_CHAIN (*tp); + if (CODE_CONTAINS_STRUCT (code, TS_COMMON)) + chain = TREE_CHAIN (*tp); /* Copy the node. */ new_tree = copy_node (*tp); @@ -3591,14 +4351,16 @@ copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED) CONSTRUCTOR_ELTS (*tp)); *tp = new_tree; } + else if (code == STATEMENT_LIST) + /* We used to just abort on STATEMENT_LIST, but we can run into them + with statement-expressions (c++/40975). */ + copy_statement_list (tp); else if (TREE_CODE_CLASS (code) == tcc_type) *walk_subtrees = 0; else if (TREE_CODE_CLASS (code) == tcc_declaration) *walk_subtrees = 0; else if (TREE_CODE_CLASS (code) == tcc_constant) *walk_subtrees = 0; - else - gcc_assert (code != STATEMENT_LIST); return NULL_TREE; } @@ -3715,7 +4477,8 @@ unsave_r (tree *tp, int *walk_subtrees, void *data) gcc_unreachable (); else if (TREE_CODE (*tp) == BIND_EXPR) copy_bind_expr (tp, walk_subtrees, id); - else if (TREE_CODE (*tp) == SAVE_EXPR) + else if (TREE_CODE (*tp) == SAVE_EXPR + || TREE_CODE (*tp) == TARGET_EXPR) remap_save_expr (tp, st, walk_subtrees); else { @@ -3746,6 +4509,7 @@ unsave_expr_now (tree expr) id.src_fn = current_function_decl; id.dst_fn = current_function_decl; id.decl_map = pointer_map_create (); + id.debug_map = NULL; id.copy_decl = copy_decl_no_change; id.transform_call_graph_edges = CB_CGE_DUPLICATE; @@ -3761,6 +4525,8 @@ unsave_expr_now (tree expr) /* Clean up. */ pointer_map_destroy (id.decl_map); + if (id.debug_map) + pointer_map_destroy (id.debug_map); return expr; } @@ -3864,7 +4630,7 @@ replace_locals_stmt (gimple_stmt_iterator *gsip, /* This will remap a lot of the same decls again, but this should be harmless. */ if (gimple_bind_vars (stmt)) - gimple_bind_set_vars (stmt, remap_decls (gimple_bind_vars (stmt), id)); + gimple_bind_set_vars (stmt, remap_decls (gimple_bind_vars (stmt), NULL, id)); } /* Keep iterating. */ @@ -3892,6 +4658,7 @@ copy_gimple_seq_and_replace_locals (gimple_seq seq) id.src_fn = current_function_decl; id.dst_fn = current_function_decl; id.decl_map = pointer_map_create (); + id.debug_map = NULL; id.copy_decl = copy_decl_no_change; id.transform_call_graph_edges = CB_CGE_DUPLICATE; @@ -3916,6 +4683,8 @@ copy_gimple_seq_and_replace_locals (gimple_seq seq) /* Clean up. */ pointer_map_destroy (id.decl_map); + if (id.debug_map) + pointer_map_destroy (id.debug_map); return copy; } @@ -3932,7 +4701,7 @@ debug_find_tree_1 (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, void *data) return NULL; } -bool +DEBUG_FUNCTION bool debug_find_tree (tree top, tree search) { return walk_tree_without_duplicates (&top, debug_find_tree_1, search) != 0; @@ -3946,11 +4715,11 @@ static void declare_inline_vars (tree block, tree vars) { tree t; - for (t = vars; t; t = TREE_CHAIN (t)) + for (t = vars; t; t = DECL_CHAIN (t)) { DECL_SEEN_IN_BIND_EXPR_P (t) = 1; gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t)); - cfun->local_decls = tree_cons (NULL_TREE, t, cfun->local_decls); + add_local_decl (cfun, t); } if (block) @@ -3970,14 +4739,14 @@ copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy) DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl); /* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what - declaration inspired this copy. */ + declaration inspired this copy. */ DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl); /* The new variable/label has no RTL, yet. */ if (CODE_CONTAINS_STRUCT (TREE_CODE (copy), TS_DECL_WRTL) && !TREE_STATIC (copy) && !DECL_EXTERNAL (copy)) - SET_DECL_RTL (copy, NULL_RTX); - + SET_DECL_RTL (copy, 0); + /* These args would always appear unused, if not for this. */ TREE_USED (copy) = 1; @@ -3998,6 +4767,14 @@ copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy) new function. */ DECL_CONTEXT (copy) = id->dst_fn; + if (TREE_CODE (decl) == VAR_DECL + /* C++ clones functions during parsing, before + referenced_vars. */ + && gimple_referenced_vars (DECL_STRUCT_FUNCTION (id->src_fn)) + && referenced_var_lookup (DECL_STRUCT_FUNCTION (id->src_fn), + DECL_UID (decl))) + add_referenced_var (copy); + return copy; } @@ -4011,12 +4788,14 @@ copy_decl_to_var (tree decl, copy_body_data *id) type = TREE_TYPE (decl); - copy = build_decl (VAR_DECL, DECL_NAME (decl), type); + copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn), + VAR_DECL, DECL_NAME (decl), type); + if (DECL_PT_UID_SET_P (decl)) + SET_DECL_PT_UID (copy, DECL_PT_UID (decl)); TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl); TREE_READONLY (copy) = TREE_READONLY (decl); TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl); DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl); - DECL_NO_TBAA_P (copy) = DECL_NO_TBAA_P (decl); return copy_decl_for_dup_finish (id, decl, copy); } @@ -4036,14 +4815,16 @@ copy_result_decl_to_var (tree decl, copy_body_data *id) if (DECL_BY_REFERENCE (decl)) type = TREE_TYPE (type); - copy = build_decl (VAR_DECL, DECL_NAME (decl), type); + copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn), + VAR_DECL, DECL_NAME (decl), type); + if (DECL_PT_UID_SET_P (decl)) + SET_DECL_PT_UID (copy, DECL_PT_UID (decl)); TREE_READONLY (copy) = TREE_READONLY (decl); TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl); if (!DECL_BY_REFERENCE (decl)) { TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl); DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl); - DECL_NO_TBAA_P (copy) = DECL_NO_TBAA_P (decl); } return copy_decl_for_dup_finish (id, decl, copy); @@ -4082,19 +4863,38 @@ copy_decl_maybe_to_var (tree decl, copy_body_data *id) /* Return a copy of the function's argument tree. */ static tree -copy_arguments_for_versioning (tree orig_parm, copy_body_data * id) +copy_arguments_for_versioning (tree orig_parm, copy_body_data * id, + bitmap args_to_skip, tree *vars) { - tree *arg_copy, *parg; + tree arg, *parg; + tree new_parm = NULL; + int i = 0; - arg_copy = &orig_parm; - for (parg = arg_copy; *parg; parg = &TREE_CHAIN (*parg)) - { - tree new_tree = remap_decl (*parg, id); - lang_hooks.dup_lang_specific_decl (new_tree); - TREE_CHAIN (new_tree) = TREE_CHAIN (*parg); - *parg = new_tree; - } - return orig_parm; + parg = &new_parm; + + for (arg = orig_parm; arg; arg = DECL_CHAIN (arg), i++) + if (!args_to_skip || !bitmap_bit_p (args_to_skip, i)) + { + tree new_tree = remap_decl (arg, id); + if (TREE_CODE (new_tree) != PARM_DECL) + new_tree = id->copy_decl (arg, id); + lang_hooks.dup_lang_specific_decl (new_tree); + *parg = new_tree; + parg = &DECL_CHAIN (new_tree); + } + else if (!pointer_map_contains (id->decl_map, arg)) + { + /* Make an equivalent VAR_DECL. If the argument was used + as temporary variable later in function, the uses will be + replaced by local variable. */ + tree var = copy_decl_to_var (arg, id); + add_referenced_var (var); + insert_decl_map (id, arg, var); + /* Declare this new variable. */ + DECL_CHAIN (var) = *vars; + *vars = var; + } + return new_parm; } /* Return a copy of the function's static chain. */ @@ -4104,11 +4904,11 @@ copy_static_chain (tree static_chain, copy_body_data * id) tree *chain_copy, *pvar; chain_copy = &static_chain; - for (pvar = chain_copy; *pvar; pvar = &TREE_CHAIN (*pvar)) + for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar)) { tree new_tree = remap_decl (*pvar, id); lang_hooks.dup_lang_specific_decl (new_tree); - TREE_CHAIN (new_tree) = TREE_CHAIN (*pvar); + DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar); *pvar = new_tree; } return static_chain; @@ -4116,30 +4916,139 @@ copy_static_chain (tree static_chain, copy_body_data * id) /* Return true if the function is allowed to be versioned. This is a guard for the versioning functionality. */ + bool tree_versionable_function_p (tree fndecl) { - if (fndecl == NULL_TREE) - return false; - /* ??? There are cases where a function is - uninlinable but can be versioned. */ - if (!tree_inlinable_function_p (fndecl)) - return false; - - return true; + return (!lookup_attribute ("noclone", DECL_ATTRIBUTES (fndecl)) + && copy_forbidden (DECL_STRUCT_FUNCTION (fndecl), fndecl) == NULL); +} + +/* Delete all unreachable basic blocks and update callgraph. + Doing so is somewhat nontrivial because we need to update all clones and + remove inline function that become unreachable. */ + +static bool +delete_unreachable_blocks_update_callgraph (copy_body_data *id) +{ + bool changed = false; + basic_block b, next_bb; + + find_unreachable_blocks (); + + /* Delete all unreachable basic blocks. */ + + for (b = ENTRY_BLOCK_PTR->next_bb; b != EXIT_BLOCK_PTR; b = next_bb) + { + next_bb = b->next_bb; + + if (!(b->flags & BB_REACHABLE)) + { + gimple_stmt_iterator bsi; + + for (bsi = gsi_start_bb (b); !gsi_end_p (bsi); gsi_next (&bsi)) + if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL) + { + struct cgraph_edge *e; + struct cgraph_node *node; + + if ((e = cgraph_edge (id->dst_node, gsi_stmt (bsi))) != NULL) + { + if (!e->inline_failed) + cgraph_remove_node_and_inline_clones (e->callee); + else + cgraph_remove_edge (e); + } + if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES + && id->dst_node->clones) + for (node = id->dst_node->clones; node != id->dst_node;) + { + if ((e = cgraph_edge (node, gsi_stmt (bsi))) != NULL) + { + if (!e->inline_failed) + cgraph_remove_node_and_inline_clones (e->callee); + else + cgraph_remove_edge (e); + } + + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != id->dst_node && !node->next_sibling_clone) + node = node->clone_of; + if (node != id->dst_node) + node = node->next_sibling_clone; + } + } + } + delete_basic_block (b); + changed = true; + } + } + + return changed; +} + +/* Update clone info after duplication. */ + +static void +update_clone_info (copy_body_data * id) +{ + struct cgraph_node *node; + if (!id->dst_node->clones) + return; + for (node = id->dst_node->clones; node != id->dst_node;) + { + /* First update replace maps to match the new body. */ + if (node->clone.tree_map) + { + unsigned int i; + for (i = 0; i < VEC_length (ipa_replace_map_p, node->clone.tree_map); i++) + { + struct ipa_replace_map *replace_info; + replace_info = VEC_index (ipa_replace_map_p, node->clone.tree_map, i); + walk_tree (&replace_info->old_tree, copy_tree_body_r, id, NULL); + walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL); + } + } + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != id->dst_node && !node->next_sibling_clone) + node = node->clone_of; + if (node != id->dst_node) + node = node->next_sibling_clone; + } + } } /* Create a copy of a function's tree. OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes of the original function and the new copied function - respectively. In case we want to replace a DECL - tree with another tree while duplicating the function's - body, TREE_MAP represents the mapping between these + respectively. In case we want to replace a DECL + tree with another tree while duplicating the function's + body, TREE_MAP represents the mapping between these trees. If UPDATE_CLONES is set, the call_stmt fields - of edges of clones of the function will be updated. */ + of edges of clones of the function will be updated. + + If non-NULL ARGS_TO_SKIP determine function parameters to remove + from new version. + If SKIP_RETURN is true, the new version will return void. + If non-NULL BLOCK_TO_COPY determine what basic blocks to copy. + If non_NULL NEW_ENTRY determine new entry BB of the clone. +*/ void -tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, - bool update_clones) +tree_function_versioning (tree old_decl, tree new_decl, + VEC(ipa_replace_map_p,gc)* tree_map, + bool update_clones, bitmap args_to_skip, + bool skip_return, bitmap blocks_to_copy, + basic_block new_entry) { struct cgraph_node *old_version_node; struct cgraph_node *new_version_node; @@ -4147,10 +5056,9 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, tree p; unsigned i; struct ipa_replace_map *replace_info; - basic_block old_entry_block; + basic_block old_entry_block, bb; VEC (gimple, heap) *init_stmts = VEC_alloc (gimple, heap, 10); - tree t_step; tree old_current_function_decl = current_function_decl; tree vars = NULL_TREE; @@ -4158,31 +5066,62 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, && TREE_CODE (new_decl) == FUNCTION_DECL); DECL_POSSIBLY_INLINED (old_decl) = 1; - old_version_node = cgraph_node (old_decl); - new_version_node = cgraph_node (new_decl); + old_version_node = cgraph_get_node (old_decl); + gcc_checking_assert (old_version_node); + new_version_node = cgraph_get_node (new_decl); + gcc_checking_assert (new_version_node); + + /* Copy over debug args. */ + if (DECL_HAS_DEBUG_ARGS_P (old_decl)) + { + VEC(tree, gc) **new_debug_args, **old_debug_args; + gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL); + DECL_HAS_DEBUG_ARGS_P (new_decl) = 0; + old_debug_args = decl_debug_args_lookup (old_decl); + if (old_debug_args) + { + new_debug_args = decl_debug_args_insert (new_decl); + *new_debug_args = VEC_copy (tree, gc, *old_debug_args); + } + } + + /* Output the inlining info for this abstract function, since it has been + inlined. If we don't do this now, we can lose the information about the + variables in the function when the blocks get blown away as soon as we + remove the cgraph node. */ + (*debug_hooks->outlining_inline_function) (old_decl); DECL_ARTIFICIAL (new_decl) = 1; DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl); + DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl); /* Prepare the data structures for the tree copy. */ memset (&id, 0, sizeof (id)); /* Generate a new name for the new version. */ - if (!update_clones) - { - DECL_NAME (new_decl) = create_tmp_var_name (NULL); - SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); - SET_DECL_RTL (new_decl, NULL_RTX); - id.statements_to_fold = pointer_set_create (); - } - + id.statements_to_fold = pointer_set_create (); + id.decl_map = pointer_map_create (); + id.debug_map = NULL; id.src_fn = old_decl; id.dst_fn = new_decl; id.src_node = old_version_node; id.dst_node = new_version_node; id.src_cfun = DECL_STRUCT_FUNCTION (old_decl); - + if (id.src_node->ipa_transforms_to_apply) + { + VEC(ipa_opt_pass,heap) * old_transforms_to_apply = id.dst_node->ipa_transforms_to_apply; + unsigned int i; + + id.dst_node->ipa_transforms_to_apply = VEC_copy (ipa_opt_pass, heap, + id.src_node->ipa_transforms_to_apply); + for (i = 0; i < VEC_length (ipa_opt_pass, old_transforms_to_apply); i++) + VEC_safe_push (ipa_opt_pass, heap, id.dst_node->ipa_transforms_to_apply, + VEC_index (ipa_opt_pass, + old_transforms_to_apply, + i)); + } + id.copy_decl = copy_decl_no_change; id.transform_call_graph_edges = update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE; @@ -4194,42 +5133,42 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (old_decl)); initialize_cfun (new_decl, old_decl, - old_entry_block->count, - old_entry_block->frequency); + old_entry_block->count); + DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta + = id.src_cfun->gimple_df->ipa_pta; push_cfun (DECL_STRUCT_FUNCTION (new_decl)); - + /* Copy the function's static chain. */ p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl; if (p) DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl = copy_static_chain (DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl, &id); - /* Copy the function's arguments. */ - if (DECL_ARGUMENTS (old_decl) != NULL_TREE) - DECL_ARGUMENTS (new_decl) = - copy_arguments_for_versioning (DECL_ARGUMENTS (old_decl), &id); - - DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id); - - /* Renumber the lexical scoping (non-code) blocks consecutively. */ - number_blocks (id.dst_fn); - + /* If there's a tree_map, prepare for substitution. */ if (tree_map) - for (i = 0; i < VARRAY_ACTIVE_SIZE (tree_map); i++) + for (i = 0; i < VEC_length (ipa_replace_map_p, tree_map); i++) { gimple init; - replace_info - = (struct ipa_replace_map *) VARRAY_GENERIC_PTR (tree_map, i); + replace_info = VEC_index (ipa_replace_map_p, tree_map, i); if (replace_info->replace_p) { tree op = replace_info->new_tree; + if (!replace_info->old_tree) + { + int i = replace_info->parm_num; + tree parm; + for (parm = DECL_ARGUMENTS (old_decl); i; parm = DECL_CHAIN (parm)) + i --; + replace_info->old_tree = parm; + } + STRIP_NOPS (op); if (TREE_CODE (op) == VIEW_CONVERT_EXPR) op = TREE_OPERAND (op, 0); - + if (TREE_CODE (op) == ADDR_EXPR) { op = TREE_OPERAND (op, 0); @@ -4247,66 +5186,117 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, VEC_safe_push (gimple, heap, init_stmts, init); } } - + /* Copy the function's arguments. */ + if (DECL_ARGUMENTS (old_decl) != NULL_TREE) + DECL_ARGUMENTS (new_decl) = + copy_arguments_for_versioning (DECL_ARGUMENTS (old_decl), &id, + args_to_skip, &vars); + + DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id); + BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl; + declare_inline_vars (DECL_INITIAL (new_decl), vars); - if (DECL_STRUCT_FUNCTION (old_decl)->local_decls != NULL_TREE) + + if (!VEC_empty (tree, DECL_STRUCT_FUNCTION (old_decl)->local_decls)) /* Add local vars. */ - for (t_step = DECL_STRUCT_FUNCTION (old_decl)->local_decls; - t_step; t_step = TREE_CHAIN (t_step)) - { - tree var = TREE_VALUE (t_step); - if (TREE_STATIC (var) && !TREE_ASM_WRITTEN (var)) - cfun->local_decls = tree_cons (NULL_TREE, var, cfun->local_decls); - else - cfun->local_decls = - tree_cons (NULL_TREE, remap_decl (var, &id), - cfun->local_decls); - } - - /* Copy the Function's body. */ - copy_body (&id, old_entry_block->count, old_entry_block->frequency, ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR); - - if (DECL_RESULT (old_decl) != NULL_TREE) + add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, &id, false); + + if (DECL_RESULT (old_decl) == NULL_TREE) + ; + else if (skip_return && !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl)))) + { + DECL_RESULT (new_decl) + = build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)), + RESULT_DECL, NULL_TREE, void_type_node); + DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl; + cfun->returns_struct = 0; + cfun->returns_pcc_struct = 0; + } + else { - tree *res_decl = &DECL_RESULT (old_decl); - DECL_RESULT (new_decl) = remap_decl (*res_decl, &id); + tree old_name; + DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), &id); lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl)); + if (gimple_in_ssa_p (id.src_cfun) + && DECL_BY_REFERENCE (DECL_RESULT (old_decl)) + && (old_name + = gimple_default_def (id.src_cfun, DECL_RESULT (old_decl)))) + { + tree new_name = make_ssa_name (DECL_RESULT (new_decl), NULL); + insert_decl_map (&id, old_name, new_name); + SSA_NAME_DEF_STMT (new_name) = gimple_build_nop (); + set_default_def (DECL_RESULT (new_decl), new_name); + } } - + + /* Copy the Function's body. */ + copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE, + ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR, blocks_to_copy, new_entry); + /* Renumber the lexical scoping (non-code) blocks consecutively. */ number_blocks (new_decl); - if (VEC_length (gimple, init_stmts)) + /* We want to create the BB unconditionally, so that the addition of + debug stmts doesn't affect BB count, which may in the end cause + codegen differences. */ + bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR)); + while (VEC_length (gimple, init_stmts)) + insert_init_stmt (&id, bb, VEC_pop (gimple, init_stmts)); + update_clone_info (&id); + + /* Remap the nonlocal_goto_save_area, if any. */ + if (cfun->nonlocal_goto_save_area) { - basic_block bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR)); - while (VEC_length (gimple, init_stmts)) - insert_init_stmt (bb, VEC_pop (gimple, init_stmts)); + struct walk_stmt_info wi; + + memset (&wi, 0, sizeof (wi)); + wi.info = &id; + walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL); } /* Clean up. */ pointer_map_destroy (id.decl_map); - if (!update_clones) - { - fold_marked_statements (0, id.statements_to_fold); - pointer_set_destroy (id.statements_to_fold); - fold_cond_expr_cond (); - } - if (gimple_in_ssa_p (cfun)) + if (id.debug_map) + pointer_map_destroy (id.debug_map); + free_dominance_info (CDI_DOMINATORS); + free_dominance_info (CDI_POST_DOMINATORS); + + fold_marked_statements (0, id.statements_to_fold); + pointer_set_destroy (id.statements_to_fold); + fold_cond_expr_cond (); + delete_unreachable_blocks_update_callgraph (&id); + if (id.dst_node->analyzed) + cgraph_rebuild_references (); + update_ssa (TODO_update_ssa); + + /* After partial cloning we need to rescale frequencies, so they are + within proper range in the cloned function. */ + if (new_entry) { - free_dominance_info (CDI_DOMINATORS); - free_dominance_info (CDI_POST_DOMINATORS); - if (!update_clones) - delete_unreachable_blocks (); - update_ssa (TODO_update_ssa); - if (!update_clones) + struct cgraph_edge *e; + rebuild_frequencies (); + + new_version_node->count = ENTRY_BLOCK_PTR->count; + for (e = new_version_node->callees; e; e = e->next_callee) + { + basic_block bb = gimple_bb (e->call_stmt); + e->frequency = compute_call_stmt_bb_frequency (current_function_decl, + bb); + e->count = bb->count; + } + for (e = new_version_node->indirect_calls; e; e = e->next_callee) { - fold_cond_expr_cond (); - if (need_ssa_update_p ()) - update_ssa (TODO_update_ssa); + basic_block bb = gimple_bb (e->call_stmt); + e->frequency = compute_call_stmt_bb_frequency (current_function_decl, + bb); + e->count = bb->count; } } + free_dominance_info (CDI_DOMINATORS); free_dominance_info (CDI_POST_DOMINATORS); + + gcc_assert (!id.debug_stmts); VEC_free (gimple, heap, init_stmts); pop_cfun (); current_function_decl = old_current_function_decl; @@ -4315,6 +5305,60 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, return; } +/* EXP is CALL_EXPR present in a GENERIC expression tree. Try to integrate + the callee and return the inlined body on success. */ + +tree +maybe_inline_call_in_expr (tree exp) +{ + tree fn = get_callee_fndecl (exp); + + /* We can only try to inline "const" functions. */ + if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn)) + { + struct pointer_map_t *decl_map = pointer_map_create (); + call_expr_arg_iterator iter; + copy_body_data id; + tree param, arg, t; + + /* Remap the parameters. */ + for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, &iter); + param; + param = DECL_CHAIN (param), arg = next_call_expr_arg (&iter)) + *pointer_map_insert (decl_map, param) = arg; + + memset (&id, 0, sizeof (id)); + id.src_fn = fn; + id.dst_fn = current_function_decl; + id.src_cfun = DECL_STRUCT_FUNCTION (fn); + id.decl_map = decl_map; + + id.copy_decl = copy_decl_no_change; + id.transform_call_graph_edges = CB_CGE_DUPLICATE; + id.transform_new_cfg = false; + id.transform_return_to_modify = true; + id.transform_lang_insert_block = NULL; + + /* Make sure not to unshare trees behind the front-end's back + since front-end specific mechanisms may rely on sharing. */ + id.regimplify = false; + id.do_not_unshare = true; + + /* We're not inside any EH region. */ + id.eh_lp_nr = 0; + + t = copy_tree_body (&id); + pointer_map_destroy (decl_map); + + /* We can only return something suitable for use in a GENERIC + expression tree. */ + if (TREE_CODE (t) == MODIFY_EXPR) + return TREE_OPERAND (t, 1); + } + + return NULL_TREE; +} + /* Duplicate a type, fields and all. */ tree @@ -4327,44 +5371,16 @@ build_duplicate_type (tree type) id.dst_fn = current_function_decl; id.src_cfun = cfun; id.decl_map = pointer_map_create (); + id.debug_map = NULL; id.copy_decl = copy_decl_no_change; type = remap_type_1 (type, &id); pointer_map_destroy (id.decl_map); + if (id.debug_map) + pointer_map_destroy (id.debug_map); TYPE_CANONICAL (type) = type; return type; } - -/* Return whether it is safe to inline a function because it used different - target specific options or different optimization options. */ -bool -tree_can_inline_p (tree caller, tree callee) -{ - /* Don't inline a function with a higher optimization level than the - caller, or with different space constraints (hot/cold functions). */ - tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller); - tree callee_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee); - - if (caller_tree != callee_tree) - { - struct cl_optimization *caller_opt - = TREE_OPTIMIZATION ((caller_tree) - ? caller_tree - : optimization_default_node); - - struct cl_optimization *callee_opt - = TREE_OPTIMIZATION ((callee_tree) - ? callee_tree - : optimization_default_node); - - if ((caller_opt->optimize > callee_opt->optimize) - || (caller_opt->optimize_size != callee_opt->optimize_size)) - return false; - } - - /* Allow the backend to decide if inlining is ok. */ - return targetm.target_option.can_inline_p (caller, callee); -}