/* Induction variable optimizations.
- Copyright (C) 2003 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004 Free Software Foundation, Inc.
This file is part of GCC.
struct iv
{
tree base; /* Initial value of the iv. */
+ tree base_object; /* A memory object to that the induction variable points. */
tree step; /* Step of the iv (constant only). */
tree ssa_name; /* The ssa name with the value. */
bool biv_p; /* Is it a biv? */
struct iv *iv; /* The induction variable it is based on. */
tree stmt; /* Statement in that it occurs. */
tree *op_p; /* The place where it occurs. */
- bitmap related_cands; /* The set of "related" iv candidates. */
+ bitmap related_cands; /* The set of "related" iv candidates, plus the common
+ important ones. */
unsigned n_map_members; /* Number of candidates in the cost_map list. */
struct cost_pair *cost_map;
/* The candidates. */
varray_type iv_candidates;
+ /* A bitmap of important candidates. */
+ bitmap important_candidates;
+
/* Whether to consider just related and important candidates when replacing a
use. */
bool consider_all_candidates;
};
+/* An assignment of iv candidates to uses. */
+
+struct iv_ca
+{
+ /* The number of uses covered by the assignment. */
+ unsigned upto;
+
+ /* Number of uses that cannot be expressed by the candidates in the set. */
+ unsigned bad_uses;
+
+ /* Candidate assigned to a use, together with the related costs. */
+ struct cost_pair **cand_for_use;
+
+ /* Number of times each candidate is used. */
+ unsigned *n_cand_uses;
+
+ /* The candidates used. */
+ bitmap cands;
+
+ /* Total number of registers needed. */
+ unsigned n_regs;
+
+ /* Total cost of expressing uses. */
+ unsigned cand_use_cost;
+
+ /* Total cost of candidates. */
+ unsigned cand_cost;
+
+ /* Number of times each invariant is used. */
+ unsigned *n_invariant_uses;
+
+ /* Total cost of the assignment. */
+ unsigned cost;
+};
+
+/* Difference of two iv candidate assignments. */
+
+struct iv_ca_delta
+{
+ /* Changed use. */
+ struct iv_use *use;
+
+ /* An old assignment (for rollback purposes). */
+ struct cost_pair *old_cp;
+
+ /* A new assignment. */
+ struct cost_pair *new_cp;
+
+ /* Next change in the list. */
+ struct iv_ca_delta *next_change;
+};
+
/* Bound on number of candidates below that all candidates are considered. */
#define CONSIDER_ALL_CANDIDATES_BOUND \
void
dump_iv (FILE *file, struct iv *iv)
{
- fprintf (file, "ssa name ");
- print_generic_expr (file, iv->ssa_name, TDF_SLIM);
+ if (iv->ssa_name)
+ {
+ fprintf (file, "ssa name ");
+ print_generic_expr (file, iv->ssa_name, TDF_SLIM);
+ fprintf (file, "\n");
+ }
+
+ fprintf (file, " type ");
+ print_generic_expr (file, TREE_TYPE (iv->base), TDF_SLIM);
fprintf (file, "\n");
if (iv->step)
fprintf (file, "\n");
}
+ if (iv->base_object)
+ {
+ fprintf (file, " base object ");
+ print_generic_expr (file, iv->base_object, TDF_SLIM);
+ fprintf (file, "\n");
+ }
+
if (iv->biv_p)
fprintf (file, " is a biv\n");
}
void
dump_use (FILE *file, struct iv_use *use)
{
- struct iv *iv = use->iv;
-
fprintf (file, "use %d\n", use->id);
switch (use->type)
break;
default:
- abort ();
+ gcc_unreachable ();
}
- fprintf (file, " in statement ");
- print_generic_expr (file, use->stmt, TDF_SLIM);
- fprintf (file, "\n");
-
- fprintf (file, " at position ");
- if (use->op_p)
- print_generic_expr (file, *use->op_p, TDF_SLIM);
- fprintf (file, "\n");
+ fprintf (file, " in statement ");
+ print_generic_expr (file, use->stmt, TDF_SLIM);
+ fprintf (file, "\n");
- if (iv->step)
- {
- fprintf (file, " base ");
- print_generic_expr (file, iv->base, TDF_SLIM);
- fprintf (file, "\n");
+ fprintf (file, " at position ");
+ if (use->op_p)
+ print_generic_expr (file, *use->op_p, TDF_SLIM);
+ fprintf (file, "\n");
- fprintf (file, " step ");
- print_generic_expr (file, iv->step, TDF_SLIM);
- fprintf (file, "\n");
- }
- else
- {
- fprintf (file, " invariant ");
- print_generic_expr (file, iv->base, TDF_SLIM);
- fprintf (file, "\n");
- }
+ dump_iv (file, use->iv);
- fprintf (file, " related candidates ");
- dump_bitmap (file, use->related_cands);
+ fprintf (file, " related candidates ");
+ dump_bitmap (file, use->related_cands);
}
/* Dumps information about the uses to FILE. */
break;
}
- if (iv->step)
- {
- fprintf (file, " base ");
- print_generic_expr (file, iv->base, TDF_SLIM);
- fprintf (file, "\n");
-
- fprintf (file, " step ");
- print_generic_expr (file, iv->step, TDF_SLIM);
- fprintf (file, "\n");
- }
- else
- {
- fprintf (file, " invariant ");
- print_generic_expr (file, iv->base, TDF_SLIM);
- fprintf (file, "\n");
- }
+ dump_iv (file, iv);
}
/* Returns the info for ssa version VER. */
val = (a * inv) & mask;
- if (((val * b) & mask) != a)
- abort ();
+ gcc_assert (((val * b) & mask) == a);
if ((val >> (bits - 1)) & 1)
val |= ~mask;
{
basic_block bb = ip_normal_pos (loop), sbb = bb_for_stmt (stmt);
- if (!bb)
- abort ();
+ gcc_assert (bb);
if (sbb == loop->latch)
return true;
return stmt_after_ip_original_pos (cand, stmt);
default:
- abort ();
+ gcc_unreachable ();
}
}
data->version_info = xcalloc (data->version_info_size,
sizeof (struct version_info));
data->relevant = BITMAP_XMALLOC ();
+ data->important_candidates = BITMAP_XMALLOC ();
data->max_inv_id = 0;
for (i = 1; i < loops->num; i++)
VARRAY_GENERIC_PTR_NOGC_INIT (decl_rtl_to_reset, 20, "decl_rtl_to_reset");
}
+/* Returns a memory object to that EXPR points. In case we are able to
+ determine that it does not point to any such object, NULL is returned. */
+
+static tree
+determine_base_object (tree expr)
+{
+ enum tree_code code = TREE_CODE (expr);
+ tree base, obj, op0, op1;
+
+ if (!POINTER_TYPE_P (TREE_TYPE (expr)))
+ return NULL_TREE;
+
+ switch (code)
+ {
+ case INTEGER_CST:
+ return NULL_TREE;
+
+ case ADDR_EXPR:
+ obj = TREE_OPERAND (expr, 0);
+ base = get_base_address (obj);
+
+ if (!base)
+ return fold_convert (ptr_type_node, expr);
+
+ if (TREE_CODE (base) == INDIRECT_REF)
+ return fold_convert (ptr_type_node, TREE_OPERAND (base, 0));
+
+ return fold (build1 (ADDR_EXPR, ptr_type_node, base));
+
+ case PLUS_EXPR:
+ case MINUS_EXPR:
+ op0 = determine_base_object (TREE_OPERAND (expr, 0));
+ op1 = determine_base_object (TREE_OPERAND (expr, 1));
+
+ if (!op1)
+ return op0;
+
+ if (!op0)
+ return (code == PLUS_EXPR
+ ? op1
+ : fold (build1 (NEGATE_EXPR, ptr_type_node, op1)));
+
+ return fold (build (code, ptr_type_node, op0, op1));
+
+ default:
+ return fold_convert (ptr_type_node, expr);
+ }
+}
+
/* Allocates an induction variable with given initial value BASE and step STEP
for loop LOOP. */
step = NULL_TREE;
iv->base = base;
+ iv->base_object = determine_base_object (base);
iv->step = step;
iv->biv_p = false;
iv->have_use_for = false;
{
struct version_info *info = name_info (data, iv);
- if (info->iv)
- abort ();
+ gcc_assert (!info->iv);
bitmap_set_bit (data->relevant, SSA_NAME_VERSION (iv));
info->iv = alloc_iv (base, step);
return NULL_TREE;
if (!step)
- return fold_convert (type, integer_zero_node);
+ return build_int_cst (type, 0);
return step;
}
-/* Returns false if INDEX is a ssa name that occurs in an
+/* Returns true if EXP is a ssa name that occurs in an abnormal phi node. */
+
+static bool
+abnormal_ssa_name_p (tree exp)
+{
+ if (!exp)
+ return false;
+
+ if (TREE_CODE (exp) != SSA_NAME)
+ return false;
+
+ return SSA_NAME_OCCURS_IN_ABNORMAL_PHI (exp) != 0;
+}
+
+/* Returns false if BASE or INDEX contains a ssa name that occurs in an
abnormal phi node. Callback for for_each_index. */
static bool
-idx_contains_abnormal_ssa_name_p (tree base ATTRIBUTE_UNUSED, tree *index,
+idx_contains_abnormal_ssa_name_p (tree base, tree *index,
void *data ATTRIBUTE_UNUSED)
{
- if (TREE_CODE (*index) != SSA_NAME)
- return true;
+ if (TREE_CODE (base) == ARRAY_REF)
+ {
+ if (abnormal_ssa_name_p (TREE_OPERAND (base, 2)))
+ return false;
+ if (abnormal_ssa_name_p (TREE_OPERAND (base, 3)))
+ return false;
+ }
- return SSA_NAME_OCCURS_IN_ABNORMAL_PHI (*index) == 0;
+ return !abnormal_ssa_name_p (*index);
}
/* Returns true if EXPR contains a ssa name that occurs in an
contains_abnormal_ssa_name_p (tree expr)
{
enum tree_code code = TREE_CODE (expr);
- char class = TREE_CODE_CLASS (code);
+ enum tree_code_class class = TREE_CODE_CLASS (code);
if (code == SSA_NAME)
return SSA_NAME_OCCURS_IN_ABNORMAL_PHI (expr) != 0;
switch (class)
{
- case '2':
+ case tcc_binary:
+ case tcc_comparison:
if (contains_abnormal_ssa_name_p (TREE_OPERAND (expr, 1)))
return true;
/* Fallthru. */
- case '1':
+ case tcc_unary:
if (contains_abnormal_ssa_name_p (TREE_OPERAND (expr, 0)))
return true;
break;
default:
- abort ();
+ gcc_unreachable ();
}
return false;
bool found = false;
struct loop *loop = data->current_loop;
- for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (loop->header); phi; phi = PHI_CHAIN (phi))
{
if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
continue;
struct loop *loop = data->current_loop;
basic_block incr_bb;
- for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (loop->header); phi; phi = PHI_CHAIN (phi))
{
iv = get_iv (data, PHI_RESULT (phi));
if (!iv)
{
unsigned i;
struct loop *loop = data->current_loop;
+ bitmap_iterator bi;
if (!find_bivs (data))
return false;
fprintf (dump_file, "Induction variables:\n\n");
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
{
if (ver_info (data, i)->iv)
dump_iv (dump_file, ver_info (data, i)->iv);
- });
+ }
}
return true;
use->op_p = use_p;
use->related_cands = BITMAP_XMALLOC ();
+ /* To avoid showing ssa name in the dumps, if it was not reset by the
+ caller. */
+ iv->ssa_name = NULL_TREE;
+
if (dump_file && (dump_flags & TDF_DETAILS))
dump_use (dump_file, use);
{
use = iv_use (data, iv->use_id);
- if (use->type != USE_NONLINEAR_EXPR
- && use->type != USE_OUTER)
- abort ();
+ gcc_assert (use->type == USE_NONLINEAR_EXPR
+ || use->type == USE_OUTER);
if (type == USE_NONLINEAR_EXPR)
use->type = USE_NONLINEAR_EXPR;
*civ = *iv;
stmt = SSA_NAME_DEF_STMT (op);
- if (TREE_CODE (stmt) != PHI_NODE
- && TREE_CODE (stmt) != MODIFY_EXPR)
- abort ();
+ gcc_assert (TREE_CODE (stmt) == PHI_NODE
+ || TREE_CODE (stmt) == MODIFY_EXPR);
use = record_use (data, NULL, civ, stmt, type);
iv->use_id = use->id;
record_use (data, cond_p, civ, stmt, USE_COMPARE);
}
+/* Returns true if expression EXPR is obviously invariant in LOOP,
+ i.e. if all its operands are defined outside of the LOOP. */
+
+bool
+expr_invariant_in_loop_p (struct loop *loop, tree expr)
+{
+ basic_block def_bb;
+ unsigned i, len;
+
+ if (is_gimple_min_invariant (expr))
+ return true;
+
+ if (TREE_CODE (expr) == SSA_NAME)
+ {
+ def_bb = bb_for_stmt (SSA_NAME_DEF_STMT (expr));
+ if (def_bb
+ && flow_bb_inside_loop_p (loop, def_bb))
+ return false;
+
+ return true;
+ }
+
+ if (!EXPR_P (expr))
+ return false;
+
+ len = first_rtl_op (TREE_CODE (expr));
+ for (i = 0; i < len; i++)
+ if (!expr_invariant_in_loop_p (loop, TREE_OPERAND (expr, i)))
+ return false;
+
+ return true;
+}
+
/* Cumulates the steps of indices into DATA and replaces their values with the
initial ones. Returns false when the value of the index cannot be determined.
Callback for for_each_index. */
{
struct ifs_ivopts_data *dta = data;
struct iv *iv;
- tree step, type, iv_type, iv_step;
-
+ tree step, type, iv_type, iv_step, lbound, off;
+ struct loop *loop = dta->ivopts_data->current_loop;
+
+ if (TREE_CODE (base) == MISALIGNED_INDIRECT_REF
+ || TREE_CODE (base) == ALIGN_INDIRECT_REF)
+ return false;
+
+ /* If base is a component ref, require that the offset of the reference
+ be invariant. */
+ if (TREE_CODE (base) == COMPONENT_REF)
+ {
+ off = component_ref_field_offset (base);
+ return expr_invariant_in_loop_p (loop, off);
+ }
+
+ /* If base is array, first check whether we will be able to move the
+ reference out of the loop (in order to take its address in strength
+ reduction). In order for this to work we need both lower bound
+ and step to be loop invariants. */
+ if (TREE_CODE (base) == ARRAY_REF)
+ {
+ step = array_ref_element_size (base);
+ lbound = array_ref_low_bound (base);
+
+ if (!expr_invariant_in_loop_p (loop, step)
+ || !expr_invariant_in_loop_p (loop, lbound))
+ return false;
+ }
+
if (TREE_CODE (*idx) != SSA_NAME)
return true;
iv_type = TREE_TYPE (iv->base);
type = build_pointer_type (TREE_TYPE (base));
if (TREE_CODE (base) == ARRAY_REF)
- step = array_ref_element_size (base);
- else
{
- /* The step for pointer arithmetics already is 1 byte. */
- step = fold_convert (type, integer_one_node);
+ step = array_ref_element_size (base);
+
+ /* We only handle addresses whose step is an integer constant. */
+ if (TREE_CODE (step) != INTEGER_CST)
+ return false;
}
+ else
+ /* The step for pointer arithmetics already is 1 byte. */
+ step = build_int_cst (type, 1);
if (TYPE_PRECISION (iv_type) < TYPE_PRECISION (type))
iv_step = can_count_iv_in_wider_type (dta->ivopts_data->current_loop,
object is passed to it in DATA. */
static bool
-idx_record_use (tree base ATTRIBUTE_UNUSED, tree *idx,
+idx_record_use (tree base, tree *idx,
void *data)
{
find_interesting_uses_op (data, *idx);
+ if (TREE_CODE (base) == ARRAY_REF)
+ {
+ find_interesting_uses_op (data, array_ref_element_size (base));
+ find_interesting_uses_op (data, array_ref_low_bound (base));
+ }
return true;
}
|| zero_p (step))
goto fail;
+ gcc_assert (TREE_CODE (base) != ALIGN_INDIRECT_REF);
+ gcc_assert (TREE_CODE (base) != MISALIGNED_INDIRECT_REF);
+
if (TREE_CODE (base) == INDIRECT_REF)
base = TREE_OPERAND (base, 0);
else
switch (TREE_CODE_CLASS (TREE_CODE (rhs)))
{
- case '<':
+ case tcc_comparison:
find_interesting_uses_cond (data, stmt, &TREE_OPERAND (stmt, 1));
return;
- case 'r':
+ case tcc_reference:
find_interesting_uses_address (data, stmt, &TREE_OPERAND (stmt, 1));
- if (TREE_CODE_CLASS (TREE_CODE (lhs)) == 'r')
+ if (REFERENCE_CLASS_P (lhs))
find_interesting_uses_address (data, stmt, &TREE_OPERAND (stmt, 0));
return;
default: ;
}
- if (TREE_CODE_CLASS (TREE_CODE (lhs)) == 'r')
+ if (REFERENCE_CLASS_P (lhs)
+ && is_gimple_val (rhs))
{
find_interesting_uses_address (data, stmt, &TREE_OPERAND (stmt, 0));
find_interesting_uses_op (data, rhs);
return;
}
+
+ /* TODO -- we should also handle address uses of type
+
+ memory = call (whatever);
+
+ and
+
+ call (memory). */
}
if (TREE_CODE (stmt) == PHI_NODE
{
tree phi, def;
- for (phi = phi_nodes (exit->dest); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (exit->dest); phi; phi = PHI_CHAIN (phi))
{
def = PHI_ARG_DEF_FROM_EDGE (phi, exit);
find_interesting_uses_outer (data, def);
for (i = 0; i < data->current_loop->num_nodes; i++)
{
+ edge_iterator ei;
bb = body[i];
- for (e = bb->succ; e; e = e->succ_next)
+ FOR_EACH_EDGE (e, ei, bb->succs)
if (e->dest != EXIT_BLOCK_PTR
&& !flow_bb_inside_loop_p (data->current_loop, e->dest))
find_interesting_uses_outside (data, e);
- for (phi = phi_nodes (bb); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
find_interesting_uses_stmt (data, phi);
for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
find_interesting_uses_stmt (data, bsi_stmt (bsi));
if (dump_file && (dump_flags & TDF_DETAILS))
{
+ bitmap_iterator bi;
+
fprintf (dump_file, "\n");
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
{
info = ver_info (data, i);
if (info->inv_id)
fprintf (dump_file, " is invariant (%d)%s\n",
info->inv_id, info->has_nonlin_use ? "" : ", eliminable");
}
- });
+ }
fprintf (dump_file, "\n");
}
{
/* Add 0 + 1 * iteration candidate. */
add_candidate (data,
- fold_convert (unsigned_type_node, integer_zero_node),
- fold_convert (unsigned_type_node, integer_one_node),
+ build_int_cst (unsigned_intSI_type_node, 0),
+ build_int_cst (unsigned_intSI_type_node, 1),
true, NULL);
- /* The same for a long type. */
- add_candidate (data,
- fold_convert (long_unsigned_type_node, integer_zero_node),
- fold_convert (long_unsigned_type_node, integer_one_node),
- true, NULL);
+ /* The same for a long type if it is still fast enough. */
+ if (BITS_PER_WORD > 32)
+ add_candidate (data,
+ build_int_cst (unsigned_intDI_type_node, 0),
+ build_int_cst (unsigned_intDI_type_node, 1),
+ true, NULL);
}
/* The same, but with initial value zero. */
add_candidate (data,
- fold_convert (TREE_TYPE (iv->base), integer_zero_node),
+ build_int_cst (TREE_TYPE (iv->base), 0),
iv->step, true, NULL);
phi = SSA_NAME_DEF_STMT (iv->ssa_name);
{
unsigned i;
struct iv *iv;
+ bitmap_iterator bi;
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
{
iv = ver_info (data, i)->iv;
if (iv && iv->biv_p && !zero_p (iv->step))
add_old_iv_candidates (data, iv);
- });
+ }
}
/* Adds candidates based on the value of the induction variable IV and USE. */
add_candidate (data, iv->base, iv->step, false, use);
/* The same, but with initial value zero. */
- add_candidate (data,
- fold_convert (TREE_TYPE (iv->base), integer_zero_node),
+ add_candidate (data, build_int_cst (TREE_TYPE (iv->base), 0),
iv->step, false, use);
}
if (base != TREE_OPERAND (iv->base, 0))
{
+ gcc_assert (TREE_CODE (base) != ALIGN_INDIRECT_REF);
+ gcc_assert (TREE_CODE (base) != MISALIGNED_INDIRECT_REF);
+
if (TREE_CODE (base) == INDIRECT_REF)
base = TREE_OPERAND (base, 0);
else
break;
default:
- abort ();
+ gcc_unreachable ();
+ }
+ }
+}
+
+/* Record important candidates and add them to related_cands bitmaps
+ if needed. */
+
+static void
+record_important_candidates (struct ivopts_data *data)
+{
+ unsigned i;
+ struct iv_use *use;
+
+ for (i = 0; i < n_iv_cands (data); i++)
+ {
+ struct iv_cand *cand = iv_cand (data, i);
+
+ if (cand->important)
+ bitmap_set_bit (data->important_candidates, i);
+ }
+
+ data->consider_all_candidates = (n_iv_cands (data)
+ <= CONSIDER_ALL_CANDIDATES_BOUND);
+
+ if (data->consider_all_candidates)
+ {
+ /* We will not need "related_cands" bitmaps in this case,
+ so release them to decrease peak memory consumption. */
+ for (i = 0; i < n_iv_uses (data); i++)
+ {
+ use = iv_use (data, i);
+ BITMAP_XFREE (use->related_cands);
}
}
+ else
+ {
+ /* Add important candidates to the related_cands bitmaps. */
+ for (i = 0; i < n_iv_uses (data); i++)
+ bitmap_ior_into (iv_use (data, i)->related_cands,
+ data->important_candidates);
+ }
}
/* Finds the candidates for the induction variables. */
/* Add induction variables derived from uses. */
add_derived_ivs_candidates (data);
+
+ /* Record the important candidates. */
+ record_important_candidates (data);
}
/* Allocates the data structure mapping the (use, candidate) pairs to costs.
static void
alloc_use_cost_map (struct ivopts_data *data)
{
- unsigned i, n_imp = 0, size, j;
-
- if (!data->consider_all_candidates)
- {
- for (i = 0; i < n_iv_cands (data); i++)
- {
- struct iv_cand *cand = iv_cand (data, i);
- if (cand->important)
- n_imp++;
- }
- }
+ unsigned i, size, s, j;
for (i = 0; i < n_iv_uses (data); i++)
{
struct iv_use *use = iv_use (data, i);
+ bitmap_iterator bi;
if (data->consider_all_candidates)
- {
- size = n_iv_cands (data);
- use->n_map_members = size;
- }
+ size = n_iv_cands (data);
else
{
- size = n_imp;
- EXECUTE_IF_SET_IN_BITMAP (use->related_cands, 0, j, size++);
- use->n_map_members = 0;
+ s = 0;
+ EXECUTE_IF_SET_IN_BITMAP (use->related_cands, 0, j, bi)
+ {
+ s++;
+ }
+
+ /* Round up to the power of two, so that moduling by it is fast. */
+ for (size = 1; size < s; size <<= 1)
+ continue;
}
+ use->n_map_members = size;
use->cost_map = xcalloc (size, sizeof (struct cost_pair));
}
}
struct iv_use *use, struct iv_cand *cand, unsigned cost,
bitmap depends_on)
{
- if (cost == INFTY
- && depends_on)
+ unsigned i, s;
+
+ if (cost == INFTY)
{
- BITMAP_XFREE (depends_on);
- depends_on = NULL;
+ if (depends_on)
+ BITMAP_XFREE (depends_on);
+ return;
}
if (data->consider_all_candidates)
return;
}
- if (cost == INFTY)
- return;
-
- use->cost_map[use->n_map_members].cand = cand;
- use->cost_map[use->n_map_members].cost = cost;
- use->cost_map[use->n_map_members].depends_on = depends_on;
- use->n_map_members++;
+ /* n_map_members is a power of two, so this computes modulo. */
+ s = cand->id & (use->n_map_members - 1);
+ for (i = s; i < use->n_map_members; i++)
+ if (!use->cost_map[i].cand)
+ goto found;
+ for (i = 0; i < s; i++)
+ if (!use->cost_map[i].cand)
+ goto found;
+
+ gcc_unreachable ();
+
+found:
+ use->cost_map[i].cand = cand;
+ use->cost_map[i].cost = cost;
+ use->cost_map[i].depends_on = depends_on;
}
-/* Gets cost of (USE, CANDIDATE) pair. Stores the bitmap of dependencies to
- DEPENDS_ON. */
+/* Gets cost of (USE, CANDIDATE) pair. */
-static unsigned
-get_use_iv_cost (struct ivopts_data *data,
- struct iv_use *use, struct iv_cand *cand, bitmap *depends_on)
+static struct cost_pair *
+get_use_iv_cost (struct ivopts_data *data, struct iv_use *use,
+ struct iv_cand *cand)
{
- unsigned i;
+ unsigned i, s;
+ struct cost_pair *ret;
if (!cand)
- return INFTY;
+ return NULL;
if (data->consider_all_candidates)
- i = cand->id;
- else
{
- for (i = 0; i < use->n_map_members; i++)
- if (use->cost_map[i].cand == cand)
- break;
+ ret = use->cost_map + cand->id;
+ if (!ret->cand)
+ return NULL;
- if (i == use->n_map_members)
- return INFTY;
+ return ret;
}
+
+ /* n_map_members is a power of two, so this computes modulo. */
+ s = cand->id & (use->n_map_members - 1);
+ for (i = s; i < use->n_map_members; i++)
+ if (use->cost_map[i].cand == cand)
+ return use->cost_map + i;
- if (depends_on)
- *depends_on = use->cost_map[i].depends_on;
- return use->cost_map[i].cost;
+ for (i = 0; i < s; i++)
+ if (use->cost_map[i].cand == cand)
+ return use->cost_map + i;
+
+ return NULL;
}
/* Returns estimate on cost of computing SEQ. */
return cost;
}
+/* Produce DECL_RTL for object obj so it looks like it is stored in memory. */
+static rtx
+produce_memory_decl_rtl (tree obj, int *regno)
+{
+ rtx x;
+ if (!obj)
+ abort ();
+ if (TREE_STATIC (obj) || DECL_EXTERNAL (obj))
+ {
+ const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (obj));
+ x = gen_rtx_SYMBOL_REF (Pmode, name);
+ }
+ else
+ x = gen_raw_REG (Pmode, (*regno)++);
+
+ return gen_rtx_MEM (DECL_MODE (obj), x);
+}
+
/* Prepares decl_rtl for variables referred in *EXPR_P. Callback for
walk_tree. DATA contains the actual fake register number. */
switch (TREE_CODE (*expr_p))
{
+ case ADDR_EXPR:
+ for (expr_p = &TREE_OPERAND (*expr_p, 0);
+ (handled_component_p (*expr_p)
+ || TREE_CODE (*expr_p) == REALPART_EXPR
+ || TREE_CODE (*expr_p) == IMAGPART_EXPR);
+ expr_p = &TREE_OPERAND (*expr_p, 0));
+ obj = *expr_p;
+ if (DECL_P (obj))
+ x = produce_memory_decl_rtl (obj, regno);
+ break;
+
case SSA_NAME:
*ws = 0;
obj = SSA_NAME_VAR (*expr_p);
break;
if (DECL_MODE (obj) == BLKmode)
- {
- if (TREE_STATIC (obj)
- || DECL_EXTERNAL (obj))
- {
- const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (obj));
- x = gen_rtx_SYMBOL_REF (Pmode, name);
- }
- else
- x = gen_raw_REG (Pmode, (*regno)++);
-
- x = gen_rtx_MEM (DECL_MODE (obj), x);
- }
+ x = produce_memory_decl_rtl (obj, regno);
else
x = gen_raw_REG (DECL_MODE (obj), (*regno)++);
get_computation_at (struct loop *loop,
struct iv_use *use, struct iv_cand *cand, tree at)
{
- tree ubase = use->iv->base, ustep = use->iv->step;
- tree cbase = cand->iv->base, cstep = cand->iv->step;
+ tree ubase = use->iv->base;
+ tree ustep = use->iv->step;
+ tree cbase = cand->iv->base;
+ tree cstep = cand->iv->step;
tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
tree uutype;
tree expr, delta;
s_offset = offset;
cost = 0;
- offset_p = (min_offset <= s_offset && s_offset <= max_offset);
+ offset_p = (s_offset != 0
+ && min_offset <= s_offset && s_offset <= max_offset);
ratio_p = (ratio != 1
&& -MAX_RATIO <= ratio && ratio <= MAX_RATIO
&& TEST_BIT (valid_mult, ratio + MAX_RATIO));
if (ratio_p)
addr = gen_rtx_fmt_ee (MULT, Pmode, addr, GEN_INT (rat));
+ if (var_present)
+ addr = gen_rtx_fmt_ee (PLUS, Pmode, addr, reg1);
+
if (symbol_present)
{
base = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (""));
gen_rtx_fmt_ee (PLUS, Pmode,
base,
GEN_INT (off)));
- if (var_present)
- base = gen_rtx_fmt_ee (PLUS, Pmode, reg1, base);
- }
-
- else if (var_present)
- {
- base = reg1;
- if (offset_p)
- base = gen_rtx_fmt_ee (PLUS, Pmode, base, GEN_INT (off));
}
else if (offset_p)
base = GEN_INT (off);
base = NULL_RTX;
if (base)
- addr = gen_rtx_fmt_ee (PLUS, Pmode, base, addr);
+ addr = gen_rtx_fmt_ee (PLUS, Pmode, addr, base);
start_sequence ();
addr = memory_address (Pmode, addr);
return NULL_TREE;
}
-/* Estimates cost of forcing EXPR into variable. DEPENDS_ON is a set of the
+/* Estimates cost of forcing EXPR into a variable. DEPENDS_ON is a set of the
invariants the computation depends on. */
static unsigned
static unsigned integer_cost;
static unsigned symbol_cost;
static unsigned address_cost;
+ tree op0, op1;
+ unsigned cost0, cost1, cost;
+ enum machine_mode mode;
if (!costs_initialized)
{
return address_cost;
}
- /* Just an arbitrary value, FIXME. */
- return target_spill_cost;
-}
-
-/* Peels a single layer of ADDR. If DIFF is not NULL, do it only if the
- offset is constant and add the offset to DIFF. */
-
-static tree
-peel_address (tree addr, unsigned HOST_WIDE_INT *diff)
-{
- tree off, size;
- HOST_WIDE_INT bit_offset;
-
- switch (TREE_CODE (addr))
+ switch (TREE_CODE (expr))
{
- case SSA_NAME:
- case INDIRECT_REF:
- case BIT_FIELD_REF:
- case VAR_DECL:
- case PARM_DECL:
- case RESULT_DECL:
- case STRING_CST:
- case REALPART_EXPR:
- case IMAGPART_EXPR:
- return NULL_TREE;
-
- case COMPONENT_REF:
- off = DECL_FIELD_BIT_OFFSET (TREE_OPERAND (addr, 1));
- bit_offset = TREE_INT_CST_LOW (off);
-
- if (bit_offset % BITS_PER_UNIT)
- abort ();
-
- if (diff)
- *diff += bit_offset / BITS_PER_UNIT;
-
- return TREE_OPERAND (addr, 0);
-
- case ARRAY_REF:
- off = TREE_OPERAND (addr, 1);
-
- if (diff)
- {
- if (!cst_and_fits_in_hwi (off))
- return NULL_TREE;
+ case PLUS_EXPR:
+ case MINUS_EXPR:
+ case MULT_EXPR:
+ op0 = TREE_OPERAND (expr, 0);
+ op1 = TREE_OPERAND (expr, 1);
- size = TYPE_SIZE_UNIT (TREE_TYPE (addr));
- if (!cst_and_fits_in_hwi (size))
- return NULL_TREE;
+ if (is_gimple_val (op0))
+ cost0 = 0;
+ else
+ cost0 = force_var_cost (data, op0, NULL);
- *diff += TREE_INT_CST_LOW (off) * TREE_INT_CST_LOW (size);
- }
+ if (is_gimple_val (op1))
+ cost1 = 0;
+ else
+ cost1 = force_var_cost (data, op1, NULL);
- return TREE_OPERAND (addr, 0);
+ break;
default:
- abort ();
+ /* Just an arbitrary value, FIXME. */
+ return target_spill_cost;
}
-}
-
-/* Checks whether E1 and E2 have constant difference, and if they do,
- store it in *DIFF. */
-static bool
-ptr_difference_const (tree e1, tree e2, unsigned HOST_WIDE_INT *diff)
-{
- int d1 = 0, d2 = 0;
- tree x;
- unsigned HOST_WIDE_INT delta1 = 0, delta2 = 0;
-
- /* Find depths of E1 and E2. */
- for (x = e1; x; x = peel_address (x, NULL))
- d1++;
- for (x = e2; x; x = peel_address (x, NULL))
- d2++;
+ mode = TYPE_MODE (TREE_TYPE (expr));
+ switch (TREE_CODE (expr))
+ {
+ case PLUS_EXPR:
+ case MINUS_EXPR:
+ cost = add_cost (mode);
+ break;
- for (; e1 && d1 > d2; e1 = peel_address (e1, &delta1))
- d1--;
- for (; e2 && d2 > d1; e2 = peel_address (e2, &delta2))
- d2--;
+ case MULT_EXPR:
+ if (cst_and_fits_in_hwi (op0))
+ cost = multiply_by_cost (int_cst_value (op0), mode);
+ else if (cst_and_fits_in_hwi (op1))
+ cost = multiply_by_cost (int_cst_value (op1), mode);
+ else
+ return target_spill_cost;
+ break;
- while (e1 && e2 && !operand_equal_p (e1, e2, 0))
- {
- e1 = peel_address (e1, &delta1);
- e2 = peel_address (e2, &delta1);
+ default:
+ gcc_unreachable ();
}
- if (!e1 || !e2)
- return false;
+ cost += cost0;
+ cost += cost1;
- *diff = delta1 - delta2;
- return true;
+ /* Bound the cost by target_spill_cost. The parts of complicated
+ computations often are either loop invariant or at least can
+ be shared between several iv uses, so letting this grow without
+ limits would not give reasonable results. */
+ return cost < target_spill_cost ? cost : target_spill_cost;
}
/* Estimates cost of expressing address ADDR as var + symbol + offset. The
tree addr, bool *symbol_present, bool *var_present,
unsigned HOST_WIDE_INT *offset, bitmap *depends_on)
{
- tree core = addr;
-
- while (core
- && TREE_CODE (core) != VAR_DECL)
- core = peel_address (core, offset);
+ tree core;
+ HOST_WIDE_INT bitsize;
+ HOST_WIDE_INT bitpos;
+ tree toffset;
+ enum machine_mode mode;
+ int unsignedp, volatilep;
+
+ core = get_inner_reference (addr, &bitsize, &bitpos, &toffset, &mode,
+ &unsignedp, &volatilep);
- if (!core)
+ if (toffset != 0
+ || bitpos % BITS_PER_UNIT != 0
+ || TREE_CODE (core) != VAR_DECL)
{
*symbol_present = false;
*var_present = true;
fd_ivopts_data = data;
walk_tree (&addr, find_depends, depends_on, NULL);
return target_spill_cost;
- }
-
+ }
+
+ *offset += bitpos / BITS_PER_UNIT;
if (TREE_STATIC (core)
|| DECL_EXTERNAL (core))
{
tree e1, tree e2, bool *symbol_present, bool *var_present,
unsigned HOST_WIDE_INT *offset, bitmap *depends_on)
{
- unsigned HOST_WIDE_INT diff = 0;
+ HOST_WIDE_INT diff = 0;
unsigned cost;
- if (TREE_CODE (e1) != ADDR_EXPR)
- abort ();
+ gcc_assert (TREE_CODE (e1) == ADDR_EXPR);
- if (TREE_CODE (e2) == ADDR_EXPR
- && ptr_difference_const (TREE_OPERAND (e1, 0),
- TREE_OPERAND (e2, 0), &diff))
+ if (ptr_difference_const (e1, e2, &diff))
{
*offset += diff;
*symbol_present = false;
return INFTY;
}
+ if (address_p)
+ {
+ /* Do not try to express address of an object with computation based
+ on address of a different object. This may cause problems in rtl
+ level alias analysis (that does not expect this to be happening,
+ as this is illegal in C), and would be unlikely to be useful
+ anyway. */
+ if (use->iv->base_object
+ && cand->iv->base_object
+ && !operand_equal_p (use->iv->base_object, cand->iv->base_object, 0))
+ return INFTY;
+ }
+
if (!cst_and_fits_in_hwi (ustep)
|| !cst_and_fits_in_hwi (cstep))
return INFTY;
(symbol/var/const parts may be omitted). If we are looking for an address,
find the cost of addressing this. */
if (address_p)
- return get_address_cost (symbol_present, var_present, offset, ratio);
+ return cost + get_address_cost (symbol_present, var_present, offset, ratio);
/* Otherwise estimate the costs for computing the expression. */
aratio = ratio > 0 ? ratio : -ratio;
/* Determines cost of basing replacement of USE on CAND in a generic
expression. */
-static void
+static bool
determine_use_iv_cost_generic (struct ivopts_data *data,
struct iv_use *use, struct iv_cand *cand)
{
unsigned cost = get_computation_cost (data, use, cand, false, &depends_on);
set_use_iv_cost (data, use, cand, cost, depends_on);
+
+ return cost != INFTY;
}
/* Determines cost of basing replacement of USE on CAND in an address. */
-static void
+static bool
determine_use_iv_cost_address (struct ivopts_data *data,
struct iv_use *use, struct iv_cand *cand)
{
unsigned cost = get_computation_cost (data, use, cand, true, &depends_on);
set_use_iv_cost (data, use, cand, cost, depends_on);
+
+ return cost != INFTY;
}
/* Computes value of induction variable IV in iteration NITER. */
struct iv_use *use, struct iv_cand *cand,
enum tree_code *compare, tree *bound)
{
+ basic_block ex_bb;
edge exit;
- struct tree_niter_desc *niter, new_niter;
+ struct tree_niter_desc niter, new_niter;
tree wider_type, type, base;
-
- /* For now just very primitive -- we work just for the single exit condition,
- and are quite conservative about the possible overflows. TODO -- both of
- these can be improved. */
- exit = single_dom_exit (loop);
- if (!exit)
+
+ /* For now works only for exits that dominate the loop latch. TODO -- extend
+ for other conditions inside loop body. */
+ ex_bb = bb_for_stmt (use->stmt);
+ if (use->stmt != last_stmt (ex_bb)
+ || TREE_CODE (use->stmt) != COND_EXPR)
return false;
- if (use->stmt != last_stmt (exit->src))
+ if (!dominated_by_p (CDI_DOMINATORS, loop->latch, ex_bb))
return false;
- niter = &loop_data (loop)->niter;
- if (!niter->niter
- || !integer_nonzerop (niter->assumptions)
- || !integer_zerop (niter->may_be_zero))
+ exit = EDGE_SUCC (ex_bb, 0);
+ if (flow_bb_inside_loop_p (loop, exit->dest))
+ exit = EDGE_SUCC (ex_bb, 1);
+ if (flow_bb_inside_loop_p (loop, exit->dest))
+ return false;
+
+ niter.niter = NULL_TREE;
+ number_of_iterations_exit (loop, exit, &niter);
+ if (!niter.niter
+ || !integer_nonzerop (niter.assumptions)
+ || !integer_zerop (niter.may_be_zero))
return false;
if (exit->flags & EDGE_TRUE_VALUE)
else
*compare = NE_EXPR;
- *bound = cand_value_at (loop, cand, use->stmt, niter->niter);
+ *bound = cand_value_at (loop, cand, use->stmt, niter.niter);
/* Let us check there is not some problem with overflows, by checking that
the number of iterations is unchanged. */
return false;
wider_type = TREE_TYPE (new_niter.niter);
- if (TYPE_PRECISION (wider_type) < TYPE_PRECISION (TREE_TYPE (niter->niter)))
- wider_type = TREE_TYPE (niter->niter);
- if (!operand_equal_p (fold_convert (wider_type, niter->niter),
+ if (TYPE_PRECISION (wider_type) < TYPE_PRECISION (TREE_TYPE (niter.niter)))
+ wider_type = TREE_TYPE (niter.niter);
+ if (!operand_equal_p (fold_convert (wider_type, niter.niter),
fold_convert (wider_type, new_niter.niter), 0))
return false;
/* Determines cost of basing replacement of USE on CAND in a condition. */
-static void
+static bool
determine_use_iv_cost_condition (struct ivopts_data *data,
struct iv_use *use, struct iv_cand *cand)
{
if (!cand->iv)
{
set_use_iv_cost (data, use, cand, INFTY, NULL);
- return;
+ return false;
}
if (may_eliminate_iv (data->current_loop, use, cand, &compare, &bound))
unsigned cost = force_var_cost (data, bound, &depends_on);
set_use_iv_cost (data, use, cand, cost, depends_on);
- return;
+ return cost != INFTY;
}
/* The induction variable elimination failed; just express the original
record_invariant (data, TREE_OPERAND (*use->op_p, 1), true);
}
- determine_use_iv_cost_generic (data, use, cand);
+ return determine_use_iv_cost_generic (data, use, cand);
}
/* Checks whether it is possible to replace the final value of USE by
if (!exit)
return false;
- if (!dominated_by_p (CDI_DOMINATORS, exit->src,
- bb_for_stmt (use->stmt)))
- abort ();
+ gcc_assert (dominated_by_p (CDI_DOMINATORS, exit->src,
+ bb_for_stmt (use->stmt)));
niter = &loop_data (loop)->niter;
if (!niter->niter
/* Determines cost of replacing final value of USE using CAND. */
-static void
+static bool
determine_use_iv_cost_outer (struct ivopts_data *data,
struct iv_use *use, struct iv_cand *cand)
{
if (!may_replace_final_value (loop, use, &value))
{
set_use_iv_cost (data, use, cand, INFTY, NULL);
- return;
+ return false;
}
depends_on = NULL;
cost /= AVG_LOOP_NITER (loop);
set_use_iv_cost (data, use, cand, cost, depends_on);
- return;
+ return cost != INFTY;
}
exit = single_dom_exit (loop);
}
set_use_iv_cost (data, use, cand, cost, depends_on);
+
+ return cost != INFTY;
}
-/* Determines cost of basing replacement of USE on CAND. */
+/* Determines cost of basing replacement of USE on CAND. Returns false
+ if USE cannot be based on CAND. */
-static void
+static bool
determine_use_iv_cost (struct ivopts_data *data,
struct iv_use *use, struct iv_cand *cand)
{
switch (use->type)
{
case USE_NONLINEAR_EXPR:
- determine_use_iv_cost_generic (data, use, cand);
- break;
+ return determine_use_iv_cost_generic (data, use, cand);
case USE_OUTER:
- determine_use_iv_cost_outer (data, use, cand);
- break;
+ return determine_use_iv_cost_outer (data, use, cand);
case USE_ADDRESS:
- determine_use_iv_cost_address (data, use, cand);
- break;
+ return determine_use_iv_cost_address (data, use, cand);
case USE_COMPARE:
- determine_use_iv_cost_condition (data, use, cand);
- break;
+ return determine_use_iv_cost_condition (data, use, cand);
default:
- abort ();
+ gcc_unreachable ();
}
}
unsigned i, j;
struct iv_use *use;
struct iv_cand *cand;
-
- data->consider_all_candidates = (n_iv_cands (data)
- <= CONSIDER_ALL_CANDIDATES_BOUND);
+ bitmap to_clear = BITMAP_XMALLOC ();
alloc_use_cost_map (data);
- if (!data->consider_all_candidates)
- {
- /* Add the important candidate entries. */
- for (j = 0; j < n_iv_cands (data); j++)
- {
- cand = iv_cand (data, j);
- if (!cand->important)
- continue;
- for (i = 0; i < n_iv_uses (data); i++)
- {
- use = iv_use (data, i);
- determine_use_iv_cost (data, use, cand);
- }
- }
- }
-
for (i = 0; i < n_iv_uses (data); i++)
{
use = iv_use (data, i);
}
else
{
- EXECUTE_IF_SET_IN_BITMAP (use->related_cands, 0, j,
+ bitmap_iterator bi;
+
+ EXECUTE_IF_SET_IN_BITMAP (use->related_cands, 0, j, bi)
{
cand = iv_cand (data, j);
- if (!cand->important)
- determine_use_iv_cost (data, use, cand);
- });
+ if (!determine_use_iv_cost (data, use, cand))
+ bitmap_set_bit (to_clear, j);
+ }
+
+ /* Remove the candidates for that the cost is infinite from
+ the list of related candidates. */
+ bitmap_and_compl_into (use->related_cands, to_clear);
+ bitmap_clear (to_clear);
}
}
+ BITMAP_XFREE (to_clear);
+
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Use-candidate costs:\n");
unsigned j, n;
tree phi, op;
struct loop *loop = data->current_loop;
+ bitmap_iterator bi;
/* We use the following model (definitely improvable, especially the
cost function -- TODO):
}
n = 0;
- for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (loop->header); phi; phi = PHI_CHAIN (phi))
{
op = PHI_RESULT (phi);
n++;
}
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
{
struct version_info *info = ver_info (data, j);
if (info->inv_id && info->has_nonlin_use)
n++;
- });
+ }
loop_data (loop)->regs_used = n;
if (dump_file && (dump_flags & TDF_DETAILS))
}
}
-/* Finds a best candidate for USE and stores it to CAND. The candidates are
- taken from the set SOL and they may depend on invariants in the set INV.
- The really used candidate and invariants are noted to USED_IVS and
- USED_INV. */
+/* Returns true if A is a cheaper cost pair than B. */
-static unsigned
-find_best_candidate (struct ivopts_data *data,
- struct iv_use *use, bitmap sol, bitmap inv,
- bitmap used_ivs, bitmap used_inv, struct iv_cand **cand)
+static bool
+cheaper_cost_pair (struct cost_pair *a, struct cost_pair *b)
{
- unsigned c, d;
- unsigned best_cost = INFTY, cost;
- struct iv_cand *cnd = NULL, *acnd;
- bitmap depends_on = NULL, asol;
+ if (!a)
+ return false;
- if (data->consider_all_candidates)
- asol = sol;
- else
+ if (!b)
+ return true;
+
+ if (a->cost < b->cost)
+ return true;
+
+ if (a->cost > b->cost)
+ return false;
+
+ /* In case the costs are the same, prefer the cheaper candidate. */
+ if (a->cand->cost < b->cand->cost)
+ return true;
+
+ return false;
+}
+
+/* Computes the cost field of IVS structure. */
+
+static void
+iv_ca_recount_cost (struct ivopts_data *data, struct iv_ca *ivs)
+{
+ unsigned cost = 0;
+
+ cost += ivs->cand_use_cost;
+ cost += ivs->cand_cost;
+ cost += ivopts_global_cost_for_size (data, ivs->n_regs);
+
+ ivs->cost = cost;
+}
+
+/* Set USE not to be expressed by any candidate in IVS. */
+
+static void
+iv_ca_set_no_cp (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_use *use)
+{
+ unsigned uid = use->id, cid, iid;
+ bitmap deps;
+ struct cost_pair *cp;
+ bitmap_iterator bi;
+
+ cp = ivs->cand_for_use[uid];
+ if (!cp)
+ return;
+ cid = cp->cand->id;
+
+ ivs->bad_uses++;
+ ivs->cand_for_use[uid] = NULL;
+ ivs->n_cand_uses[cid]--;
+
+ if (ivs->n_cand_uses[cid] == 0)
+ {
+ bitmap_clear_bit (ivs->cands, cid);
+ /* Do not count the pseudocandidates. */
+ if (cp->cand->iv)
+ ivs->n_regs--;
+ ivs->cand_cost -= cp->cand->cost;
+ }
+
+ ivs->cand_use_cost -= cp->cost;
+
+ deps = cp->depends_on;
+
+ if (deps)
{
- asol = BITMAP_XMALLOC ();
- bitmap_a_and_b (asol, sol, use->related_cands);
+ EXECUTE_IF_SET_IN_BITMAP (deps, 0, iid, bi)
+ {
+ ivs->n_invariant_uses[iid]--;
+ if (ivs->n_invariant_uses[iid] == 0)
+ ivs->n_regs--;
+ }
}
- EXECUTE_IF_SET_IN_BITMAP (asol, 0, c,
+ iv_ca_recount_cost (data, ivs);
+}
+
+/* Set cost pair for USE in set IVS to CP. */
+
+static void
+iv_ca_set_cp (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_use *use, struct cost_pair *cp)
+{
+ unsigned uid = use->id, cid, iid;
+ bitmap deps;
+ bitmap_iterator bi;
+
+ if (ivs->cand_for_use[uid] == cp)
+ return;
+
+ if (ivs->cand_for_use[uid])
+ iv_ca_set_no_cp (data, ivs, use);
+
+ if (cp)
{
- acnd = iv_cand (data, c);
- cost = get_use_iv_cost (data, use, acnd, &depends_on);
+ cid = cp->cand->id;
- if (cost == INFTY)
- goto next_cand;
- if (cost > best_cost)
- goto next_cand;
- if (cost == best_cost)
+ ivs->bad_uses--;
+ ivs->cand_for_use[uid] = cp;
+ ivs->n_cand_uses[cid]++;
+ if (ivs->n_cand_uses[cid] == 1)
{
- /* Prefer the cheaper iv. */
- if (acnd->cost >= cnd->cost)
- goto next_cand;
+ bitmap_set_bit (ivs->cands, cid);
+ /* Do not count the pseudocandidates. */
+ if (cp->cand->iv)
+ ivs->n_regs++;
+ ivs->cand_cost += cp->cand->cost;
}
- if (depends_on)
+ ivs->cand_use_cost += cp->cost;
+
+ deps = cp->depends_on;
+
+ if (deps)
{
- EXECUTE_IF_AND_COMPL_IN_BITMAP (depends_on, inv, 0, d,
- goto next_cand);
- if (used_inv)
- bitmap_a_or_b (used_inv, used_inv, depends_on);
+ EXECUTE_IF_SET_IN_BITMAP (deps, 0, iid, bi)
+ {
+ ivs->n_invariant_uses[iid]++;
+ if (ivs->n_invariant_uses[iid] == 1)
+ ivs->n_regs++;
+ }
}
- cnd = acnd;
- best_cost = cost;
-next_cand: ;
- });
+ iv_ca_recount_cost (data, ivs);
+ }
+}
- if (cnd && used_ivs)
- bitmap_set_bit (used_ivs, cnd->id);
+/* Extend set IVS by expressing USE by some of the candidates in it
+ if possible. */
- if (cand)
- *cand = cnd;
+static void
+iv_ca_add_use (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_use *use)
+{
+ struct cost_pair *best_cp = NULL, *cp;
+ bitmap_iterator bi;
+ unsigned i;
+
+ gcc_assert (ivs->upto >= use->id);
+
+ if (ivs->upto == use->id)
+ {
+ ivs->upto++;
+ ivs->bad_uses++;
+ }
- if (!data->consider_all_candidates)
- BITMAP_XFREE (asol);
+ EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
+ {
+ cp = get_use_iv_cost (data, use, iv_cand (data, i));
- return best_cost;
+ if (cheaper_cost_pair (cp, best_cp))
+ best_cp = cp;
+ }
+
+ iv_ca_set_cp (data, ivs, use, best_cp);
}
-/* Returns cost of set of ivs SOL + invariants INV. Removes unnecessary
- induction variable candidates and invariants from the sets. Only
- uses 0 .. MAX_USE - 1 are taken into account. */
+/* Get cost for assignment IVS. */
static unsigned
-set_cost_up_to (struct ivopts_data *data, bitmap sol, bitmap inv,
- unsigned max_use)
+iv_ca_cost (struct iv_ca *ivs)
+{
+ return (ivs->bad_uses ? INFTY : ivs->cost);
+}
+
+/* Returns true if all dependences of CP are among invariants in IVS. */
+
+static bool
+iv_ca_has_deps (struct iv_ca *ivs, struct cost_pair *cp)
{
unsigned i;
- unsigned cost = 0, size = 0, acost;
- struct iv_use *use;
- struct iv_cand *cand;
- bitmap used_ivs = BITMAP_XMALLOC (), used_inv = BITMAP_XMALLOC ();
+ bitmap_iterator bi;
+
+ if (!cp->depends_on)
+ return true;
- for (i = 0; i < max_use; i++)
+ EXECUTE_IF_SET_IN_BITMAP (cp->depends_on, 0, i, bi)
{
- use = iv_use (data, i);
- acost = find_best_candidate (data, use, sol, inv,
- used_ivs, used_inv, NULL);
- if (acost == INFTY)
+ if (ivs->n_invariant_uses[i] == 0)
+ return false;
+ }
+
+ return true;
+}
+
+/* Creates change of expressing USE by NEW_CP instead of OLD_CP and chains
+ it before NEXT_CHANGE. */
+
+static struct iv_ca_delta *
+iv_ca_delta_add (struct iv_use *use, struct cost_pair *old_cp,
+ struct cost_pair *new_cp, struct iv_ca_delta *next_change)
+{
+ struct iv_ca_delta *change = xmalloc (sizeof (struct iv_ca_delta));
+
+ change->use = use;
+ change->old_cp = old_cp;
+ change->new_cp = new_cp;
+ change->next_change = next_change;
+
+ return change;
+}
+
+/* Returns candidate by that USE is expressed in IVS. */
+
+static struct cost_pair *
+iv_ca_cand_for_use (struct iv_ca *ivs, struct iv_use *use)
+{
+ return ivs->cand_for_use[use->id];
+}
+
+/* Commit changes in DELTA to IVS. If FORWARD is false, the changes are
+ reverted instead. */
+
+static void
+iv_ca_delta_commit (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_ca_delta *delta, bool forward)
+{
+ struct cost_pair *from, *to;
+
+ for (; delta; delta = delta->next_change)
+ {
+ if (forward)
{
- BITMAP_XFREE (used_ivs);
- BITMAP_XFREE (used_inv);
- return INFTY;
+ from = delta->old_cp;
+ to = delta->new_cp;
}
- cost += acost;
+ else
+ {
+ from = delta->new_cp;
+ to = delta->old_cp;
+ }
+
+ gcc_assert (iv_ca_cand_for_use (ivs, delta->use) == from);
+ iv_ca_set_cp (data, ivs, delta->use, to);
}
+}
+
+/* Returns true if CAND is used in IVS. */
+
+static bool
+iv_ca_cand_used_p (struct iv_ca *ivs, struct iv_cand *cand)
+{
+ return ivs->n_cand_uses[cand->id] > 0;
+}
- EXECUTE_IF_SET_IN_BITMAP (used_ivs, 0, i,
+/* Free the list of changes DELTA. */
+
+static void
+iv_ca_delta_free (struct iv_ca_delta **delta)
+{
+ struct iv_ca_delta *act, *next;
+
+ for (act = *delta; act; act = next)
{
- cand = iv_cand (data, i);
+ next = act->next_change;
+ free (act);
+ }
- /* Do not count the pseudocandidates. */
- if (cand->iv)
- size++;
+ *delta = NULL;
+}
+
+/* Allocates new iv candidates assignment. */
+
+static struct iv_ca *
+iv_ca_new (struct ivopts_data *data)
+{
+ struct iv_ca *nw = xmalloc (sizeof (struct iv_ca));
+
+ nw->upto = 0;
+ nw->bad_uses = 0;
+ nw->cand_for_use = xcalloc (n_iv_uses (data), sizeof (struct cost_pair *));
+ nw->n_cand_uses = xcalloc (n_iv_cands (data), sizeof (unsigned));
+ nw->cands = BITMAP_XMALLOC ();
+ nw->n_regs = 0;
+ nw->cand_use_cost = 0;
+ nw->cand_cost = 0;
+ nw->n_invariant_uses = xcalloc (data->max_inv_id + 1, sizeof (unsigned));
+ nw->cost = 0;
+
+ return nw;
+}
+
+/* Free memory occupied by the set IVS. */
- cost += cand->cost;
- });
- EXECUTE_IF_SET_IN_BITMAP (used_inv, 0, i, size++);
- cost += ivopts_global_cost_for_size (data, size);
+static void
+iv_ca_free (struct iv_ca **ivs)
+{
+ free ((*ivs)->cand_for_use);
+ free ((*ivs)->n_cand_uses);
+ BITMAP_XFREE ((*ivs)->cands);
+ free ((*ivs)->n_invariant_uses);
+ free (*ivs);
+ *ivs = NULL;
+}
+
+/* Dumps IVS to FILE. */
+
+static void
+iv_ca_dump (struct ivopts_data *data, FILE *file, struct iv_ca *ivs)
+{
+ const char *pref = " invariants ";
+ unsigned i;
- bitmap_copy (sol, used_ivs);
- bitmap_copy (inv, used_inv);
+ fprintf (file, " cost %d\n", iv_ca_cost (ivs));
+ bitmap_print (file, ivs->cands, " candidates ","\n");
- BITMAP_XFREE (used_ivs);
- BITMAP_XFREE (used_inv);
+ for (i = 1; i <= data->max_inv_id; i++)
+ if (ivs->n_invariant_uses[i])
+ {
+ fprintf (file, "%s%d", pref, i);
+ pref = ", ";
+ }
+ fprintf (file, "\n");
+}
+
+/* Try changing candidate in IVS to CAND for each use. Return cost of the
+ new set, and store differences in DELTA. */
+
+static unsigned
+iv_ca_extend (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_cand *cand, struct iv_ca_delta **delta)
+{
+ unsigned i, cost;
+ struct iv_use *use;
+ struct cost_pair *old_cp, *new_cp;
+
+ *delta = NULL;
+ for (i = 0; i < ivs->upto; i++)
+ {
+ use = iv_use (data, i);
+ old_cp = iv_ca_cand_for_use (ivs, use);
+
+ if (old_cp
+ && old_cp->cand == cand)
+ continue;
+
+ new_cp = get_use_iv_cost (data, use, cand);
+ if (!new_cp)
+ continue;
+
+ if (!iv_ca_has_deps (ivs, new_cp))
+ continue;
+
+ if (!cheaper_cost_pair (new_cp, old_cp))
+ continue;
+
+ *delta = iv_ca_delta_add (use, old_cp, new_cp, *delta);
+ }
+
+ iv_ca_delta_commit (data, ivs, *delta, true);
+ cost = iv_ca_cost (ivs);
+ iv_ca_delta_commit (data, ivs, *delta, false);
return cost;
}
-/* Computes cost of set of ivs SOL + invariants INV. Removes unnecessary
- induction variable candidates and invariants from the sets. */
+/* Try narrowing set IVS by removing CAND. Return the cost of
+ the new set and store the differences in DELTA. */
static unsigned
-set_cost (struct ivopts_data *data, bitmap sol, bitmap inv)
+iv_ca_narrow (struct ivopts_data *data, struct iv_ca *ivs,
+ struct iv_cand *cand, struct iv_ca_delta **delta)
{
- return set_cost_up_to (data, sol, inv, n_iv_uses (data));
+ unsigned i, ci;
+ struct iv_use *use;
+ struct cost_pair *old_cp, *new_cp, *cp;
+ bitmap_iterator bi;
+ struct iv_cand *cnd;
+ unsigned cost;
+
+ *delta = NULL;
+ for (i = 0; i < n_iv_uses (data); i++)
+ {
+ use = iv_use (data, i);
+
+ old_cp = iv_ca_cand_for_use (ivs, use);
+ if (old_cp->cand != cand)
+ continue;
+
+ new_cp = NULL;
+
+ if (data->consider_all_candidates)
+ {
+ EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, ci, bi)
+ {
+ if (ci == cand->id)
+ continue;
+
+ cnd = iv_cand (data, ci);
+
+ cp = get_use_iv_cost (data, use, cnd);
+ if (!cp)
+ continue;
+ if (!iv_ca_has_deps (ivs, cp))
+ continue;
+
+ if (!cheaper_cost_pair (cp, new_cp))
+ continue;
+
+ new_cp = cp;
+ }
+ }
+ else
+ {
+ EXECUTE_IF_AND_IN_BITMAP (use->related_cands, ivs->cands, 0, ci, bi)
+ {
+ if (ci == cand->id)
+ continue;
+
+ cnd = iv_cand (data, ci);
+
+ cp = get_use_iv_cost (data, use, cnd);
+ if (!cp)
+ continue;
+ if (!iv_ca_has_deps (ivs, cp))
+ continue;
+
+ if (!cheaper_cost_pair (cp, new_cp))
+ continue;
+
+ new_cp = cp;
+ }
+ }
+
+ if (!new_cp)
+ {
+ iv_ca_delta_free (delta);
+ return INFTY;
+ }
+
+ *delta = iv_ca_delta_add (use, old_cp, new_cp, *delta);
+ }
+
+ iv_ca_delta_commit (data, ivs, *delta, true);
+ cost = iv_ca_cost (ivs);
+ iv_ca_delta_commit (data, ivs, *delta, false);
+
+ return cost;
}
-/* Tries to extend the sets IVS and INV in the best possible way in order
+/* Tries to extend the sets IVS in the best possible way in order
to express the USE. */
static bool
-try_add_cand_for (struct ivopts_data *data, bitmap ivs, bitmap inv,
+try_add_cand_for (struct ivopts_data *data, struct iv_ca *ivs,
struct iv_use *use)
{
- unsigned best_cost = set_cost_up_to (data, ivs, inv, use->id + 1), act_cost;
- bitmap best_ivs = BITMAP_XMALLOC ();
- bitmap best_inv = BITMAP_XMALLOC ();
- bitmap act_ivs = BITMAP_XMALLOC ();
- bitmap act_inv = BITMAP_XMALLOC ();
+ unsigned best_cost, act_cost;
unsigned i;
+ bitmap_iterator bi;
+ struct iv_cand *cand;
+ struct iv_ca_delta *best_delta = NULL, *act_delta;
struct cost_pair *cp;
- bitmap_copy (best_ivs, ivs);
- bitmap_copy (best_inv, inv);
+ iv_ca_add_use (data, ivs, use);
+ best_cost = iv_ca_cost (ivs);
+
+ cp = iv_ca_cand_for_use (ivs, use);
+ if (cp)
+ {
+ best_delta = iv_ca_delta_add (use, NULL, cp, NULL);
+ iv_ca_set_no_cp (data, ivs, use);
+ }
- for (i = 0; i < use->n_map_members; i++)
+ /* First try important candidates. Only if it fails, try the specific ones.
+ Rationale -- in loops with many variables the best choice often is to use
+ just one generic biv. If we added here many ivs specific to the uses,
+ the optimization algorithm later would be likely to get stuck in a local
+ minimum, thus causing us to create too many ivs. The approach from
+ few ivs to more seems more likely to be successful -- starting from few
+ ivs, replacing an expensive use by a specific iv should always be a
+ win. */
+ EXECUTE_IF_SET_IN_BITMAP (data->important_candidates, 0, i, bi)
{
- cp = use->cost_map + i;
- if (cp->cost == INFTY)
+ cand = iv_cand (data, i);
+
+ if (iv_ca_cand_used_p (ivs, cand))
continue;
- bitmap_copy (act_ivs, ivs);
- bitmap_set_bit (act_ivs, cp->cand->id);
- if (cp->depends_on)
- bitmap_a_or_b (act_inv, inv, cp->depends_on);
- else
- bitmap_copy (act_inv, inv);
- act_cost = set_cost_up_to (data, act_ivs, act_inv, use->id + 1);
+ cp = get_use_iv_cost (data, use, cand);
+ if (!cp)
+ continue;
+
+ iv_ca_set_cp (data, ivs, use, cp);
+ act_cost = iv_ca_extend (data, ivs, cand, &act_delta);
+ iv_ca_set_no_cp (data, ivs, use);
+ act_delta = iv_ca_delta_add (use, NULL, cp, act_delta);
if (act_cost < best_cost)
{
best_cost = act_cost;
- bitmap_copy (best_ivs, act_ivs);
- bitmap_copy (best_inv, act_inv);
+
+ iv_ca_delta_free (&best_delta);
+ best_delta = act_delta;
}
+ else
+ iv_ca_delta_free (&act_delta);
}
- bitmap_copy (ivs, best_ivs);
- bitmap_copy (inv, best_inv);
+ if (best_cost == INFTY)
+ {
+ for (i = 0; i < use->n_map_members; i++)
+ {
+ cp = use->cost_map + i;
+ cand = cp->cand;
+ if (!cand)
+ continue;
+
+ /* Already tried this. */
+ if (cand->important)
+ continue;
+
+ if (iv_ca_cand_used_p (ivs, cand))
+ continue;
+
+ act_delta = NULL;
+ iv_ca_set_cp (data, ivs, use, cp);
+ act_cost = iv_ca_extend (data, ivs, cand, &act_delta);
+ iv_ca_set_no_cp (data, ivs, use);
+ act_delta = iv_ca_delta_add (use, iv_ca_cand_for_use (ivs, use),
+ cp, act_delta);
- BITMAP_XFREE (best_ivs);
- BITMAP_XFREE (best_inv);
- BITMAP_XFREE (act_ivs);
- BITMAP_XFREE (act_inv);
+ if (act_cost < best_cost)
+ {
+ best_cost = act_cost;
+
+ if (best_delta)
+ iv_ca_delta_free (&best_delta);
+ best_delta = act_delta;
+ }
+ else
+ iv_ca_delta_free (&act_delta);
+ }
+ }
+
+ iv_ca_delta_commit (data, ivs, best_delta, true);
+ iv_ca_delta_free (&best_delta);
return (best_cost != INFTY);
}
-/* Finds an initial set of IVS and invariants INV. We do this by simply
- choosing the best candidate for each use. */
+/* Finds an initial assignment of candidates to uses. */
-static unsigned
-get_initial_solution (struct ivopts_data *data, bitmap ivs, bitmap inv)
+static struct iv_ca *
+get_initial_solution (struct ivopts_data *data)
{
+ struct iv_ca *ivs = iv_ca_new (data);
unsigned i;
for (i = 0; i < n_iv_uses (data); i++)
- if (!try_add_cand_for (data, ivs, inv, iv_use (data, i)))
- return INFTY;
+ if (!try_add_cand_for (data, ivs, iv_use (data, i)))
+ {
+ iv_ca_free (&ivs);
+ return NULL;
+ }
- return set_cost (data, ivs, inv);
+ return ivs;
}
-/* Tries to improve set of induction variables IVS and invariants INV to get
- it better than COST. */
+/* Tries to improve set of induction variables IVS. */
static bool
-try_improve_iv_set (struct ivopts_data *data,
- bitmap ivs, bitmap inv, unsigned *cost)
+try_improve_iv_set (struct ivopts_data *data, struct iv_ca *ivs)
{
- unsigned i, acost;
- bitmap new_ivs = BITMAP_XMALLOC (), new_inv = BITMAP_XMALLOC ();
- bitmap best_new_ivs = NULL, best_new_inv = NULL;
+ unsigned i, acost, best_cost = iv_ca_cost (ivs);
+ struct iv_ca_delta *best_delta = NULL, *act_delta;
+ struct iv_cand *cand;
/* Try altering the set of induction variables by one. */
for (i = 0; i < n_iv_cands (data); i++)
{
- bitmap_copy (new_ivs, ivs);
- bitmap_copy (new_inv, inv);
-
- if (bitmap_bit_p (ivs, i))
- bitmap_clear_bit (new_ivs, i);
+ cand = iv_cand (data, i);
+
+ if (iv_ca_cand_used_p (ivs, cand))
+ acost = iv_ca_narrow (data, ivs, cand, &act_delta);
else
- bitmap_set_bit (new_ivs, i);
+ acost = iv_ca_extend (data, ivs, cand, &act_delta);
- acost = set_cost (data, new_ivs, new_inv);
- if (acost >= *cost)
- continue;
-
- if (!best_new_ivs)
+ if (acost < best_cost)
{
- best_new_ivs = BITMAP_XMALLOC ();
- best_new_inv = BITMAP_XMALLOC ();
+ best_cost = acost;
+ if (best_delta)
+ iv_ca_delta_free (&best_delta);
+ best_delta = act_delta;
}
-
- *cost = acost;
- bitmap_copy (best_new_ivs, new_ivs);
- bitmap_copy (best_new_inv, new_inv);
- }
-
- /* Ditto for invariants. */
- for (i = 1; i <= data->max_inv_id; i++)
- {
- if (ver_info (data, i)->has_nonlin_use)
- continue;
-
- bitmap_copy (new_ivs, ivs);
- bitmap_copy (new_inv, inv);
-
- if (bitmap_bit_p (inv, i))
- bitmap_clear_bit (new_inv, i);
else
- bitmap_set_bit (new_inv, i);
-
- acost = set_cost (data, new_ivs, new_inv);
- if (acost >= *cost)
- continue;
-
- if (!best_new_ivs)
- {
- best_new_ivs = BITMAP_XMALLOC ();
- best_new_inv = BITMAP_XMALLOC ();
- }
-
- *cost = acost;
- bitmap_copy (best_new_ivs, new_ivs);
- bitmap_copy (best_new_inv, new_inv);
+ iv_ca_delta_free (&act_delta);
}
- BITMAP_XFREE (new_ivs);
- BITMAP_XFREE (new_inv);
-
- if (!best_new_ivs)
+ if (!best_delta)
return false;
- bitmap_copy (ivs, best_new_ivs);
- bitmap_copy (inv, best_new_inv);
- BITMAP_XFREE (best_new_ivs);
- BITMAP_XFREE (best_new_inv);
+ iv_ca_delta_commit (data, ivs, best_delta, true);
+ iv_ca_delta_free (&best_delta);
return true;
}
greedy heuristic -- we try to replace at most one candidate in the selected
solution and remove the unused ivs while this improves the cost. */
-static bitmap
+static struct iv_ca *
find_optimal_iv_set (struct ivopts_data *data)
{
- unsigned cost, i;
- bitmap set = BITMAP_XMALLOC ();
- bitmap inv = BITMAP_XMALLOC ();
+ unsigned i;
+ struct iv_ca *set;
struct iv_use *use;
- /* Set the upper bound. */
- cost = get_initial_solution (data, set, inv);
- if (cost == INFTY)
+ /* Get the initial solution. */
+ set = get_initial_solution (data);
+ if (!set)
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Unable to substitute for ivs, failed.\n");
- BITMAP_XFREE (inv);
- BITMAP_XFREE (set);
return NULL;
}
if (dump_file && (dump_flags & TDF_DETAILS))
{
- fprintf (dump_file, "Initial set of candidates (cost %d): ", cost);
- bitmap_print (dump_file, set, "", "");
- fprintf (dump_file, " invariants ");
- bitmap_print (dump_file, inv, "", "");
- fprintf (dump_file, "\n");
+ fprintf (dump_file, "Initial set of candidates:\n");
+ iv_ca_dump (data, dump_file, set);
}
- while (try_improve_iv_set (data, set, inv, &cost))
+ while (try_improve_iv_set (data, set))
{
if (dump_file && (dump_flags & TDF_DETAILS))
{
- fprintf (dump_file, "Improved to (cost %d): ", cost);
- bitmap_print (dump_file, set, "", "");
- fprintf (dump_file, " invariants ");
- bitmap_print (dump_file, inv, "", "");
- fprintf (dump_file, "\n");
+ fprintf (dump_file, "Improved to:\n");
+ iv_ca_dump (data, dump_file, set);
}
}
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Final cost %d\n\n", cost);
+ fprintf (dump_file, "Final cost %d\n\n", iv_ca_cost (set));
for (i = 0; i < n_iv_uses (data); i++)
{
use = iv_use (data, i);
- find_best_candidate (data, use, set, inv, NULL, NULL, &use->selected);
+ use->selected = iv_ca_cand_for_use (set, use)->cand;
}
- BITMAP_XFREE (inv);
-
return set;
}
/* Creates new induction variables described in SET. */
static void
-create_new_ivs (struct ivopts_data *data, bitmap set)
+create_new_ivs (struct ivopts_data *data, struct iv_ca *set)
{
unsigned i;
struct iv_cand *cand;
+ bitmap_iterator bi;
- EXECUTE_IF_SET_IN_BITMAP (set, 0, i,
+ EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
{
cand = iv_cand (data, i);
create_new_iv (data, cand);
- });
+ }
}
/* Removes statement STMT (real or a phi node). If INCLUDING_DEFINED_NAME
}
else
{
- block_stmt_iterator bsi = stmt_for_bsi (stmt);
+ block_stmt_iterator bsi = bsi_for_stmt (stmt);
bsi_remove (&bsi);
}
tree op, stmts, tgt, ass;
block_stmt_iterator bsi, pbsi;
- if (TREE_CODE (use->stmt) == PHI_NODE)
+ switch (TREE_CODE (use->stmt))
{
+ case PHI_NODE:
tgt = PHI_RESULT (use->stmt);
/* If we should keep the biv, do not replace it. */
bsi = pbsi;
bsi_next (&pbsi);
}
- }
- else if (TREE_CODE (use->stmt) == MODIFY_EXPR)
- {
+ break;
+
+ case MODIFY_EXPR:
tgt = TREE_OPERAND (use->stmt, 0);
- bsi = stmt_for_bsi (use->stmt);
+ bsi = bsi_for_stmt (use->stmt);
+ break;
+
+ default:
+ gcc_unreachable ();
}
- else
- abort ();
op = force_gimple_operand (comp, &stmts, false, SSA_NAME_VAR (tgt));
for_each_index. */
static bool
-idx_remove_ssa_names (tree base ATTRIBUTE_UNUSED, tree *idx,
+idx_remove_ssa_names (tree base, tree *idx,
void *data ATTRIBUTE_UNUSED)
{
+ tree *op;
+
if (TREE_CODE (*idx) == SSA_NAME)
*idx = SSA_NAME_VAR (*idx);
+
+ if (TREE_CODE (base) == ARRAY_REF)
+ {
+ op = &TREE_OPERAND (base, 2);
+ if (*op
+ && TREE_CODE (*op) == SSA_NAME)
+ *op = SSA_NAME_VAR (*op);
+ op = &TREE_OPERAND (base, 3);
+ if (*op
+ && TREE_CODE (*op) == SSA_NAME)
+ *op = SSA_NAME_VAR (*op);
+ }
+
return true;
}
static void
rewrite_address_base (block_stmt_iterator *bsi, tree *op, tree with)
{
- tree var = get_base_address (*op), new_var, new_name, copy, name;
+ tree bvar, var, new_var, new_name, copy, name;
tree orig;
+ var = bvar = get_base_address (*op);
+
if (!var || TREE_CODE (with) != SSA_NAME)
goto do_rewrite;
+ gcc_assert (TREE_CODE (var) != ALIGN_INDIRECT_REF);
+ gcc_assert (TREE_CODE (var) != MISALIGNED_INDIRECT_REF);
if (TREE_CODE (var) == INDIRECT_REF)
var = TREE_OPERAND (var, 0);
if (TREE_CODE (var) == SSA_NAME)
do_rewrite:
orig = NULL_TREE;
+ gcc_assert (TREE_CODE (*op) != ALIGN_INDIRECT_REF);
+ gcc_assert (TREE_CODE (*op) != MISALIGNED_INDIRECT_REF);
+
if (TREE_CODE (*op) == INDIRECT_REF)
orig = REF_ORIGINAL (*op);
if (!orig)
orig = unshare_and_remove_ssa_names (*op);
*op = build1 (INDIRECT_REF, TREE_TYPE (*op), with);
+
/* Record the original reference, for purposes of alias analysis. */
REF_ORIGINAL (*op) = orig;
}
{
tree comp = unshare_expr (get_computation (data->current_loop,
use, cand));
- block_stmt_iterator bsi = stmt_for_bsi (use->stmt);
+ block_stmt_iterator bsi = bsi_for_stmt (use->stmt);
tree stmts;
tree op = force_gimple_operand (comp, &stmts, true, NULL_TREE);
{
tree comp;
tree *op_p, cond, op, stmts, bound;
- block_stmt_iterator bsi = stmt_for_bsi (use->stmt);
+ block_stmt_iterator bsi = bsi_for_stmt (use->stmt);
enum tree_code compare;
if (may_eliminate_iv (data->current_loop,
return;
/* Try finding a phi node that copies the value out of the loop. */
- for (phi = phi_nodes (exit->dest); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (exit->dest); phi; phi = PHI_CHAIN (phi))
if (PHI_ARG_DEF_FROM_EDGE (phi, exit) == use)
break;
block_stmt_iterator bsi;
tree phi, stmt, def, next;
- if (exit->dest->pred->pred_next)
+ if (EDGE_COUNT (exit->dest->preds) > 1)
split_loop_exit_edge (exit);
if (TREE_CODE (stmts) == STATEMENT_LIST)
for (phi = phi_nodes (exit->dest); phi; phi = next)
{
- next = TREE_CHAIN (phi);
+ next = PHI_CHAIN (phi);
if (PHI_ARG_DEF_FROM_EDGE (phi, exit) == op)
{
tree value, op, stmts, tgt;
tree phi;
- if (TREE_CODE (use->stmt) == PHI_NODE)
- tgt = PHI_RESULT (use->stmt);
- else if (TREE_CODE (use->stmt) == MODIFY_EXPR)
- tgt = TREE_OPERAND (use->stmt, 0);
- else
- abort ();
+ switch (TREE_CODE (use->stmt))
+ {
+ case PHI_NODE:
+ tgt = PHI_RESULT (use->stmt);
+ break;
+ case MODIFY_EXPR:
+ tgt = TREE_OPERAND (use->stmt, 0);
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
exit = single_dom_exit (data->current_loop);
if (exit)
{
if (!cand->iv)
{
- if (!may_replace_final_value (data->current_loop, use, &value))
- abort ();
+ bool ok = may_replace_final_value (data->current_loop, use, &value);
+ gcc_assert (ok);
}
else
value = get_computation_at (data->current_loop,
use, cand, last_stmt (exit->src));
+ value = unshare_expr (value);
op = force_gimple_operand (value, &stmts, true, SSA_NAME_VAR (tgt));
/* If we will preserve the iv anyway and we would need to perform
if (stmts && name_info (data, tgt)->preserve_biv)
return;
- for (phi = phi_nodes (exit->dest); phi; phi = TREE_CHAIN (phi))
+ for (phi = phi_nodes (exit->dest); phi; phi = PHI_CHAIN (phi))
{
use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, exit);
break;
default:
- abort ();
+ gcc_unreachable ();
}
modify_stmt (use->stmt);
}
{
use = iv_use (data, i);
cand = use->selected;
- if (!cand)
- abort ();
+ gcc_assert (cand);
rewrite_use (data, use, cand);
}
remove_unused_ivs (struct ivopts_data *data)
{
unsigned j;
+ bitmap_iterator bi;
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
{
struct version_info *info;
&& !info->iv->have_use_for
&& !info->preserve_biv)
remove_statement (SSA_NAME_DEF_STMT (info->iv->ssa_name), true);
- });
+ }
}
/* Frees data allocated by the optimization of a single loop. */
free_loop_data (struct ivopts_data *data)
{
unsigned i, j;
+ bitmap_iterator bi;
- EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i,
+ EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
{
struct version_info *info;
info->has_nonlin_use = false;
info->preserve_biv = false;
info->inv_id = 0;
- });
+ }
bitmap_clear (data->relevant);
+ bitmap_clear (data->important_candidates);
for (i = 0; i < n_iv_uses (data); i++)
{
free_loop_data (data);
free (data->version_info);
BITMAP_XFREE (data->relevant);
+ BITMAP_XFREE (data->important_candidates);
VARRAY_FREE (decl_rtl_to_reset);
VARRAY_FREE (data->iv_uses);
tree_ssa_iv_optimize_loop (struct ivopts_data *data, struct loop *loop)
{
bool changed = false;
- bitmap iv_set;
+ struct iv_ca *iv_ca;
edge exit;
data->current_loop = loop;
determine_set_costs (data);
/* Find the optimal set of induction variables (item 3, part 2). */
- iv_set = find_optimal_iv_set (data);
- if (!iv_set)
+ iv_ca = find_optimal_iv_set (data);
+ if (!iv_ca)
goto finish;
changed = true;
/* Create the new induction variables (item 4, part 1). */
- create_new_ivs (data, iv_set);
+ create_new_ivs (data, iv_ca);
+ iv_ca_free (&iv_ca);
/* Rewrite the uses (item 4, part 2). */
rewrite_uses (data);
loop_commit_inserts ();
- BITMAP_XFREE (iv_set);
-
/* We have changed the structure of induction variables; it might happen
that definitions in the scev database refer to some of them that were
eliminated. */
#ifdef ENABLE_CHECKING
verify_loop_closed_ssa ();
+ verify_stmts ();
#endif
/* Scan the loops, inner ones first. */
while (loop != loops->tree_root)
{
- if (tree_ssa_iv_optimize_loop (&data, loop))
- {
-#ifdef ENABLE_CHECKING
- verify_loop_closed_ssa ();
-#endif
- }
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ flow_loop_dump (loop, dump_file, NULL, 1);
+
+ tree_ssa_iv_optimize_loop (&data, loop);
if (loop->next)
{
loop = loop->outer;
}
+#ifdef ENABLE_CHECKING
+ verify_loop_closed_ssa ();
+ verify_stmts ();
+#endif
+
tree_ssa_iv_optimize_finalize (loops, &data);
}
OSDN Git Service
