/* Instruction scheduling pass. Selective scheduler and pipeliner.
- Copyright (C) 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
+ Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011
+ Free Software Foundation, Inc.
This file is part of GCC.
#include "system.h"
#include "coretypes.h"
#include "tm.h"
-#include "toplev.h"
-#include "rtl.h"
+#include "rtl-error.h"
#include "tm_p.h"
#include "hard-reg-set.h"
#include "regs.h"
#include "insn-config.h"
#include "insn-attr.h"
#include "except.h"
-#include "toplev.h"
#include "recog.h"
#include "params.h"
#include "target.h"
#include "langhooks.h"
#include "rtlhooks-def.h"
#include "output.h"
+#include "emit-rtl.h"
#ifdef INSN_SCHEDULING
#include "sel-sched-ir.h"
FENCE_ISSUED_INSNS (fence) = 0;
FENCE_STARTS_CYCLE_P (fence) = 1;
can_issue_more = issue_rate;
+ FENCE_ISSUE_MORE (fence) = can_issue_more;
for (i = 0; VEC_iterate (rtx, FENCE_EXECUTING_INSNS (fence), i, insn); )
{
in_fallthru_bb_p (rtx insn, rtx succ)
{
basic_block bb = BLOCK_FOR_INSN (insn);
+ edge e;
if (bb == BLOCK_FOR_INSN (succ))
return true;
- if (find_fallthru_edge (bb))
- bb = find_fallthru_edge (bb)->dest;
+ e = find_fallthru_edge_from (bb);
+ if (e)
+ bb = e->dest;
else
return false;
/* Do not allow clobbering the address register of speculative
insns. */
if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE)
- && bitmap_bit_p (VINSN_REG_USES (EXPR_VINSN (expr)),
- expr_dest_regno (expr)))
+ && register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)),
+ expr_dest_reg (expr)))
EXPR_TARGET_AVAILABLE (expr) = false;
return true;
{
rtx_search_arg_p p = (rtx_search_arg_p) arg;
- /* The last param FOR_GCSE is true, because otherwise it performs excessive
- substitutions like
- r8 = r33
- r16 = r33
- for the last insn it presumes r33 equivalent to r8, so it changes it to
- r33. Actually, there's no change, but it spoils debugging. */
- if (exp_equiv_p (*cur_rtx, p->x, 0, true))
- {
- /* Bail out if we occupy more than one register. */
- if (REG_P (*cur_rtx)
- && HARD_REGISTER_P (*cur_rtx)
- && hard_regno_nregs[REGNO(*cur_rtx)][GET_MODE (*cur_rtx)] > 1)
+ if (REG_P (*cur_rtx) && REGNO (*cur_rtx) == REGNO (p->x))
+ {
+ /* Bail out if mode is different or more than one register is used. */
+ if (GET_MODE (*cur_rtx) != GET_MODE (p->x)
+ || (HARD_REGISTER_P (*cur_rtx)
+ && hard_regno_nregs[REGNO(*cur_rtx)][GET_MODE (*cur_rtx)] > 1))
{
p->n = 0;
return 1;
}
if (GET_CODE (*cur_rtx) == SUBREG
- && REG_P (p->x)
- && REGNO (SUBREG_REG (*cur_rtx)) == REGNO (p->x))
+ && (!REG_P (SUBREG_REG (*cur_rtx))
+ || REGNO (SUBREG_REG (*cur_rtx)) == REGNO (p->x)))
{
/* ??? Do not support substituting regs inside subregs. In that case,
simplify_subreg will be called by validate_replace_rtx, and
{
struct rtx_search_arg arg;
+ gcc_assert (REG_P (what));
arg.x = what;
arg.n = 0;
/* Can't use regs which aren't saved by
the prologue. */
|| !TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg + i)
+ /* Can't use regs with non-null REG_BASE_VALUE, because adjusting
+ it affects aliasing globally and invalidates all AV sets. */
+ || get_reg_base_value (cur_reg + i)
#ifdef LEAF_REGISTERS
/* We can't use a non-leaf register if we're in a
leaf function. */
if (!reload_completed && !HARD_REGISTER_NUM_P (regno))
return;
- mode = GET_MODE (orig_dest);
+ if (reload_completed)
+ cl = get_reg_class (def->orig_insn);
- /* Stop when mode is not supported for renaming. Also can't proceed
- if the original register is one of the fixed_regs, global_regs or
- frame pointer. */
+ /* Stop if the original register is one of the fixed_regs, global_regs or
+ frame pointer, or we could not discover its class. */
if (fixed_regs[regno]
|| global_regs[regno]
-#if FRAME_POINTER_REGNUM != HARD_FRAME_POINTER_REGNUM
- || (frame_pointer_needed && regno == HARD_FRAME_POINTER_REGNUM)
+#if !HARD_FRAME_POINTER_IS_FRAME_POINTER
+ || (frame_pointer_needed && regno == HARD_FRAME_POINTER_REGNUM)
#else
- || (frame_pointer_needed && regno == FRAME_POINTER_REGNUM)
+ || (frame_pointer_needed && regno == FRAME_POINTER_REGNUM)
#endif
- )
+ || (reload_completed && cl == NO_REGS))
{
SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs);
FIXME: it is enough to do this once per all original defs. */
if (frame_pointer_needed)
{
- int i;
-
- for (i = hard_regno_nregs[FRAME_POINTER_REGNUM][Pmode]; i--;)
- SET_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs,
- FRAME_POINTER_REGNUM + i);
+ add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
+ Pmode, FRAME_POINTER_REGNUM);
-#if FRAME_POINTER_REGNUM != HARD_FRAME_POINTER_REGNUM
- for (i = hard_regno_nregs[HARD_FRAME_POINTER_REGNUM][Pmode]; i--;)
- SET_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs,
- HARD_FRAME_POINTER_REGNUM + i);
-#endif
+ if (!HARD_FRAME_POINTER_IS_FRAME_POINTER)
+ add_to_hard_reg_set (®_rename_p->unavailable_hard_regs,
+ Pmode, HARD_FRAME_POINTER_IS_FRAME_POINTER);
}
#ifdef STACK_REGS
/* Leave regs as 'available' only from the current
register class. */
- cl = get_reg_class (def->orig_insn);
- gcc_assert (cl != NO_REGS);
COPY_HARD_REG_SET (reg_rename_p->available_for_renaming,
reg_class_contents[cl]);
+ mode = GET_MODE (orig_dest);
+
/* Leave only registers available for this mode. */
if (!sel_hrd.regs_for_mode_ok[mode])
init_regs_for_mode (mode);
0, cur_reg, hrsi)
if (! TEST_HARD_REG_BIT (hard_regs_used, cur_reg))
{
+ /* Check that all hard regs for mode are available. */
+ for (i = 1, n = hard_regno_nregs[cur_reg][mode]; i < n; i++)
+ if (TEST_HARD_REG_BIT (hard_regs_used, cur_reg + i)
+ || !TEST_HARD_REG_BIT (reg_rename_p->available_for_renaming,
+ cur_reg + i))
+ break;
+
+ if (i < n)
+ continue;
+
/* All hard registers are available. */
if (best_new_reg < 0
|| reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg])
rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p,
original_insns, is_orig_reg_p_ptr);
+ /* FIXME loop over hard_regno_nregs here. */
gcc_assert (best_reg == NULL_RTX
|| TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg)));
regno = expr_dest_regno (expr);
mode = GET_MODE (EXPR_LHS (expr));
target_available = EXPR_TARGET_AVAILABLE (expr) == 1;
- n = reload_completed ? hard_regno_nregs[regno][mode] : 1;
+ n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs[regno][mode] : 1;
live_available = hard_available = true;
for (i = 0; i < n; i++)
if (recovery_block != NULL)
{
rtx twin_rtx;
- insn_t twin;
twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr)));
twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX);
- twin = sel_gen_recovery_insn_from_rtx_after (twin_rtx,
- INSN_EXPR (orig_insn),
- INSN_SEQNO (insn),
- bb_note (recovery_block));
+ sel_gen_recovery_insn_from_rtx_after (twin_rtx,
+ INSN_EXPR (orig_insn),
+ INSN_SEQNO (insn),
+ bb_note (recovery_block));
}
/* If we've generated a data speculation check, make sure
ds_t *has_dep_p;
ds_t full_ds;
+ /* ??? We use dependencies of non-debug insns on debug insns to
+ indicate that the debug insns need to be reset if the non-debug
+ insn is pulled ahead of it. It's hard to figure out how to
+ introduce such a notion in sel-sched, but it already fails to
+ support debug insns in other ways, so we just go ahead and
+ let the deug insns go corrupt for now. */
+ if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn))
+ return MOVEUP_EXPR_SAME;
+
/* When inside_insn_group, delegate to the helper. */
if (inside_insn_group)
return moveup_expr_inside_insn_group (expr, through_insn);
|| ! in_current_region_p (fallthru_bb))
return MOVEUP_EXPR_NULL;
- /* And it should be mutually exclusive with through_insn, or
- be an unconditional jump. */
- if (! any_uncondjump_p (insn)
- && ! sched_insns_conditions_mutex_p (insn, through_insn)
+ /* And it should be mutually exclusive with through_insn. */
+ if (! sched_insns_conditions_mutex_p (insn, through_insn)
&& ! DEBUG_INSN_P (through_insn))
return MOVEUP_EXPR_NULL;
}
EXPR_TARGET_AVAILABLE (expr) = false;
if (pti->type == TRANS_SPECULATION)
{
- ds_t ds;
-
- ds = EXPR_SPEC_DONE_DS (expr);
-
EXPR_SPEC_DONE_DS (expr) = pti->ds;
EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check;
}
sel_print ("real successors num: %d\n", sinfo->all_succs_n);
}
- /* Add insn to to the tail of current path. */
+ /* Add insn to the tail of current path. */
ilist_add (&p, insn);
- for (is = 0; VEC_iterate (rtx, sinfo->succs_ok, is, succ); is++)
+ FOR_EACH_VEC_ELT (rtx, sinfo->succs_ok, is, succ)
{
av_set_t succ_set;
VEC_index (int, sinfo->probs_ok, is),
sinfo->all_prob);
- if (sinfo->all_succs_n > 1
- && sinfo->all_succs_n == sinfo->succs_ok_n)
+ if (sinfo->all_succs_n > 1)
{
/* Find EXPR'es that came from *all* successors and save them
into expr_in_all_succ_branches. This set will be used later
/* Check liveness restrictions via hard way when there are more than
two successors. */
if (sinfo->succs_ok_n > 2)
- for (is = 0; VEC_iterate (rtx, sinfo->succs_ok, is, succ); is++)
+ FOR_EACH_VEC_ELT (rtx, sinfo->succs_ok, is, succ)
{
basic_block succ_bb = BLOCK_FOR_INSN (succ);
/* Finally, check liveness restrictions on paths leaving the region. */
if (sinfo->all_succs_n > sinfo->succs_ok_n)
- for (is = 0; VEC_iterate (rtx, sinfo->succs_other, is, succ); is++)
+ FOR_EACH_VEC_ELT (rtx, sinfo->succs_other, is, succ)
mark_unavailable_targets
(av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ)));
gcc_assert (EXPR_PRIORITY_ADJ (expr) >= 0);
- if (sched_verbose >= 2)
- sel_print ("sel_target_adjust_priority: insn %d, %d +%d = %d.\n",
+ if (sched_verbose >= 4)
+ sel_print ("sel_target_adjust_priority: insn %d, %d+%d = %d.\n",
INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr),
EXPR_PRIORITY_ADJ (expr), new_priority);
vinsn_t vinsn;
int n;
- for (n = 0; VEC_iterate (vinsn_t, vinsn_vec, n, vinsn); n++)
+ FOR_EACH_VEC_ELT (vinsn_t, vinsn_vec, n, vinsn)
if (VINSN_SEPARABLE_P (vinsn))
{
if (vinsn_equal_p (vinsn, EXPR_VINSN (expr)))
renaming. Check with the right register instead. */
if (sparams->dest && REG_P (sparams->dest))
{
- unsigned regno = REGNO (sparams->dest);
+ rtx reg = sparams->dest;
vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn);
- if (bitmap_bit_p (VINSN_REG_SETS (failed_vinsn), regno)
- || bitmap_bit_p (VINSN_REG_USES (failed_vinsn), regno)
- || bitmap_bit_p (VINSN_REG_CLOBBERS (failed_vinsn), regno))
+ if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg)
+ || register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg)
+ || register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg))
return true;
}
vinsn_t vinsn;
int n;
- for (n = 0; VEC_iterate (vinsn_t, *vinsn_vec, n, vinsn); n++)
+ FOR_EACH_VEC_ELT (vinsn_t, *vinsn_vec, n, vinsn)
vinsn_detach (vinsn);
VEC_block_remove (vinsn_t, *vinsn_vec, 0, len);
}
}
/* Sort the vector. */
- qsort (VEC_address (expr_t, vec_av_set), VEC_length (expr_t, vec_av_set),
- sizeof (expr_t), sel_rank_for_schedule);
+ VEC_qsort (expr_t, vec_av_set, sel_rank_for_schedule);
/* We record maximal priority of insns in av set for current instruction
group. */
{
expr_t expr = VEC_index (expr_t, vec_av_set, n);
insn_t insn = EXPR_INSN_RTX (expr);
- char target_available;
+ signed char target_available;
bool is_orig_reg_p = true;
int need_cycles, new_prio;
gcc_assert (min_need_stall == 0);
/* Sort the vector. */
- qsort (VEC_address (expr_t, vec_av_set), VEC_length (expr_t, vec_av_set),
- sizeof (expr_t), sel_rank_for_schedule);
+ VEC_qsort (expr_t, vec_av_set, sel_rank_for_schedule);
if (sched_verbose >= 4)
{
sel_print ("Total ready exprs: %d, stalled: %d\n",
VEC_length (expr_t, vec_av_set), stalled);
sel_print ("Sorted av set (%d): ", VEC_length (expr_t, vec_av_set));
- for (n = 0; VEC_iterate (expr_t, vec_av_set, n, expr); n++)
+ FOR_EACH_VEC_ELT (expr_t, vec_av_set, n, expr)
dump_expr (expr);
sel_print ("\n");
}
sched_extend_ready_list (ready.n_ready);
}
- for (n = 0; VEC_iterate (expr_t, vec_av_set, n, expr); n++)
+ FOR_EACH_VEC_ELT (expr_t, vec_av_set, n, expr)
{
vinsn_t vi = EXPR_VINSN (expr);
insn_t insn = VINSN_INSN_RTX (vi);
ran_hook = true;
}
else
- issue_more = issue_rate;
+ issue_more = FENCE_ISSUE_MORE (fence);
/* Ensure that ready list and vec_av_set are in line with each other,
i.e. vec_av_set[i] == ready_element (&ready, i). */
calculate_privileged_insns (void)
{
expr_t cur_expr, min_spec_expr = NULL;
- insn_t cur_insn, min_spec_insn;
int privileged_n = 0, i;
for (i = 0; i < ready.n_ready; i++)
continue;
if (! min_spec_expr)
- {
- min_spec_insn = ready_element (&ready, i);
- min_spec_expr = find_expr_for_ready (i, true);
- }
+ min_spec_expr = find_expr_for_ready (i, true);
- cur_insn = ready_element (&ready, i);
cur_expr = find_expr_for_ready (i, true);
if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr))
if (recog_memoized (insn) < 0)
{
if (!FENCE_STARTS_CYCLE_P (fence)
- /* FIXME: Is this condition necessary? */
- && VINSN_UNIQUE_P (EXPR_VINSN (expr))
&& INSN_ASM_P (insn))
/* This is asm insn which is tryed to be issued on the
cycle not first. Issue it on the next cycle. */
if (dfa_lookahead > 0)
{
cycle_issued_insns = FENCE_ISSUED_INSNS (fence);
+ /* TODO: pass equivalent of first_cycle_insn_p to max_issue (). */
can_issue = max_issue (&ready, privileged_n,
- FENCE_STATE (fence), index);
+ FENCE_STATE (fence), true, index);
if (sched_verbose >= 2)
sel_print ("max_issue: we can issue %d insns, already did %d insns\n",
can_issue, FENCE_ISSUED_INSNS (fence));
best = fill_ready_list (av_vliw_ptr, bnds, fence, pneed_stall);
if (best == NULL && ready.n_ready > 0)
{
- int privileged_n, index, avail_n;
+ int privileged_n, index;
can_issue_more = invoke_reorder_hooks (fence);
if (can_issue_more > 0)
scheduled due to liveness restrictions on its destination register.
In the future, we'd like to choose once and then just probe insns
in the order of their priority. */
- avail_n = invoke_dfa_lookahead_guard ();
+ invoke_dfa_lookahead_guard ();
privileged_n = calculate_privileged_insns ();
can_issue_more = choose_best_insn (fence, privileged_n, &index);
if (can_issue_more)
{
can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best),
can_issue_more);
- if (can_issue_more == 0)
+ if (targetm.sched.variable_issue
+ && can_issue_more == 0)
*pneed_stall = 1;
}
if (INSN_P (insn))
EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index;
- if (bitmap_bit_p (code_motion_visited_blocks, new_bb->index))
- {
- bitmap_set_bit (code_motion_visited_blocks, succ->index);
- bitmap_clear_bit (code_motion_visited_blocks, new_bb->index);
- }
+ if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index))
+ bitmap_set_bit (code_motion_visited_blocks, succ->index);
gcc_assert (LABEL_P (BB_HEAD (new_bb))
&& LABEL_P (BB_HEAD (succ)));
}
/* Return insn after which we must insert bookkeeping code for path(s) incoming
- into E2->dest, except from E1->src. */
+ into E2->dest, except from E1->src. If the returned insn immediately
+ precedes a fence, assign that fence to *FENCE_TO_REWIND. */
static insn_t
-find_place_for_bookkeeping (edge e1, edge e2)
+find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind)
{
insn_t place_to_insert;
/* Find a basic block that can hold bookkeeping. If it can be found, do not
sel_print ("Pre-existing bookkeeping block is %i\n", book_block->index);
}
- /* If basic block ends with a jump, insert bookkeeping code right before it. */
+ *fence_to_rewind = NULL;
+ /* If basic block ends with a jump, insert bookkeeping code right before it.
+ Notice if we are crossing a fence when taking PREV_INSN. */
if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert))
- place_to_insert = PREV_INSN (place_to_insert);
+ {
+ *fence_to_rewind = flist_lookup (fences, place_to_insert);
+ place_to_insert = PREV_INSN (place_to_insert);
+ }
return place_to_insert;
}
insn_t join_point, place_to_insert, new_insn;
int new_seqno;
bool need_to_exchange_data_sets;
+ fence_t fence_to_rewind;
if (sched_verbose >= 4)
sel_print ("Generating bookkeeping insn (%d->%d)\n", e1->src->index,
e2->dest->index);
join_point = sel_bb_head (e2->dest);
- place_to_insert = find_place_for_bookkeeping (e1, e2);
- if (!place_to_insert)
- return NULL;
+ place_to_insert = find_place_for_bookkeeping (e1, e2, &fence_to_rewind);
new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point);
need_to_exchange_data_sets
= sel_bb_empty_p (BLOCK_FOR_INSN (place_to_insert));
new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno);
+ if (fence_to_rewind)
+ FENCE_INSN (fence_to_rewind) = new_insn;
+
/* When inserting bookkeeping insn in new block, av sets should be
following: old basic block (that now holds bookkeeping) data sets are
the same as was before generation of bookkeeping, and new basic block
move_cond_jump (rtx insn, bnd_t bnd)
{
edge ft_edge;
- basic_block block_from, block_next, block_new;
- rtx next, prev, link;
+ basic_block block_from, block_next, block_new, block_bnd, bb;
+ rtx next, prev, link, head;
- /* BLOCK_FROM holds basic block of the jump. */
block_from = BLOCK_FOR_INSN (insn);
+ block_bnd = BLOCK_FOR_INSN (BND_TO (bnd));
+ prev = BND_TO (bnd);
- /* Moving of jump should not cross any other jumps or
- beginnings of new basic blocks. */
- gcc_assert (block_from == BLOCK_FOR_INSN (BND_TO (bnd)));
+#ifdef ENABLE_CHECKING
+ /* Moving of jump should not cross any other jumps or beginnings of new
+ basic blocks. The only exception is when we move a jump through
+ mutually exclusive insns along fallthru edges. */
+ if (block_from != block_bnd)
+ {
+ bb = block_from;
+ for (link = PREV_INSN (insn); link != PREV_INSN (prev);
+ link = PREV_INSN (link))
+ {
+ if (INSN_P (link))
+ gcc_assert (sched_insns_conditions_mutex_p (insn, link));
+ if (BLOCK_FOR_INSN (link) && BLOCK_FOR_INSN (link) != bb)
+ {
+ gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link));
+ bb = BLOCK_FOR_INSN (link);
+ }
+ }
+ }
+#endif
/* Jump is moved to the boundary. */
- prev = BND_TO (bnd);
next = PREV_INSN (insn);
BND_TO (bnd) = insn;
- ft_edge = find_fallthru_edge (block_from);
+ ft_edge = find_fallthru_edge_from (block_from);
block_next = ft_edge->dest;
/* There must be a fallthrough block (or where should go
control flow in case of false jump predicate otherwise?). */
gcc_assert (block_new->next_bb == block_next
&& block_from->next_bb == block_new);
- gcc_assert (BB_END (block_from) == insn);
-
- /* Move all instructions except INSN from BLOCK_FROM to
- BLOCK_NEW. */
- for (link = prev; link != insn; link = NEXT_INSN (link))
+ /* Move all instructions except INSN to BLOCK_NEW. */
+ bb = block_bnd;
+ head = BB_HEAD (block_new);
+ while (bb != block_from->next_bb)
{
- EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
- df_insn_change_bb (link, block_new);
- }
+ rtx from, to;
+ from = bb == block_bnd ? prev : sel_bb_head (bb);
+ to = bb == block_from ? next : sel_bb_end (bb);
+
+ /* The jump being moved can be the first insn in the block.
+ In this case we don't have to move anything in this block. */
+ if (NEXT_INSN (to) != from)
+ {
+ reorder_insns (from, to, head);
- /* Set correct basic block and instructions properties. */
- BB_END (block_new) = PREV_INSN (insn);
+ for (link = to; link != head; link = PREV_INSN (link))
+ EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
+ head = to;
+ }
- NEXT_INSN (PREV_INSN (prev)) = insn;
- PREV_INSN (insn) = PREV_INSN (prev);
+ /* Cleanup possibly empty blocks left. */
+ block_next = bb->next_bb;
+ if (bb != block_from)
+ tidy_control_flow (bb, false);
+ bb = block_next;
+ }
/* Assert there is no jump to BLOCK_NEW, only fallthrough edge. */
gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new)));
- PREV_INSN (prev) = BB_HEAD (block_new);
- NEXT_INSN (next) = NEXT_INSN (BB_HEAD (block_new));
- NEXT_INSN (BB_HEAD (block_new)) = prev;
- PREV_INSN (NEXT_INSN (next)) = next;
gcc_assert (!sel_bb_empty_p (block_from)
&& !sel_bb_empty_p (block_new));
int i;
insn_t insn;
- for (i = 0; VEC_iterate (insn_t, vec_temp_moveop_nops, i, insn); i++)
+ FOR_EACH_VEC_ELT (insn_t, vec_temp_moveop_nops, i, insn)
{
gcc_assert (INSN_NOP_P (insn));
return_nop_to_pool (insn, full_tidying);
EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi)
{
+ unsigned uid;
+ bitmap_iterator bi;
+
/* We allocate these bitmaps lazily. */
if (! INSN_ORIGINATORS_BY_UID (book_uid))
INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL);
bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid),
current_originators);
+
+ /* Transitively add all originators' originators. */
+ EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi)
+ if (INSN_ORIGINATORS_BY_UID (uid))
+ bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid),
+ INSN_ORIGINATORS_BY_UID (uid));
}
return should_move;
debug_state (FENCE_STATE (fence));
if (!DEBUG_INSN_P (insn))
FENCE_STARTS_CYCLE_P (fence) = 0;
+ FENCE_ISSUE_MORE (fence) = can_issue_more;
return asm_p;
}
blist_add (&bnds, insn, NULL, FENCE_DC (fence));
bnds_tailp = &BLIST_NEXT (bnds);
set_target_context (FENCE_TC (fence));
+ can_issue_more = FENCE_ISSUE_MORE (fence);
target_bb = INSN_BB (insn);
/* Do while we can add any operation to the current group. */
{
blist_t *bnds_tailp1, *bndsp;
expr_t expr_vliw;
- int need_stall;
- int was_stall = 0, scheduled_insns = 0, stall_iterations = 0;
+ int need_stall = false;
+ int was_stall = 0, scheduled_insns = 0;
int max_insns = pipelining_p ? issue_rate : 2 * issue_rate;
int max_stall = pipelining_p ? 1 : 3;
bool last_insn_was_debug = false;
do
{
expr_vliw = find_best_expr (&av_vliw, bnds, fence, &need_stall);
- if (!expr_vliw && need_stall)
+ if (! expr_vliw && need_stall)
{
/* All expressions required a stall. Do not recompute av sets
as we'll get the same answer (modulo the insns between
the fence and its boundary, which will not be available for
- pipelining). */
- gcc_assert (! expr_vliw && stall_iterations < 2);
- was_stall++;
- /* If we are going to stall for too long, break to recompute av
+ pipelining).
+ If we are going to stall for too long, break to recompute av
sets and bring more insns for pipelining. */
+ was_stall++;
if (need_stall <= 3)
stall_for_cycles (fence, need_stall);
else
we still need to count it as an originator. */
bitmap_set_bit (current_originators, INSN_UID (insn));
- if (!bitmap_bit_p (current_copies, INSN_UID (insn)))
+ if (!bitmap_clear_bit (current_copies, INSN_UID (insn)))
{
/* Note that original block needs to be rescheduled, as we pulled an
instruction out of it. */
else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn))
num_insns_scheduled++;
}
- else
- bitmap_clear_bit (current_copies, INSN_UID (insn));
/* For instructions we must immediately remove insn from the
stream, so subsequent update_data_sets () won't include this
moveop_static_params_p params)
{
bool insn_emitted = false;
- rtx cur_reg = expr_dest_reg (params->c_expr);
+ rtx cur_reg;
- gcc_assert (!cur_reg || (params->dest && REG_P (params->dest)));
+ /* Bail out early when expression can not be renamed at all. */
+ if (!EXPR_SEPARABLE_P (params->c_expr))
+ return false;
+
+ cur_reg = expr_dest_reg (params->c_expr);
+ gcc_assert (cur_reg && params->dest && REG_P (params->dest));
/* If original operation has expr and the register chosen for
that expr is not original operation's dest reg, substitute
operation's right hand side with the register chosen. */
- if (cur_reg != NULL_RTX && REGNO (params->dest) != REGNO (cur_reg))
+ if (REGNO (params->dest) != REGNO (cur_reg))
{
insn_t reg_move_insn, reg_move_insn_rtx;
the iterator becomes invalid. We need to try again. */
if (BLOCK_FOR_INSN (insn)->index != old_index
|| EDGE_COUNT (bb->succs) != old_succs)
- goto rescan;
+ {
+ insn = sel_bb_end (BLOCK_FOR_INSN (insn));
+ goto rescan;
+ }
}
#ifdef ENABLE_CHECKING
bookkeeping generated for another fence or for another path in current
move_op. */
gcc_assert (res == 1
- || (res == 0
- && av_set_could_be_blocked_by_bookkeeping_p (orig_ops,
+ || (res == 0
+ && av_set_could_be_blocked_by_bookkeeping_p (orig_ops,
static_params))
- || res == -1);
+ || res == -1);
#endif
/* Merge data, clean up, etc. */
/* Filter the orig_ops set. */
if (AV_SET_VALID_P (insn))
- av_set_intersect (&orig_ops, AV_SET (insn));
+ av_set_code_motion_filter (&orig_ops, AV_SET (insn));
/* If no more original ops, return immediately. */
if (!orig_ops)
if (!expr)
{
int res;
+ rtx last_insn = PREV_INSN (insn);
+ bool added_to_path;
gcc_assert (insn == sel_bb_end (bb));
/* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head -
it's already in PATH then). */
if (insn != first_insn)
- ilist_add (&path, insn);
+ {
+ ilist_add (&path, insn);
+ added_to_path = true;
+ }
+ else
+ added_to_path = false;
/* Process_successors should be able to find at least one
successor for which code_motion_path_driver returns TRUE. */
res = code_motion_process_successors (insn, orig_ops,
path, static_params);
+ /* Jump in the end of basic block could have been removed or replaced
+ during code_motion_process_successors, so recompute insn as the
+ last insn in bb. */
+ if (NEXT_INSN (last_insn) != insn)
+ {
+ insn = sel_bb_end (bb);
+ first_insn = sel_bb_head (bb);
+ }
+
/* Remove bb tail from path. */
- if (insn != first_insn)
+ if (added_to_path)
ilist_remove (&path);
if (res != 1)
{
struct moveop_static_params sparams;
struct cmpd_local_params lparams;
- bool res;
+ int res;
/* Init params for code_motion_path_driver. */
sparams.dest = dest;
code_motion_path_driver_info = &move_op_hooks;
res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
+ gcc_assert (res != -1);
+
if (sparams.was_renamed)
EXPR_WAS_RENAMED (expr_vliw) = true;
init_seqno_1 (succ, visited_bbs, blocks_to_reschedule);
}
+ else if (blocks_to_reschedule)
+ bitmap_set_bit (forced_ebb_heads, succ->index);
}
for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn))
INSN_SEQNO (insn) = cur_seqno--;
}
-/* Initialize seqnos for the current region. NUMBER_OF_INSNS is the number
- of instructions in the region, BLOCKS_TO_RESCHEDULE contains blocks on
- which we're rescheduling when pipelining, FROM is the block where
+/* Initialize seqnos for the current region. BLOCKS_TO_RESCHEDULE contains
+ blocks on which we're rescheduling when pipelining, FROM is the block where
traversing region begins (it may not be the head of the region when
pipelining, but the head of the loop instead).
Returns the maximal seqno found. */
static int
-init_seqno (int number_of_insns, bitmap blocks_to_reschedule, basic_block from)
+init_seqno (bitmap blocks_to_reschedule, basic_block from)
{
sbitmap visited_bbs;
bitmap_iterator bi;
from = EBB_FIRST_BB (0);
}
- cur_seqno = number_of_insns > 0 ? number_of_insns : sched_max_luid - 1;
+ cur_seqno = sched_max_luid - 1;
init_seqno_1 (from, visited_bbs, blocks_to_reschedule);
- gcc_assert (cur_seqno == 0 || number_of_insns == 0);
+
+ /* cur_seqno may be positive if the number of instructions is less than
+ sched_max_luid - 1 (when rescheduling or if some instructions have been
+ removed by the call to purge_empty_blocks in sel_sched_region_1). */
+ gcc_assert (cur_seqno >= 0);
sbitmap_free (visited_bbs);
return sched_max_luid - 1;
bookkeeping_p = 1;
pipelining_p = (bookkeeping_p
&& (flag_sel_sched_pipelining != 0)
- && current_loop_nest != NULL);
+ && current_loop_nest != NULL
+ && loop_has_exit_edges (current_loop_nest));
max_insns_to_rename = PARAM_VALUE (PARAM_SELSCHED_INSNS_TO_RENAME);
max_ws = MAX_WS;
}
/* Prepare and verify loop nest for pipelining. */
static void
-setup_current_loop_nest (int rgn)
+setup_current_loop_nest (int rgn, bb_vec_t *bbs)
{
current_loop_nest = get_loop_nest_for_rgn (rgn);
/* If this loop has any saved loop preheaders from nested loops,
add these basic blocks to the current region. */
- sel_add_loop_preheaders ();
+ sel_add_loop_preheaders (bbs);
/* Check that we're starting with a valid information. */
gcc_assert (loop_latch_edge (current_loop_nest));
gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest));
}
-/* Purge meaningless empty blocks in the middle of a region. */
-static void
-purge_empty_blocks (void)
-{
- /* Do not attempt to delete preheader. */
- int i = sel_is_loop_preheader_p (BASIC_BLOCK (BB_TO_BLOCK (0))) ? 1 : 0;
-
- while (i < current_nr_blocks)
- {
- basic_block b = BASIC_BLOCK (BB_TO_BLOCK (i));
-
- if (maybe_tidy_empty_bb (b))
- continue;
-
- i++;
- }
-}
-
/* Compute instruction priorities for current region. */
static void
sel_compute_priorities (int rgn)
if (current_region_empty_p ())
return true;
- if (flag_sel_sched_pipelining)
- setup_current_loop_nest (rgn);
-
- sel_setup_region_sched_flags ();
-
bbs = VEC_alloc (basic_block, heap, current_nr_blocks);
for (i = 0; i < current_nr_blocks; i++)
VEC_quick_push (basic_block, bbs, BASIC_BLOCK (BB_TO_BLOCK (i)));
- sel_init_bbs (bbs, NULL);
+ sel_init_bbs (bbs);
+
+ if (flag_sel_sched_pipelining)
+ setup_current_loop_nest (rgn, &bbs);
+
+ sel_setup_region_sched_flags ();
/* Initialize luids and dependence analysis which both sel-sched and haifa
need. */
- sched_init_luids (bbs, NULL, NULL, NULL);
+ sched_init_luids (bbs);
sched_deps_init (false);
/* Initialize haifa data. */
rgn_setup_sched_infos ();
sel_set_sched_flags ();
- haifa_init_h_i_d (bbs, NULL, NULL, NULL);
+ haifa_init_h_i_d (bbs);
sel_compute_priorities (rgn);
init_deps_global ();
/* Set hooks so that no newly generated insn will go out unnoticed. */
sel_register_cfg_hooks ();
- /* !!! We call target.sched.md_init () for the whole region, but we invoke
- targetm.sched.md_finish () for every ebb. */
- if (targetm.sched.md_init)
+ /* !!! We call target.sched.init () for the whole region, but we invoke
+ targetm.sched.finish () for every ebb. */
+ if (targetm.sched.init)
/* None of the arguments are actually used in any target. */
- targetm.sched.md_init (sched_dump, sched_verbose, -1);
+ targetm.sched.init (sched_dump, sched_verbose, -1);
first_emitted_uid = get_max_uid () + 1;
preheader_removed = false;
int last_clock = 0;
int haifa_last_clock = -1;
int haifa_clock = 0;
+ int issued_insns = 0;
insn_t insn;
- if (targetm.sched.md_init)
+ if (targetm.sched.init)
{
/* None of the arguments are actually used in any target.
NB: We should have md_reset () hook for cases like this. */
- targetm.sched.md_init (sched_dump, sched_verbose, -1);
+ targetm.sched.init (sched_dump, sched_verbose, -1);
}
state_reset (curr_state);
{
int cost, haifa_cost;
int sort_p;
- bool asm_p, real_insn, after_stall;
+ bool asm_p, real_insn, after_stall, all_issued;
int clock;
if (!INSN_P (insn))
haifa_cost = cost;
after_stall = 1;
}
-
+ all_issued = issued_insns == issue_rate;
+ if (haifa_cost == 0 && all_issued)
+ haifa_cost = 1;
if (haifa_cost > 0)
{
int i = 0;
while (haifa_cost--)
{
advance_state (curr_state);
+ issued_insns = 0;
i++;
if (sched_verbose >= 2)
&& haifa_cost > 0
&& estimate_insn_cost (insn, curr_state) == 0)
break;
- }
+
+ /* When the data dependency stall is longer than the DFA stall,
+ and when we have issued exactly issue_rate insns and stalled,
+ it could be that after this longer stall the insn will again
+ become unavailable to the DFA restrictions. Looks strange
+ but happens e.g. on x86-64. So recheck DFA on the last
+ iteration. */
+ if ((after_stall || all_issued)
+ && real_insn
+ && haifa_cost == 0)
+ haifa_cost = estimate_insn_cost (insn, curr_state);
+ }
haifa_clock += i;
+ if (sched_verbose >= 2)
+ sel_print ("haifa clock: %d\n", haifa_clock);
}
else
gcc_assert (haifa_cost == 0);
&sort_p))
{
advance_state (curr_state);
+ issued_insns = 0;
haifa_clock++;
if (sched_verbose >= 2)
{
sel_print ("advance_state (dfa_new_cycle)\n");
debug_state (curr_state);
+ sel_print ("haifa clock: %d\n", haifa_clock + 1);
}
}
if (real_insn)
{
cost = state_transition (curr_state, insn);
+ issued_insns++;
if (sched_verbose >= 2)
- debug_state (curr_state);
-
+ {
+ sel_print ("scheduled insn %d, clock %d\n", INSN_UID (insn),
+ haifa_clock + 1);
+ debug_state (curr_state);
+ }
gcc_assert (cost < 0);
}
if (reset_sched_cycles_p)
reset_sched_cycles_in_current_ebb ();
- if (targetm.sched.md_init)
- targetm.sched.md_init (sched_dump, sched_verbose, -1);
+ if (targetm.sched.init)
+ targetm.sched.init (sched_dump, sched_verbose, -1);
put_TImodes ();
- if (targetm.sched.md_finish)
+ if (targetm.sched.finish)
{
- targetm.sched.md_finish (sched_dump, sched_verbose);
+ targetm.sched.finish (sched_dump, sched_verbose);
/* Extend luids so that insns generated by the target will
get zero luid. */
- sched_init_luids (NULL, NULL, NULL, NULL);
+ sched_extend_luids ();
}
}
finish_deps_global ();
sched_finish_luids ();
+ VEC_free (haifa_deps_insn_data_def, heap, h_d_i_d);
sel_finish_bbs ();
BITMAP_FREE (blocks_to_reschedule);
static void
sel_sched_region_1 (void)
{
- int number_of_insns;
int orig_max_seqno;
- /* Remove empty blocks that might be in the region from the beginning.
- We need to do save sched_max_luid before that, as it actually shows
- the number of insns in the region, and purge_empty_blocks can
- alter it. */
- number_of_insns = sched_max_luid - 1;
+ /* Remove empty blocks that might be in the region from the beginning. */
purge_empty_blocks ();
- orig_max_seqno = init_seqno (number_of_insns, NULL, NULL);
+ orig_max_seqno = init_seqno (NULL, NULL);
gcc_assert (orig_max_seqno >= 1);
/* When pipelining outer loops, create fences on the loop header,
{
basic_block bb = EBB_FIRST_BB (i);
- if (sel_bb_empty_p (bb))
- {
- bitmap_clear_bit (blocks_to_reschedule, bb->index);
- continue;
- }
-
if (bitmap_bit_p (blocks_to_reschedule, bb->index))
{
+ if (! bb_ends_ebb_p (bb))
+ bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index);
+ if (sel_bb_empty_p (bb))
+ {
+ bitmap_clear_bit (blocks_to_reschedule, bb->index);
+ continue;
+ }
clear_outdated_rtx_info (bb);
if (sel_insn_is_speculation_check (BB_END (bb))
&& JUMP_P (BB_END (bb)))
bitmap_set_bit (blocks_to_reschedule,
BRANCH_EDGE (bb)->dest->index);
}
- else if (INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
+ else if (! sel_bb_empty_p (bb)
+ && INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
bitmap_set_bit (blocks_to_reschedule, bb->index);
}
{
flist_tail_init (new_fences);
- orig_max_seqno = init_seqno (0, blocks_to_reschedule, bb);
+ orig_max_seqno = init_seqno (blocks_to_reschedule, bb);
/* Mark BB as head of the new ebb. */
bitmap_set_bit (forced_ebb_heads, bb->index);
- bitmap_clear_bit (blocks_to_reschedule, bb->index);
-
gcc_assert (fences == NULL);
init_fences (bb_note (bb));
OSDN Git Service
