+/* Implement targetm.sched.h_i_d_extended hook.
+ Extend internal data structures. */
+static void
+ia64_h_i_d_extended (void)
+{
+ if (current_sched_info->flags & DO_SPECULATION)
+ {
+ int new_max_uid = get_max_uid () + 1;
+
+ spec_check_no = xrecalloc (spec_check_no, new_max_uid,
+ max_uid, sizeof (*spec_check_no));
+ max_uid = new_max_uid;
+ }
+
+ if (stops_p != NULL)
+ {
+ int new_clocks_length = get_max_uid () + 1;
+
+ stops_p = xrecalloc (stops_p, new_clocks_length, clocks_length, 1);
+
+ if (ia64_tune == PROCESSOR_ITANIUM)
+ {
+ clocks = xrecalloc (clocks, new_clocks_length, clocks_length,
+ sizeof (int));
+ add_cycles = xrecalloc (add_cycles, new_clocks_length, clocks_length,
+ sizeof (int));
+ }
+
+ clocks_length = new_clocks_length;
+ }
+}
+
+/* Constants that help mapping 'enum machine_mode' to int. */
+enum SPEC_MODES
+ {
+ SPEC_MODE_INVALID = -1,
+ SPEC_MODE_FIRST = 0,
+ SPEC_MODE_FOR_EXTEND_FIRST = 1,
+ SPEC_MODE_FOR_EXTEND_LAST = 3,
+ SPEC_MODE_LAST = 8
+ };
+
+/* Return index of the MODE. */
+static int
+ia64_mode_to_int (enum machine_mode mode)
+{
+ switch (mode)
+ {
+ case BImode: return 0; /* SPEC_MODE_FIRST */
+ case QImode: return 1; /* SPEC_MODE_FOR_EXTEND_FIRST */
+ case HImode: return 2;
+ case SImode: return 3; /* SPEC_MODE_FOR_EXTEND_LAST */
+ case DImode: return 4;
+ case SFmode: return 5;
+ case DFmode: return 6;
+ case XFmode: return 7;
+ case TImode:
+ /* ??? This mode needs testing. Bypasses for ldfp8 instruction are not
+ mentioned in itanium[12].md. Predicate fp_register_operand also
+ needs to be defined. Bottom line: better disable for now. */
+ return SPEC_MODE_INVALID;
+ default: return SPEC_MODE_INVALID;
+ }
+}
+
+/* Provide information about speculation capabilities. */
+static void
+ia64_set_sched_flags (spec_info_t spec_info)
+{
+ unsigned int *flags = &(current_sched_info->flags);
+
+ if (*flags & SCHED_RGN
+ || *flags & SCHED_EBB)
+ {
+ int mask = 0;
+
+ if ((mflag_sched_br_data_spec && !reload_completed && optimize > 0)
+ || (mflag_sched_ar_data_spec && reload_completed))
+ {
+ mask |= BEGIN_DATA;
+
+ if ((mflag_sched_br_in_data_spec && !reload_completed)
+ || (mflag_sched_ar_in_data_spec && reload_completed))
+ mask |= BE_IN_DATA;
+ }
+
+ if (mflag_sched_control_spec)
+ {
+ mask |= BEGIN_CONTROL;
+
+ if (mflag_sched_in_control_spec)
+ mask |= BE_IN_CONTROL;
+ }
+
+ gcc_assert (*flags & USE_GLAT);
+
+ if (mask)
+ {
+ *flags |= USE_DEPS_LIST | DETACH_LIFE_INFO | DO_SPECULATION;
+
+ spec_info->mask = mask;
+ spec_info->flags = 0;
+
+ if ((mask & DATA_SPEC) && mflag_sched_prefer_non_data_spec_insns)
+ spec_info->flags |= PREFER_NON_DATA_SPEC;
+
+ if ((mask & CONTROL_SPEC)
+ && mflag_sched_prefer_non_control_spec_insns)
+ spec_info->flags |= PREFER_NON_CONTROL_SPEC;
+
+ if (mflag_sched_spec_verbose)
+ {
+ if (sched_verbose >= 1)
+ spec_info->dump = sched_dump;
+ else
+ spec_info->dump = stderr;
+ }
+ else
+ spec_info->dump = 0;
+
+ if (mflag_sched_count_spec_in_critical_path)
+ spec_info->flags |= COUNT_SPEC_IN_CRITICAL_PATH;
+ }
+ }
+}
+
+/* Implement targetm.sched.speculate_insn hook.
+ Check if the INSN can be TS speculative.
+ If 'no' - return -1.
+ If 'yes' - generate speculative pattern in the NEW_PAT and return 1.
+ If current pattern of the INSN already provides TS speculation, return 0. */
+static int
+ia64_speculate_insn (rtx insn, ds_t ts, rtx *new_pat)
+{
+ rtx pat, reg, mem, mem_reg;
+ int mode_no, gen_p = 1;
+ bool extend_p;
+
+ gcc_assert (!(ts & ~BEGIN_SPEC) && ts);
+
+ pat = PATTERN (insn);
+
+ if (GET_CODE (pat) == COND_EXEC)
+ pat = COND_EXEC_CODE (pat);
+
+ if (GET_CODE (pat) != SET)
+ return -1;
+ reg = SET_DEST (pat);
+ if (!REG_P (reg))
+ return -1;
+
+ mem = SET_SRC (pat);
+ if (GET_CODE (mem) == ZERO_EXTEND)
+ {
+ mem = XEXP (mem, 0);
+ extend_p = true;
+ }
+ else
+ extend_p = false;
+
+ if (GET_CODE (mem) == UNSPEC)
+ {
+ int code;
+
+ code = XINT (mem, 1);
+ if (code != UNSPEC_LDA && code != UNSPEC_LDS && code != UNSPEC_LDSA)
+ return -1;
+
+ if ((code == UNSPEC_LDA && !(ts & BEGIN_CONTROL))
+ || (code == UNSPEC_LDS && !(ts & BEGIN_DATA))
+ || code == UNSPEC_LDSA)
+ gen_p = 0;
+
+ mem = XVECEXP (mem, 0, 0);
+ gcc_assert (MEM_P (mem));
+ }
+ if (!MEM_P (mem))
+ return -1;
+ mem_reg = XEXP (mem, 0);
+ if (!REG_P (mem_reg))
+ return -1;
+
+ /* We should use MEM's mode since REG's mode in presence of ZERO_EXTEND
+ will always be DImode. */
+ mode_no = ia64_mode_to_int (GET_MODE (mem));
+
+ if (mode_no == SPEC_MODE_INVALID
+ || (extend_p
+ && !(SPEC_MODE_FOR_EXTEND_FIRST <= mode_no
+ && mode_no <= SPEC_MODE_FOR_EXTEND_LAST)))
+ return -1;
+
+ extract_insn_cached (insn);
+ gcc_assert (reg == recog_data.operand[0] && mem == recog_data.operand[1]);
+ *new_pat = ia64_gen_spec_insn (insn, ts, mode_no, gen_p != 0, extend_p);
+
+ return gen_p;
+}
+
+enum
+ {
+ /* Offset to reach ZERO_EXTEND patterns. */
+ SPEC_GEN_EXTEND_OFFSET = SPEC_MODE_LAST - SPEC_MODE_FOR_EXTEND_FIRST + 1,
+ /* Number of patterns for each speculation mode. */
+ SPEC_N = (SPEC_MODE_LAST
+ + SPEC_MODE_FOR_EXTEND_LAST - SPEC_MODE_FOR_EXTEND_FIRST + 2)
+ };
+
+enum SPEC_GEN_LD_MAP
+ {
+ /* Offset to ld.a patterns. */
+ SPEC_GEN_A = 0 * SPEC_N,
+ /* Offset to ld.s patterns. */
+ SPEC_GEN_S = 1 * SPEC_N,
+ /* Offset to ld.sa patterns. */
+ SPEC_GEN_SA = 2 * SPEC_N,
+ /* Offset to ld.sa patterns. For this patterns corresponding ld.c will
+ mutate to chk.s. */
+ SPEC_GEN_SA_FOR_S = 3 * SPEC_N
+ };
+
+/* These offsets are used to get (4 * SPEC_N). */
+enum SPEC_GEN_CHECK_OFFSET
+ {
+ SPEC_GEN_CHKA_FOR_A_OFFSET = 4 * SPEC_N - SPEC_GEN_A,
+ SPEC_GEN_CHKA_FOR_SA_OFFSET = 4 * SPEC_N - SPEC_GEN_SA
+ };
+
+/* If GEN_P is true, calculate the index of needed speculation check and return
+ speculative pattern for INSN with speculative mode TS, machine mode
+ MODE_NO and with ZERO_EXTEND (if EXTEND_P is true).
+ If GEN_P is false, just calculate the index of needed speculation check. */
+static rtx
+ia64_gen_spec_insn (rtx insn, ds_t ts, int mode_no, bool gen_p, bool extend_p)
+{
+ rtx pat, new_pat;
+ int load_no;
+ int shift = 0;
+
+ static rtx (* const gen_load[]) (rtx, rtx) = {
+ gen_movbi_advanced,
+ gen_movqi_advanced,
+ gen_movhi_advanced,
+ gen_movsi_advanced,
+ gen_movdi_advanced,
+ gen_movsf_advanced,
+ gen_movdf_advanced,
+ gen_movxf_advanced,
+ gen_movti_advanced,
+ gen_zero_extendqidi2_advanced,
+ gen_zero_extendhidi2_advanced,
+ gen_zero_extendsidi2_advanced,
+
+ gen_movbi_speculative,
+ gen_movqi_speculative,
+ gen_movhi_speculative,
+ gen_movsi_speculative,
+ gen_movdi_speculative,
+ gen_movsf_speculative,
+ gen_movdf_speculative,
+ gen_movxf_speculative,
+ gen_movti_speculative,
+ gen_zero_extendqidi2_speculative,
+ gen_zero_extendhidi2_speculative,
+ gen_zero_extendsidi2_speculative,
+
+ gen_movbi_speculative_advanced,
+ gen_movqi_speculative_advanced,
+ gen_movhi_speculative_advanced,
+ gen_movsi_speculative_advanced,
+ gen_movdi_speculative_advanced,
+ gen_movsf_speculative_advanced,
+ gen_movdf_speculative_advanced,
+ gen_movxf_speculative_advanced,
+ gen_movti_speculative_advanced,
+ gen_zero_extendqidi2_speculative_advanced,
+ gen_zero_extendhidi2_speculative_advanced,
+ gen_zero_extendsidi2_speculative_advanced,
+
+ gen_movbi_speculative_advanced,
+ gen_movqi_speculative_advanced,
+ gen_movhi_speculative_advanced,
+ gen_movsi_speculative_advanced,
+ gen_movdi_speculative_advanced,
+ gen_movsf_speculative_advanced,
+ gen_movdf_speculative_advanced,
+ gen_movxf_speculative_advanced,
+ gen_movti_speculative_advanced,
+ gen_zero_extendqidi2_speculative_advanced,
+ gen_zero_extendhidi2_speculative_advanced,
+ gen_zero_extendsidi2_speculative_advanced
+ };
+
+ load_no = extend_p ? mode_no + SPEC_GEN_EXTEND_OFFSET : mode_no;
+
+ if (ts & BEGIN_DATA)
+ {
+ /* We don't need recovery because even if this is ld.sa
+ ALAT entry will be allocated only if NAT bit is set to zero.
+ So it is enough to use ld.c here. */
+
+ if (ts & BEGIN_CONTROL)
+ {
+ load_no += SPEC_GEN_SA;
+
+ if (!mflag_sched_ldc)
+ shift = SPEC_GEN_CHKA_FOR_SA_OFFSET;
+ }
+ else
+ {
+ load_no += SPEC_GEN_A;
+
+ if (!mflag_sched_ldc)
+ shift = SPEC_GEN_CHKA_FOR_A_OFFSET;
+ }
+ }
+ else if (ts & BEGIN_CONTROL)
+ {
+ /* ld.sa can be used instead of ld.s to avoid basic block splitting. */
+ if (!mflag_control_ldc)
+ load_no += SPEC_GEN_S;
+ else
+ {
+ gcc_assert (mflag_sched_ldc);
+ load_no += SPEC_GEN_SA_FOR_S;
+ }
+ }
+ else
+ gcc_unreachable ();
+
+ /* Set the desired check index. We add '1', because zero element in this
+ array means, that instruction with such uid is non-speculative. */
+ spec_check_no[INSN_UID (insn)] = load_no + shift + 1;
+
+ if (!gen_p)
+ return 0;
+
+ new_pat = gen_load[load_no] (copy_rtx (recog_data.operand[0]),
+ copy_rtx (recog_data.operand[1]));
+
+ pat = PATTERN (insn);
+ if (GET_CODE (pat) == COND_EXEC)
+ new_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx
+ (COND_EXEC_TEST (pat)), new_pat);
+
+ return new_pat;
+}
+
+/* Offset to branchy checks. */
+enum { SPEC_GEN_CHECK_MUTATION_OFFSET = 5 * SPEC_N };
+
+/* Return nonzero, if INSN needs branchy recovery check. */
+static bool
+ia64_needs_block_p (rtx insn)
+{
+ int check_no;
+
+ check_no = spec_check_no[INSN_UID(insn)] - 1;
+ gcc_assert (0 <= check_no && check_no < SPEC_GEN_CHECK_MUTATION_OFFSET);
+
+ return ((SPEC_GEN_S <= check_no && check_no < SPEC_GEN_S + SPEC_N)
+ || (4 * SPEC_N <= check_no && check_no < 4 * SPEC_N + SPEC_N));
+}
+
+/* Generate (or regenerate, if (MUTATE_P)) recovery check for INSN.
+ If (LABEL != 0 || MUTATE_P), generate branchy recovery check.
+ Otherwise, generate a simple check. */
+static rtx
+ia64_gen_check (rtx insn, rtx label, bool mutate_p)
+{
+ rtx op1, pat, check_pat;
+
+ static rtx (* const gen_check[]) (rtx, rtx) = {
+ gen_movbi_clr,
+ gen_movqi_clr,
+ gen_movhi_clr,
+ gen_movsi_clr,
+ gen_movdi_clr,
+ gen_movsf_clr,
+ gen_movdf_clr,
+ gen_movxf_clr,
+ gen_movti_clr,
+ gen_zero_extendqidi2_clr,
+ gen_zero_extendhidi2_clr,
+ gen_zero_extendsidi2_clr,
+
+ gen_speculation_check_bi,
+ gen_speculation_check_qi,
+ gen_speculation_check_hi,
+ gen_speculation_check_si,
+ gen_speculation_check_di,
+ gen_speculation_check_sf,
+ gen_speculation_check_df,
+ gen_speculation_check_xf,
+ gen_speculation_check_ti,
+ gen_speculation_check_di,
+ gen_speculation_check_di,
+ gen_speculation_check_di,
+
+ gen_movbi_clr,
+ gen_movqi_clr,
+ gen_movhi_clr,
+ gen_movsi_clr,
+ gen_movdi_clr,
+ gen_movsf_clr,
+ gen_movdf_clr,
+ gen_movxf_clr,
+ gen_movti_clr,
+ gen_zero_extendqidi2_clr,
+ gen_zero_extendhidi2_clr,
+ gen_zero_extendsidi2_clr,
+
+ gen_movbi_clr,
+ gen_movqi_clr,
+ gen_movhi_clr,
+ gen_movsi_clr,
+ gen_movdi_clr,
+ gen_movsf_clr,
+ gen_movdf_clr,
+ gen_movxf_clr,
+ gen_movti_clr,
+ gen_zero_extendqidi2_clr,
+ gen_zero_extendhidi2_clr,
+ gen_zero_extendsidi2_clr,
+
+ gen_advanced_load_check_clr_bi,
+ gen_advanced_load_check_clr_qi,
+ gen_advanced_load_check_clr_hi,
+ gen_advanced_load_check_clr_si,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_sf,
+ gen_advanced_load_check_clr_df,
+ gen_advanced_load_check_clr_xf,
+ gen_advanced_load_check_clr_ti,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+
+ /* Following checks are generated during mutation. */
+ gen_advanced_load_check_clr_bi,
+ gen_advanced_load_check_clr_qi,
+ gen_advanced_load_check_clr_hi,
+ gen_advanced_load_check_clr_si,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_sf,
+ gen_advanced_load_check_clr_df,
+ gen_advanced_load_check_clr_xf,
+ gen_advanced_load_check_clr_ti,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+
+ 0,0,0,0,0,0,0,0,0,0,0,0,
+
+ gen_advanced_load_check_clr_bi,
+ gen_advanced_load_check_clr_qi,
+ gen_advanced_load_check_clr_hi,
+ gen_advanced_load_check_clr_si,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_sf,
+ gen_advanced_load_check_clr_df,
+ gen_advanced_load_check_clr_xf,
+ gen_advanced_load_check_clr_ti,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+ gen_advanced_load_check_clr_di,
+
+ gen_speculation_check_bi,
+ gen_speculation_check_qi,
+ gen_speculation_check_hi,
+ gen_speculation_check_si,
+ gen_speculation_check_di,
+ gen_speculation_check_sf,
+ gen_speculation_check_df,
+ gen_speculation_check_xf,
+ gen_speculation_check_ti,
+ gen_speculation_check_di,
+ gen_speculation_check_di,
+ gen_speculation_check_di
+ };
+
+ extract_insn_cached (insn);
+
+ if (label)
+ {
+ gcc_assert (mutate_p || ia64_needs_block_p (insn));
+ op1 = label;
+ }
+ else
+ {
+ gcc_assert (!mutate_p && !ia64_needs_block_p (insn));
+ op1 = copy_rtx (recog_data.operand[1]);
+ }
+
+ if (mutate_p)
+ /* INSN is ld.c.
+ Find the speculation check number by searching for original
+ speculative load in the RESOLVED_DEPS list of INSN.
+ As long as patterns are unique for each instruction, this can be
+ accomplished by matching ORIG_PAT fields. */
+ {
+ rtx link;
+ int check_no = 0;
+ rtx orig_pat = ORIG_PAT (insn);
+
+ for (link = RESOLVED_DEPS (insn); link; link = XEXP (link, 1))
+ {
+ rtx x = XEXP (link, 0);
+
+ if (ORIG_PAT (x) == orig_pat)
+ check_no = spec_check_no[INSN_UID (x)];
+ }
+ gcc_assert (check_no);
+
+ spec_check_no[INSN_UID (insn)] = (check_no
+ + SPEC_GEN_CHECK_MUTATION_OFFSET);
+ }
+
+ check_pat = (gen_check[spec_check_no[INSN_UID (insn)] - 1]
+ (copy_rtx (recog_data.operand[0]), op1));
+
+ pat = PATTERN (insn);
+ if (GET_CODE (pat) == COND_EXEC)
+ check_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (COND_EXEC_TEST (pat)),
+ check_pat);
+
+ return check_pat;
+}
+
+/* Return nonzero, if X is branchy recovery check. */
+static int
+ia64_spec_check_p (rtx x)
+{
+ x = PATTERN (x);
+ if (GET_CODE (x) == COND_EXEC)
+ x = COND_EXEC_CODE (x);
+ if (GET_CODE (x) == SET)
+ return ia64_spec_check_src_p (SET_SRC (x));
+ return 0;
+}
+
+/* Return nonzero, if SRC belongs to recovery check. */
+static int
+ia64_spec_check_src_p (rtx src)
+{
+ if (GET_CODE (src) == IF_THEN_ELSE)
+ {
+ rtx t;
+
+ t = XEXP (src, 0);
+ if (GET_CODE (t) == NE)
+ {
+ t = XEXP (t, 0);
+
+ if (GET_CODE (t) == UNSPEC)
+ {
+ int code;
+
+ code = XINT (t, 1);
+
+ if (code == UNSPEC_CHKACLR
+ || code == UNSPEC_CHKS
+ || code == UNSPEC_LDCCLR)
+ {
+ gcc_assert (code != 0);
+ return code;
+ }
+ }
+ }
+ }
+ return 0;
+}