From 758df2833be13ffcf3ddd2b04d4cc375f28dae0a Mon Sep 17 00:00:00 2001 From: bernds Date: Fri, 30 Sep 2011 15:37:43 +0000 Subject: [PATCH 1/1] * common/config/c6x/c6x-common.c (c6x_option_optimization_table): Enable -fmodulo-sched at -O2 and above. * config/c6x/c6x.md (doloop_end): New expander. (mvilc, sploop, spkernel, loop_end): New patterns. (loop_end with memory destination splitter): New. * config/c6x/c6x.c: Include "hw-doloop.h". (enum unitreqs): New. (unit_req_table): New typedef. (unit_reqs): New static variable. (unit_req_factor, get_unit_reqs, count_unit_reqs, merge_unit_reqs, res_mii, split_delayed_nonbranch, undo_split_delayed_nonbranch, hwloop_pattern_reg, bb_earliest_end_cycle, filter_insns_above, hwloop_optimize, hwloop_fail, c6x_hwloops): New static functions. (struct c6x_sched_context): New member last_scheduled_iter0. (init_sched_state): Initialize it. (c6x_variable_issue): Update it. (sploop_max_uid_iter0): New static variable. (c6x_sched_reorder_1): Be careful about issuing sploop. (c6x_reorg): Call c6x_hwlooops before the final schedule. git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@179393 138bc75d-0d04-0410-961f-82ee72b054a4 --- gcc/ChangeLog | 22 ++ gcc/common/config/c6x/c6x-common.c | 1 + gcc/config/c6x/c6x.c | 782 ++++++++++++++++++++++++++++++++++++- gcc/config/c6x/c6x.md | 100 +++++ 4 files changed, 898 insertions(+), 7 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 3201f3fdaed..485e839129d 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2011-09-30 Bernd Schmidt + + * common/config/c6x/c6x-common.c (c6x_option_optimization_table): + Enable -fmodulo-sched at -O2 and above. + * config/c6x/c6x.md (doloop_end): New expander. + (mvilc, sploop, spkernel, loop_end): New patterns. + (loop_end with memory destination splitter): New. + * config/c6x/c6x.c: Include "hw-doloop.h". + (enum unitreqs): New. + (unit_req_table): New typedef. + (unit_reqs): New static variable. + (unit_req_factor, get_unit_reqs, count_unit_reqs, merge_unit_reqs, + res_mii, split_delayed_nonbranch, undo_split_delayed_nonbranch, + hwloop_pattern_reg, bb_earliest_end_cycle, filter_insns_above, + hwloop_optimize, hwloop_fail, c6x_hwloops): New static functions. + (struct c6x_sched_context): New member last_scheduled_iter0. + (init_sched_state): Initialize it. + (c6x_variable_issue): Update it. + (sploop_max_uid_iter0): New static variable. + (c6x_sched_reorder_1): Be careful about issuing sploop. + (c6x_reorg): Call c6x_hwlooops before the final schedule. + 2011-09-30 Georg-Johann Lay PR target/50566 diff --git a/gcc/common/config/c6x/c6x-common.c b/gcc/common/config/c6x/c6x-common.c index 133dddadbf4..a9c98af2266 100644 --- a/gcc/common/config/c6x/c6x-common.c +++ b/gcc/common/config/c6x/c6x-common.c @@ -33,6 +33,7 @@ static const struct default_options c6x_option_optimization_table[] = { { OPT_LEVELS_1_PLUS, OPT_fomit_frame_pointer, NULL, 1 }, { OPT_LEVELS_1_PLUS, OPT_frename_registers, NULL, 1 }, + { OPT_LEVELS_2_PLUS, OPT_fmodulo_sched, NULL, 1 }, { OPT_LEVELS_ALL, OPT_freciprocal_math, NULL, 1 }, { OPT_LEVELS_NONE, 0, NULL, 0 } }; diff --git a/gcc/config/c6x/c6x.c b/gcc/config/c6x/c6x.c index 98c65b56cc8..57e62d9b98d 100644 --- a/gcc/config/c6x/c6x.c +++ b/gcc/config/c6x/c6x.c @@ -50,6 +50,7 @@ #include "sel-sched.h" #include "debug.h" #include "opts.h" +#include "hw-doloop.h" /* Table of supported architecture variants. */ typedef struct @@ -160,6 +161,27 @@ static int c6x_unit_codes[ARRAY_SIZE (c6x_unit_names)]; #define RESERVATION_S1 2 #define RESERVATION_S2 10 + +/* An enum for the unit requirements we count in the UNIT_REQS table. */ +enum unitreqs +{ + UNIT_REQ_D, + UNIT_REQ_L, + UNIT_REQ_S, + UNIT_REQ_M, + UNIT_REQ_DL, + UNIT_REQ_DS, + UNIT_REQ_LS, + UNIT_REQ_DLS, + UNIT_REQ_T, + UNIT_REQ_X, + UNIT_REQ_MAX +}; + +/* A table used to count unit requirements. Used when computing minimum + iteration intervals. */ +typedef int unit_req_table[2][UNIT_REQ_MAX]; +static unit_req_table unit_reqs; /* Register map for debugging. */ int const dbx_register_map[FIRST_PSEUDO_REGISTER] = @@ -3164,6 +3186,149 @@ assign_reservations (rtx head, rtx end) } } } + +/* Return a factor by which to weight unit imbalances for a reservation + R. */ +static int +unit_req_factor (enum unitreqs r) +{ + switch (r) + { + case UNIT_REQ_D: + case UNIT_REQ_L: + case UNIT_REQ_S: + case UNIT_REQ_M: + case UNIT_REQ_X: + case UNIT_REQ_T: + return 1; + case UNIT_REQ_DL: + case UNIT_REQ_LS: + case UNIT_REQ_DS: + return 2; + case UNIT_REQ_DLS: + return 3; + default: + gcc_unreachable (); + } +} + +/* Examine INSN, and store in REQ1/SIDE1 and REQ2/SIDE2 the unit + requirements. Returns zero if INSN can't be handled, otherwise + either one or two to show how many of the two pairs are in use. + REQ1 is always used, it holds what is normally thought of as the + instructions reservation, e.g. UNIT_REQ_DL. REQ2 is used to either + describe a cross path, or for loads/stores, the T unit. */ +static int +get_unit_reqs (rtx insn, int *req1, int *side1, int *req2, int *side2) +{ + enum attr_units units; + enum attr_cross cross; + int side, req; + + if (!NONDEBUG_INSN_P (insn) || recog_memoized (insn) < 0) + return 0; + units = get_attr_units (insn); + if (units == UNITS_UNKNOWN) + return 0; + side = get_insn_side (insn, units); + cross = get_attr_cross (insn); + + req = (units == UNITS_D ? UNIT_REQ_D + : units == UNITS_D_ADDR ? UNIT_REQ_D + : units == UNITS_DL ? UNIT_REQ_DL + : units == UNITS_DS ? UNIT_REQ_DS + : units == UNITS_L ? UNIT_REQ_L + : units == UNITS_LS ? UNIT_REQ_LS + : units == UNITS_S ? UNIT_REQ_S + : units == UNITS_M ? UNIT_REQ_M + : units == UNITS_DLS ? UNIT_REQ_DLS + : -1); + gcc_assert (req != -1); + *req1 = req; + *side1 = side; + if (units == UNITS_D_ADDR) + { + *req2 = UNIT_REQ_T; + *side2 = side ^ (cross == CROSS_Y ? 1 : 0); + return 2; + } + else if (cross == CROSS_Y) + { + *req2 = UNIT_REQ_X; + *side2 = side; + return 2; + } + return 1; +} + +/* Walk the insns between and including HEAD and TAIL, and mark the + resource requirements in the unit_reqs table. */ +static void +count_unit_reqs (unit_req_table reqs, rtx head, rtx tail) +{ + rtx insn; + + memset (reqs, 0, sizeof (unit_req_table)); + + for (insn = head; insn != NEXT_INSN (tail); insn = NEXT_INSN (insn)) + { + int side1, side2, req1, req2; + + switch (get_unit_reqs (insn, &req1, &side1, &req2, &side2)) + { + case 2: + reqs[side2][req2]++; + /* fall through */ + case 1: + reqs[side1][req1]++; + break; + } + } +} + +/* Update the table REQS by merging more specific unit reservations into + more general ones, i.e. counting (for example) UNIT_REQ_D also in + UNIT_REQ_DL, DS, and DLS. */ +static void +merge_unit_reqs (unit_req_table reqs) +{ + int side; + for (side = 0; side < 2; side++) + { + int d = reqs[side][UNIT_REQ_D]; + int l = reqs[side][UNIT_REQ_L]; + int s = reqs[side][UNIT_REQ_S]; + int dl = reqs[side][UNIT_REQ_DL]; + int ls = reqs[side][UNIT_REQ_LS]; + int ds = reqs[side][UNIT_REQ_DS]; + + reqs[side][UNIT_REQ_DL] += d; + reqs[side][UNIT_REQ_DL] += l; + reqs[side][UNIT_REQ_DS] += d; + reqs[side][UNIT_REQ_DS] += s; + reqs[side][UNIT_REQ_LS] += l; + reqs[side][UNIT_REQ_LS] += s; + reqs[side][UNIT_REQ_DLS] += ds + dl + ls + d + l + s; + } +} + +/* Return the resource-constrained minimum iteration interval given the + data in the REQS table. This must have been processed with + merge_unit_reqs already. */ +static int +res_mii (unit_req_table reqs) +{ + int side, req; + int worst = 1; + for (side = 0; side < 2; side++) + for (req = 0; req < UNIT_REQ_MAX; req++) + { + int factor = unit_req_factor (req); + worst = MAX ((reqs[side][UNIT_REQ_D] + factor - 1) / factor, worst); + } + + return worst; +} /* Backend scheduling state. */ typedef struct c6x_sched_context @@ -3196,13 +3361,15 @@ typedef struct c6x_sched_context /* The following variable value is the last issued insn. */ rtx last_scheduled_insn; + /* The last issued insn that isn't a shadow of another. */ + rtx last_scheduled_iter0; /* The following variable value is DFA state before issuing the first insn in the current clock cycle. We do not use this member of the structure directly; we copy the data in and out of prev_cycle_state. */ state_t prev_cycle_state_ctx; - + int reg_n_accesses[FIRST_PSEUDO_REGISTER]; int reg_n_xaccesses[FIRST_PSEUDO_REGISTER]; int reg_set_in_cycle[FIRST_PSEUDO_REGISTER]; @@ -3223,6 +3390,10 @@ static state_t prev_cycle_state; many accesses of the same register. */ static bool reg_access_stall; +/* The highest insn uid after delayed insns were split, but before loop bodies + were copied by the modulo scheduling code. */ +static int sploop_max_uid_iter0; + /* Look up the jump cycle with index N. For an out-of-bounds N, we return 0, so the caller does not specifically have to test for it. */ static int @@ -3421,6 +3592,7 @@ static void init_sched_state (c6x_sched_context_t sc) { sc->last_scheduled_insn = NULL_RTX; + sc->last_scheduled_iter0 = NULL_RTX; sc->issued_this_cycle = 0; memset (sc->jump_cycles, 0, sizeof sc->jump_cycles); memset (sc->jump_cond, 0, sizeof sc->jump_cond); @@ -3481,7 +3653,7 @@ c6x_clear_sched_context (void *_sc) c6x_sched_context_t sc = (c6x_sched_context_t) _sc; gcc_assert (_sc != NULL); - free (sc->prev_cycle_state_ctx); + free (sc->prev_cycle_state_ctx); } /* Free _SC. */ @@ -3716,7 +3888,7 @@ c6x_sched_reorder_1 (rtx *ready, int *pn_ready, int clock_var) bool is_asm = (icode < 0 && (GET_CODE (PATTERN (insn)) == ASM_INPUT || asm_noperands (PATTERN (insn)) >= 0)); - bool no_parallel = (is_asm + bool no_parallel = (is_asm || icode == CODE_FOR_sploop || (icode >= 0 && get_attr_type (insn) == TYPE_ATOMIC)); @@ -3725,7 +3897,8 @@ c6x_sched_reorder_1 (rtx *ready, int *pn_ready, int clock_var) code always assumes at least 1 cycle, which may be wrong. */ if ((no_parallel && (ss.issued_this_cycle > 0 || clock_var < ss.delays_finished_at)) - || c6x_registers_update (insn)) + || c6x_registers_update (insn) + || (ss.issued_this_cycle > 0 && icode == CODE_FOR_sploop)) { memmove (ready + 1, ready, (insnp - ready) * sizeof (rtx)); *ready = insn; @@ -3924,6 +4097,8 @@ c6x_variable_issue (FILE *dump ATTRIBUTE_UNUSED, rtx insn, int can_issue_more ATTRIBUTE_UNUSED) { ss.last_scheduled_insn = insn; + if (INSN_UID (insn) < sploop_max_uid_iter0 && !JUMP_P (insn)) + ss.last_scheduled_iter0 = insn; if (GET_CODE (PATTERN (insn)) != USE && GET_CODE (PATTERN (insn)) != CLOBBER) ss.issued_this_cycle++; if (insn_info) @@ -4813,6 +4988,117 @@ split_delayed_branch (rtx insn) record_delay_slot_pair (i1, insn, 5, 0); } +/* If INSN is a multi-cycle insn that should be handled properly in + modulo-scheduling, split it into a real insn and a shadow. + Return true if we made a change. + + It is valid for us to fail to split an insn; the caller has to deal + with the possibility. Currently we handle loads and most mpy2 and + mpy4 insns. */ +static bool +split_delayed_nonbranch (rtx insn) +{ + int code = recog_memoized (insn); + enum attr_type type; + rtx i1, newpat, src, dest; + rtx pat = PATTERN (insn); + rtvec rtv; + int delay; + + if (GET_CODE (pat) == COND_EXEC) + pat = COND_EXEC_CODE (pat); + + if (code < 0 || GET_CODE (pat) != SET) + return false; + src = SET_SRC (pat); + dest = SET_DEST (pat); + if (!REG_P (dest)) + return false; + + type = get_attr_type (insn); + if (code >= 0 + && (type == TYPE_LOAD + || type == TYPE_LOADN)) + { + if (!MEM_P (src) + && (GET_CODE (src) != ZERO_EXTEND + || !MEM_P (XEXP (src, 0)))) + return false; + + if (GET_MODE_SIZE (GET_MODE (dest)) > 4 + && (GET_MODE_SIZE (GET_MODE (dest)) != 8 || !TARGET_LDDW)) + return false; + + rtv = gen_rtvec (2, GEN_INT (REGNO (SET_DEST (pat))), + SET_SRC (pat)); + newpat = gen_load_shadow (SET_DEST (pat)); + pat = gen_rtx_UNSPEC (VOIDmode, rtv, UNSPEC_REAL_LOAD); + delay = 4; + } + else if (code >= 0 + && (type == TYPE_MPY2 + || type == TYPE_MPY4)) + { + /* We don't handle floating point multiplies yet. */ + if (GET_MODE (dest) == SFmode) + return false; + + rtv = gen_rtvec (2, GEN_INT (REGNO (SET_DEST (pat))), + SET_SRC (pat)); + newpat = gen_mult_shadow (SET_DEST (pat)); + pat = gen_rtx_UNSPEC (VOIDmode, rtv, UNSPEC_REAL_MULT); + delay = type == TYPE_MPY2 ? 1 : 3; + } + else + return false; + + pat = duplicate_cond (pat, insn); + newpat = duplicate_cond (newpat, insn); + i1 = emit_insn_before (pat, insn); + PATTERN (insn) = newpat; + INSN_CODE (insn) = -1; + recog_memoized (insn); + recog_memoized (i1); + record_delay_slot_pair (i1, insn, delay, 0); + return true; +} + +/* Examine if INSN is the result of splitting a load into a real load and a + shadow, and if so, undo the transformation. */ +static void +undo_split_delayed_nonbranch (rtx insn) +{ + int icode = recog_memoized (insn); + enum attr_type type; + rtx prev_pat, insn_pat, prev; + + if (icode < 0) + return; + type = get_attr_type (insn); + if (type != TYPE_LOAD_SHADOW && type != TYPE_MULT_SHADOW) + return; + prev = PREV_INSN (insn); + prev_pat = PATTERN (prev); + insn_pat = PATTERN (insn); + if (GET_CODE (prev_pat) == COND_EXEC) + { + prev_pat = COND_EXEC_CODE (prev_pat); + insn_pat = COND_EXEC_CODE (insn_pat); + } + + gcc_assert (GET_CODE (prev_pat) == UNSPEC + && ((XINT (prev_pat, 1) == UNSPEC_REAL_LOAD + && type == TYPE_LOAD_SHADOW) + || (XINT (prev_pat, 1) == UNSPEC_REAL_MULT + && type == TYPE_MULT_SHADOW))); + insn_pat = gen_rtx_SET (VOIDmode, SET_DEST (insn_pat), + XVECEXP (prev_pat, 0, 1)); + insn_pat = duplicate_cond (insn_pat, prev); + PATTERN (insn) = insn_pat; + INSN_CODE (insn) = -1; + delete_insn (prev); +} + /* Split every insn (i.e. jumps and calls) which can have delay slots into two parts: the first one is scheduled normally and emits the instruction, while the second one is a shadow insn which shows the side effect taking @@ -4853,6 +5139,481 @@ conditionalize_after_sched (void) } } +/* A callback for the hw-doloop pass. This function examines INSN; if + it is a loop_end pattern we recognize, return the reg rtx for the + loop counter. Otherwise, return NULL_RTX. */ + +static rtx +hwloop_pattern_reg (rtx insn) +{ + rtx pat, reg; + + if (!JUMP_P (insn) || recog_memoized (insn) != CODE_FOR_loop_end) + return NULL_RTX; + + pat = PATTERN (insn); + reg = SET_DEST (XVECEXP (pat, 0, 1)); + if (!REG_P (reg)) + return NULL_RTX; + return reg; +} + +/* Return the number of cycles taken by BB, as computed by scheduling, + including the latencies of all insns with delay slots. IGNORE is + an insn we should ignore in the calculation, usually the final + branch. */ +static int +bb_earliest_end_cycle (basic_block bb, rtx ignore) +{ + int earliest = 0; + rtx insn; + + FOR_BB_INSNS (bb, insn) + { + int cycles, this_clock; + + if (LABEL_P (insn) || NOTE_P (insn) || DEBUG_INSN_P (insn) + || GET_CODE (PATTERN (insn)) == USE + || GET_CODE (PATTERN (insn)) == CLOBBER + || insn == ignore) + continue; + + this_clock = insn_get_clock (insn); + cycles = get_attr_cycles (insn); + + if (earliest < this_clock + cycles) + earliest = this_clock + cycles; + } + return earliest; +} + +/* Examine the insns in BB and remove all which have a uid greater or + equal to MAX_UID. */ +static void +filter_insns_above (basic_block bb, int max_uid) +{ + rtx insn, next; + bool prev_ti = false; + int prev_cycle = -1; + + FOR_BB_INSNS_SAFE (bb, insn, next) + { + int this_cycle; + if (!NONDEBUG_INSN_P (insn)) + continue; + if (insn == BB_END (bb)) + return; + this_cycle = insn_get_clock (insn); + if (prev_ti && this_cycle == prev_cycle) + { + gcc_assert (GET_MODE (insn) != TImode); + PUT_MODE (insn, TImode); + } + prev_ti = false; + if (INSN_UID (insn) >= max_uid) + { + if (GET_MODE (insn) == TImode) + { + prev_ti = true; + prev_cycle = this_cycle; + } + delete_insn (insn); + } + } +} + +/* A callback for the hw-doloop pass. Called to optimize LOOP in a + machine-specific fashion; returns true if successful and false if + the hwloop_fail function should be called. */ + +static bool +hwloop_optimize (hwloop_info loop) +{ + basic_block entry_bb, bb; + rtx seq, insn, prev, entry_after, end_packet; + rtx head_insn, tail_insn, new_insns, last_insn; + int loop_earliest, entry_earliest, entry_end_cycle; + int n_execute_packets; + edge entry_edge; + unsigned ix; + int max_uid_before, delayed_splits; + int i, sp_ii, min_ii, max_ii, max_parallel, n_insns, n_real_insns, stages; + rtx *orig_vec; + rtx *copies; + rtx **insn_copies; + + if (!c6x_flag_modulo_sched || !c6x_flag_schedule_insns2 + || !TARGET_INSNS_64PLUS) + return false; + + if (loop->iter_reg_used || loop->depth > 1) + return false; + if (loop->has_call || loop->has_asm) + return false; + + if (loop->head != loop->tail) + return false; + + gcc_assert (loop->incoming_dest == loop->head); + + entry_edge = NULL; + FOR_EACH_VEC_ELT (edge, loop->incoming, i, entry_edge) + if (entry_edge->flags & EDGE_FALLTHRU) + break; + if (entry_edge == NULL) + return false; + + schedule_ebbs_init (); + schedule_ebb (BB_HEAD (loop->tail), loop->loop_end, true); + schedule_ebbs_finish (); + + bb = loop->head; + loop_earliest = bb_earliest_end_cycle (bb, loop->loop_end) + 1; + + max_uid_before = get_max_uid (); + + /* Split all multi-cycle operations, such as loads. For normal + scheduling, we only do this for branches, as the generated code + would otherwise not be interrupt-safe. When using sploop, it is + safe and beneficial to split them. If any multi-cycle operations + remain after splitting (because we don't handle them yet), we + cannot pipeline the loop. */ + delayed_splits = 0; + FOR_BB_INSNS (bb, insn) + { + if (NONDEBUG_INSN_P (insn)) + { + recog_memoized (insn); + if (split_delayed_nonbranch (insn)) + delayed_splits++; + else if (INSN_CODE (insn) >= 0 + && get_attr_cycles (insn) > 1) + goto undo_splits; + } + } + + /* Count the number of insns as well as the number real insns, and save + the original sequence of insns in case we must restore it later. */ + n_insns = n_real_insns = 0; + FOR_BB_INSNS (bb, insn) + { + n_insns++; + if (NONDEBUG_INSN_P (insn) && insn != loop->loop_end) + n_real_insns++; + } + orig_vec = XNEWVEC (rtx, n_insns); + n_insns = 0; + FOR_BB_INSNS (bb, insn) + orig_vec[n_insns++] = insn; + + /* Count the unit reservations, and compute a minimum II from that + table. */ + count_unit_reqs (unit_reqs, loop->start_label, + PREV_INSN (loop->loop_end)); + merge_unit_reqs (unit_reqs); + + min_ii = res_mii (unit_reqs); + max_ii = loop_earliest < 15 ? loop_earliest : 14; + + /* Make copies of the loop body, up to a maximum number of stages we want + to handle. */ + max_parallel = loop_earliest / min_ii + 1; + + copies = XCNEWVEC (rtx, (max_parallel + 1) * n_real_insns); + insn_copies = XNEWVEC (rtx *, max_parallel + 1); + for (i = 0; i < max_parallel + 1; i++) + insn_copies[i] = copies + i * n_real_insns; + + head_insn = next_nonnote_nondebug_insn (loop->start_label); + tail_insn = prev_real_insn (BB_END (bb)); + + i = 0; + FOR_BB_INSNS (bb, insn) + if (NONDEBUG_INSN_P (insn) && insn != loop->loop_end) + insn_copies[0][i++] = insn; + + sploop_max_uid_iter0 = get_max_uid (); + + /* Generate the copies of the loop body, and save them in the + INSN_COPIES array. */ + start_sequence (); + for (i = 0; i < max_parallel; i++) + { + int j; + rtx this_iter; + + this_iter = duplicate_insn_chain (head_insn, tail_insn); + j = 0; + while (this_iter) + { + rtx prev_stage_insn = insn_copies[i][j]; + gcc_assert (INSN_CODE (this_iter) == INSN_CODE (prev_stage_insn)); + + if (INSN_CODE (this_iter) >= 0 + && (get_attr_type (this_iter) == TYPE_LOAD_SHADOW + || get_attr_type (this_iter) == TYPE_MULT_SHADOW)) + { + rtx prev = PREV_INSN (this_iter); + record_delay_slot_pair (prev, this_iter, + get_attr_cycles (prev) - 1, 0); + } + else + record_delay_slot_pair (prev_stage_insn, this_iter, i, 1); + + insn_copies[i + 1][j] = this_iter; + j++; + this_iter = next_nonnote_nondebug_insn (this_iter); + } + } + new_insns = get_insns (); + last_insn = insn_copies[max_parallel][n_real_insns - 1]; + end_sequence (); + emit_insn_before (new_insns, BB_END (bb)); + + /* Try to schedule the loop using varying initiation intervals, + starting with the smallest possible and incrementing it + on failure. */ + for (sp_ii = min_ii; sp_ii <= max_ii; sp_ii++) + { + basic_block tmp_bb; + if (dump_file) + fprintf (dump_file, "Trying to schedule for II %d\n", sp_ii); + + df_clear_flags (DF_LR_RUN_DCE); + + schedule_ebbs_init (); + set_modulo_params (sp_ii, max_parallel, n_real_insns, + sploop_max_uid_iter0); + tmp_bb = schedule_ebb (BB_HEAD (bb), last_insn, true); + schedule_ebbs_finish (); + + if (tmp_bb) + { + if (dump_file) + fprintf (dump_file, "Found schedule with II %d\n", sp_ii); + break; + } + } + + discard_delay_pairs_above (max_uid_before); + + if (sp_ii > max_ii) + goto restore_loop; + + stages = insn_get_clock (ss.last_scheduled_iter0) / sp_ii + 1; + + if (stages == 1 && sp_ii > 5) + goto restore_loop; + + /* At this point, we know we've been successful, unless we find later that + there are too many execute packets for the loop buffer to hold. */ + + /* Assign reservations to the instructions in the loop. We must find + the stage that contains the full loop kernel, and transfer the + reservations of the instructions contained in it to the corresponding + instructions from iteration 0, which are the only ones we'll keep. */ + assign_reservations (BB_HEAD (bb), ss.last_scheduled_insn); + PREV_INSN (BB_END (bb)) = ss.last_scheduled_iter0; + NEXT_INSN (ss.last_scheduled_iter0) = BB_END (bb); + filter_insns_above (bb, sploop_max_uid_iter0); + + for (i = 0; i < n_real_insns; i++) + { + rtx insn = insn_copies[0][i]; + int uid = INSN_UID (insn); + int stage = insn_uid_get_clock (uid) / sp_ii; + + if (stage + 1 < stages) + { + int copy_uid; + stage = stages - stage - 1; + copy_uid = INSN_UID (insn_copies[stage][i]); + INSN_INFO_ENTRY (uid).reservation + = INSN_INFO_ENTRY (copy_uid).reservation; + } + } + if (stages == 1) + stages++; + + /* Compute the number of execute packets the pipelined form of the loop will + require. */ + prev = NULL_RTX; + n_execute_packets = 0; + for (insn = loop->start_label; insn != loop->loop_end; insn = NEXT_INSN (insn)) + { + if (NONDEBUG_INSN_P (insn) && GET_MODE (insn) == TImode + && !shadow_p (insn)) + { + n_execute_packets++; + if (prev && insn_get_clock (prev) + 1 != insn_get_clock (insn)) + /* We need an extra NOP instruction. */ + n_execute_packets++; + + prev = insn; + } + } + + end_packet = ss.last_scheduled_iter0; + while (!NONDEBUG_INSN_P (end_packet) || GET_MODE (end_packet) != TImode) + end_packet = PREV_INSN (end_packet); + + /* The earliest cycle in which we can emit the SPKERNEL instruction. */ + loop_earliest = (stages - 1) * sp_ii; + if (loop_earliest > insn_get_clock (end_packet)) + { + n_execute_packets++; + end_packet = loop->loop_end; + } + else + loop_earliest = insn_get_clock (end_packet); + + if (n_execute_packets > 14) + goto restore_loop; + + /* Generate the spkernel instruction, and place it at the appropriate + spot. */ + PUT_MODE (end_packet, VOIDmode); + + insn = gen_spkernel (GEN_INT (stages - 1), + const0_rtx, JUMP_LABEL (loop->loop_end)); + insn = emit_jump_insn_before (insn, end_packet); + JUMP_LABEL (insn) = JUMP_LABEL (loop->loop_end); + insn_set_clock (insn, loop_earliest); + PUT_MODE (insn, TImode); + INSN_INFO_ENTRY (INSN_UID (insn)).ebb_start = false; + delete_insn (loop->loop_end); + + /* Place the mvc and sploop instructions before the loop. */ + entry_bb = entry_edge->src; + + start_sequence (); + + insn = emit_insn (gen_mvilc (loop->iter_reg)); + insn = emit_insn (gen_sploop (GEN_INT (sp_ii))); + + seq = get_insns (); + + if (!single_succ_p (entry_bb) || VEC_length (edge, loop->incoming) > 1) + { + basic_block new_bb; + edge e; + edge_iterator ei; + + emit_insn_before (seq, BB_HEAD (loop->head)); + seq = emit_label_before (gen_label_rtx (), seq); + + new_bb = create_basic_block (seq, insn, entry_bb); + FOR_EACH_EDGE (e, ei, loop->incoming) + { + if (!(e->flags & EDGE_FALLTHRU)) + redirect_edge_and_branch_force (e, new_bb); + else + redirect_edge_succ (e, new_bb); + } + make_edge (new_bb, loop->head, 0); + } + else + { + entry_after = BB_END (entry_bb); + while (DEBUG_INSN_P (entry_after) + || (NOTE_P (entry_after) + && NOTE_KIND (entry_after) != NOTE_INSN_BASIC_BLOCK)) + entry_after = PREV_INSN (entry_after); + emit_insn_after (seq, entry_after); + } + + end_sequence (); + + /* Make sure we don't try to schedule this loop again. */ + for (ix = 0; VEC_iterate (basic_block, loop->blocks, ix, bb); ix++) + bb->flags |= BB_DISABLE_SCHEDULE; + + return true; + + restore_loop: + if (dump_file) + fprintf (dump_file, "Unable to pipeline loop.\n"); + + for (i = 1; i < n_insns; i++) + { + NEXT_INSN (orig_vec[i - 1]) = orig_vec[i]; + PREV_INSN (orig_vec[i]) = orig_vec[i - 1]; + } + PREV_INSN (orig_vec[0]) = PREV_INSN (BB_HEAD (bb)); + NEXT_INSN (PREV_INSN (BB_HEAD (bb))) = orig_vec[0]; + NEXT_INSN (orig_vec[n_insns - 1]) = NEXT_INSN (BB_END (bb)); + PREV_INSN (NEXT_INSN (BB_END (bb))) = orig_vec[n_insns - 1]; + BB_HEAD (bb) = orig_vec[0]; + BB_END (bb) = orig_vec[n_insns - 1]; + undo_splits: + free_delay_pairs (); + FOR_BB_INSNS (bb, insn) + if (NONDEBUG_INSN_P (insn)) + undo_split_delayed_nonbranch (insn); + return false; +} + +/* A callback for the hw-doloop pass. Called when a loop we have discovered + turns out not to be optimizable; we have to split the doloop_end pattern + into a subtract and a test. */ +static void +hwloop_fail (hwloop_info loop) +{ + rtx insn, test, testreg; + + if (dump_file) + fprintf (dump_file, "splitting doloop insn %d\n", + INSN_UID (loop->loop_end)); + insn = gen_addsi3 (loop->iter_reg, loop->iter_reg, constm1_rtx); + /* See if we can emit the add at the head of the loop rather than at the + end. */ + if (loop->head == NULL + || loop->iter_reg_used_outside + || loop->iter_reg_used + || TEST_HARD_REG_BIT (loop->regs_set_in_loop, REGNO (loop->iter_reg)) + || loop->incoming_dest != loop->head + || EDGE_COUNT (loop->head->preds) != 2) + emit_insn_before (insn, loop->loop_end); + else + { + rtx t = loop->start_label; + while (!NOTE_P (t) || NOTE_KIND (t) != NOTE_INSN_BASIC_BLOCK) + t = NEXT_INSN (t); + emit_insn_after (insn, t); + } + + testreg = SET_DEST (XVECEXP (PATTERN (loop->loop_end), 0, 2)); + if (GET_CODE (testreg) == SCRATCH) + testreg = loop->iter_reg; + else + emit_insn_before (gen_movsi (testreg, loop->iter_reg), loop->loop_end); + + test = gen_rtx_NE (VOIDmode, testreg, const0_rtx); + insn = emit_jump_insn_before (gen_cbranchsi4 (test, testreg, const0_rtx, + loop->start_label), + loop->loop_end); + + JUMP_LABEL (insn) = loop->start_label; + LABEL_NUSES (loop->start_label)++; + delete_insn (loop->loop_end); +} + +static struct hw_doloop_hooks c6x_doloop_hooks = +{ + hwloop_pattern_reg, + hwloop_optimize, + hwloop_fail +}; + +/* Run the hw-doloop pass to modulo-schedule hardware loops, or split the + doloop_end patterns where such optimizations are impossible. */ +static void +c6x_hwloops (void) +{ + if (optimize) + reorg_loops (true, &c6x_doloop_hooks); +} + /* Implement the TARGET_MACHINE_DEPENDENT_REORG pass. We split call insns here into a sequence that loads the return register and performs the call, and emit the return label. @@ -4881,10 +5642,17 @@ c6x_reorg (void) int sz = get_max_uid () * 3 / 2 + 1; insn_info = VEC_alloc (c6x_sched_insn_info, heap, sz); + } + + /* Make sure the real-jump insns we create are not deleted. When modulo- + scheduling, situations where a reg is only stored in a loop can also + cause dead code when doing the initial unrolling. */ + sched_no_dce = true; - /* Make sure the real-jump insns we create are not deleted. */ - sched_no_dce = true; + c6x_hwloops (); + if (c6x_flag_schedule_insns2) + { split_delayed_insns (); timevar_push (TV_SCHED2); if (do_selsched) @@ -4895,8 +5663,8 @@ c6x_reorg (void) timevar_pop (TV_SCHED2); free_delay_pairs (); - sched_no_dce = false; } + sched_no_dce = false; call_labels = XCNEWVEC (rtx, get_max_uid () + 1); diff --git a/gcc/config/c6x/c6x.md b/gcc/config/c6x/c6x.md index c2502ca1a42..c2196ac6fe2 100644 --- a/gcc/config/c6x/c6x.md +++ b/gcc/config/c6x/c6x.md @@ -1391,6 +1391,106 @@ ) ;; ------------------------------------------------------------------------- +;; Doloop +;; ------------------------------------------------------------------------- + +; operand 0 is the loop count pseudo register +; operand 1 is the number of loop iterations or 0 if it is unknown +; operand 2 is the maximum number of loop iterations +; operand 3 is the number of levels of enclosed loops +; operand 4 is the label to jump to at the top of the loop +(define_expand "doloop_end" + [(parallel [(set (pc) (if_then_else + (ne (match_operand:SI 0 "" "") + (const_int 1)) + (label_ref (match_operand 4 "" "")) + (pc))) + (set (match_dup 0) + (plus:SI (match_dup 0) + (const_int -1))) + (clobber (match_scratch:SI 5 ""))])] + "TARGET_INSNS_64PLUS && optimize" +{ + /* The loop optimizer doesn't check the predicates... */ + if (GET_MODE (operands[0]) != SImode) + FAIL; +}) + +(define_insn "mvilc" + [(set (reg:SI REG_ILC) + (unspec [(match_operand:SI 0 "register_operand" "a,b")] UNSPEC_MVILC))] + "TARGET_INSNS_64PLUS" + "%|%.\\tmvc\\t%$\\t%0, ILC" + [(set_attr "predicable" "no") + (set_attr "cross" "y,n") + (set_attr "units" "s") + (set_attr "dest_regfile" "b") + (set_attr "type" "mvilc")]) + +(define_insn "sploop" + [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "i") + (reg:SI REG_ILC)] + UNSPECV_SPLOOP)] + "TARGET_INSNS_64PLUS" + "%|%.\\tsploop\t%0" + [(set_attr "predicable" "no") + (set_attr "type" "sploop")]) + +(define_insn "spkernel" + [(set (pc) + (if_then_else + (ne (unspec_volatile:SI + [(match_operand:SI 0 "const_int_operand" "i") + (match_operand:SI 1 "const_int_operand" "i")] + UNSPECV_SPKERNEL) + (const_int 1)) + (label_ref (match_operand 2 "" "")) + (pc)))] + "TARGET_INSNS_64PLUS" + "%|%.\\tspkernel\t%0, %1" + [(set_attr "predicable" "no") + (set_attr "type" "spkernel")]) + +(define_insn "loop_end" + [(set (pc) + (if_then_else (ne (match_operand:SI 3 "nonimmediate_operand" "0,0,0,*r") + (const_int 1)) + (label_ref (match_operand 1 "" "")) + (pc))) + (set (match_operand:SI 0 "nonimmediate_operand" "=AB,*r,m,m") + (plus:SI (match_dup 3) + (const_int -1))) + (clobber (match_scratch:SI 2 "=X,&AB,&AB,&AB"))] + "TARGET_INSNS_64PLUS && optimize" + "#" + [(set_attr "type" "spkernel")]) + +(define_split + [(set (pc) + (if_then_else (ne (match_operand:SI 3 "nonimmediate_operand" "") + (const_int 1)) + (label_ref (match_operand 1 "" "")) + (pc))) + (set (match_operand:SI 0 "memory_operand" "") + (plus:SI (match_dup 3) + (const_int -1))) + (clobber (match_scratch 2))] + "" + [(set (match_dup 2) (plus:SI (match_dup 3) (const_int -1))) + (set (match_dup 0) (match_dup 2)) + (set (pc) + (if_then_else (ne (match_dup 2) (const_int 0)) + (label_ref (match_dup 1)) + (pc)))] +{ + if (!REG_P (operands[3])) + { + emit_move_insn (operands[2], operands[3]); + operands[3] = operands[2]; + } +}) + +;; ------------------------------------------------------------------------- ;; Delayed-branch real jumps and shadows ;; ------------------------------------------------------------------------- -- 2.11.0