1 /* Perform various loop optimizations, including strength reduction.
2 Copyright (C) 1987, 1988, 1989, 1991, 1992, 1993, 1994, 1995, 1996, 1997,
3 1998, 1999, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
22 /* This is the loop optimization pass of the compiler.
23 It finds invariant computations within loops and moves them
24 to the beginning of the loop. Then it identifies basic and
25 general induction variables.
27 Basic induction variables (BIVs) are a pseudo registers which are set within
28 a loop only by incrementing or decrementing its value. General induction
29 variables (GIVs) are pseudo registers with a value which is a linear function
30 of a basic induction variable. BIVs are recognized by `basic_induction_var';
31 GIVs by `general_induction_var'.
33 Once induction variables are identified, strength reduction is applied to the
34 general induction variables, and induction variable elimination is applied to
35 the basic induction variables.
37 It also finds cases where
38 a register is set within the loop by zero-extending a narrower value
39 and changes these to zero the entire register once before the loop
40 and merely copy the low part within the loop.
42 Most of the complexity is in heuristics to decide when it is worth
43 while to do these things. */
47 #include "coretypes.h"
53 #include "hard-reg-set.h"
54 #include "basic-block.h"
55 #include "insn-config.h"
65 #include "insn-flags.h"
69 /* Not really meaningful values, but at least something. */
70 #ifndef SIMULTANEOUS_PREFETCHES
71 #define SIMULTANEOUS_PREFETCHES 3
73 #ifndef PREFETCH_BLOCK
74 #define PREFETCH_BLOCK 32
77 #define HAVE_prefetch 0
78 #define CODE_FOR_prefetch 0
79 #define gen_prefetch(a,b,c) (abort(), NULL_RTX)
82 /* Give up the prefetch optimizations once we exceed a given threshold.
83 It is unlikely that we would be able to optimize something in a loop
84 with so many detected prefetches. */
85 #define MAX_PREFETCHES 100
86 /* The number of prefetch blocks that are beneficial to fetch at once before
87 a loop with a known (and low) iteration count. */
88 #define PREFETCH_BLOCKS_BEFORE_LOOP_MAX 6
89 /* For very tiny loops it is not worthwhile to prefetch even before the loop,
90 since it is likely that the data are already in the cache. */
91 #define PREFETCH_BLOCKS_BEFORE_LOOP_MIN 2
93 /* Parameterize some prefetch heuristics so they can be turned on and off
94 easily for performance testing on new architectures. These can be
95 defined in target-dependent files. */
97 /* Prefetch is worthwhile only when loads/stores are dense. */
98 #ifndef PREFETCH_ONLY_DENSE_MEM
99 #define PREFETCH_ONLY_DENSE_MEM 1
102 /* Define what we mean by "dense" loads and stores; This value divided by 256
103 is the minimum percentage of memory references that worth prefetching. */
104 #ifndef PREFETCH_DENSE_MEM
105 #define PREFETCH_DENSE_MEM 220
108 /* Do not prefetch for a loop whose iteration count is known to be low. */
109 #ifndef PREFETCH_NO_LOW_LOOPCNT
110 #define PREFETCH_NO_LOW_LOOPCNT 1
113 /* Define what we mean by a "low" iteration count. */
114 #ifndef PREFETCH_LOW_LOOPCNT
115 #define PREFETCH_LOW_LOOPCNT 32
118 /* Do not prefetch for a loop that contains a function call; such a loop is
119 probably not an internal loop. */
120 #ifndef PREFETCH_NO_CALL
121 #define PREFETCH_NO_CALL 1
124 /* Do not prefetch accesses with an extreme stride. */
125 #ifndef PREFETCH_NO_EXTREME_STRIDE
126 #define PREFETCH_NO_EXTREME_STRIDE 1
129 /* Define what we mean by an "extreme" stride. */
130 #ifndef PREFETCH_EXTREME_STRIDE
131 #define PREFETCH_EXTREME_STRIDE 4096
134 /* Define a limit to how far apart indices can be and still be merged
135 into a single prefetch. */
136 #ifndef PREFETCH_EXTREME_DIFFERENCE
137 #define PREFETCH_EXTREME_DIFFERENCE 4096
140 /* Issue prefetch instructions before the loop to fetch data to be used
141 in the first few loop iterations. */
142 #ifndef PREFETCH_BEFORE_LOOP
143 #define PREFETCH_BEFORE_LOOP 1
146 /* Do not handle reversed order prefetches (negative stride). */
147 #ifndef PREFETCH_NO_REVERSE_ORDER
148 #define PREFETCH_NO_REVERSE_ORDER 1
151 /* Prefetch even if the GIV is in conditional code. */
152 #ifndef PREFETCH_CONDITIONAL
153 #define PREFETCH_CONDITIONAL 1
156 #define LOOP_REG_LIFETIME(LOOP, REGNO) \
157 ((REGNO_LAST_LUID (REGNO) - REGNO_FIRST_LUID (REGNO)))
159 #define LOOP_REG_GLOBAL_P(LOOP, REGNO) \
160 ((REGNO_LAST_LUID (REGNO) > INSN_LUID ((LOOP)->end) \
161 || REGNO_FIRST_LUID (REGNO) < INSN_LUID ((LOOP)->start)))
163 #define LOOP_REGNO_NREGS(REGNO, SET_DEST) \
164 ((REGNO) < FIRST_PSEUDO_REGISTER \
165 ? (int) HARD_REGNO_NREGS ((REGNO), GET_MODE (SET_DEST)) : 1)
168 /* Vector mapping INSN_UIDs to luids.
169 The luids are like uids but increase monotonically always.
170 We use them to see whether a jump comes from outside a given loop. */
174 /* Indexed by INSN_UID, contains the ordinal giving the (innermost) loop
175 number the insn is contained in. */
177 struct loop **uid_loop;
179 /* 1 + largest uid of any insn. */
181 int max_uid_for_loop;
183 /* Number of loops detected in current function. Used as index to the
186 static int max_loop_num;
188 /* Bound on pseudo register number before loop optimization.
189 A pseudo has valid regscan info if its number is < max_reg_before_loop. */
190 unsigned int max_reg_before_loop;
192 /* The value to pass to the next call of reg_scan_update. */
193 static int loop_max_reg;
195 /* During the analysis of a loop, a chain of `struct movable's
196 is made to record all the movable insns found.
197 Then the entire chain can be scanned to decide which to move. */
201 rtx insn; /* A movable insn */
202 rtx set_src; /* The expression this reg is set from. */
203 rtx set_dest; /* The destination of this SET. */
204 rtx dependencies; /* When INSN is libcall, this is an EXPR_LIST
205 of any registers used within the LIBCALL. */
206 int consec; /* Number of consecutive following insns
207 that must be moved with this one. */
208 unsigned int regno; /* The register it sets */
209 short lifetime; /* lifetime of that register;
210 may be adjusted when matching movables
211 that load the same value are found. */
212 short savings; /* Number of insns we can move for this reg,
213 including other movables that force this
214 or match this one. */
215 ENUM_BITFIELD(machine_mode) savemode : 8; /* Nonzero means it is a mode for
216 a low part that we should avoid changing when
217 clearing the rest of the reg. */
218 unsigned int cond : 1; /* 1 if only conditionally movable */
219 unsigned int force : 1; /* 1 means MUST move this insn */
220 unsigned int global : 1; /* 1 means reg is live outside this loop */
221 /* If PARTIAL is 1, GLOBAL means something different:
222 that the reg is live outside the range from where it is set
223 to the following label. */
224 unsigned int done : 1; /* 1 inhibits further processing of this */
226 unsigned int partial : 1; /* 1 means this reg is used for zero-extending.
227 In particular, moving it does not make it
229 unsigned int move_insn : 1; /* 1 means that we call emit_move_insn to
230 load SRC, rather than copying INSN. */
231 unsigned int move_insn_first:1;/* Same as above, if this is necessary for the
232 first insn of a consecutive sets group. */
233 unsigned int is_equiv : 1; /* 1 means a REG_EQUIV is present on INSN. */
234 unsigned int insert_temp : 1; /* 1 means we copy to a new pseudo and replace
235 the original insn with a copy from that
236 pseudo, rather than deleting it. */
237 struct movable *match; /* First entry for same value */
238 struct movable *forces; /* An insn that must be moved if this is */
239 struct movable *next;
243 FILE *loop_dump_stream;
245 /* Forward declarations. */
247 static void invalidate_loops_containing_label (rtx);
248 static void find_and_verify_loops (rtx, struct loops *);
249 static void mark_loop_jump (rtx, struct loop *);
250 static void prescan_loop (struct loop *);
251 static int reg_in_basic_block_p (rtx, rtx);
252 static int consec_sets_invariant_p (const struct loop *, rtx, int, rtx);
253 static int labels_in_range_p (rtx, int);
254 static void count_one_set (struct loop_regs *, rtx, rtx, rtx *);
255 static void note_addr_stored (rtx, rtx, void *);
256 static void note_set_pseudo_multiple_uses (rtx, rtx, void *);
257 static int loop_reg_used_before_p (const struct loop *, rtx, rtx);
258 static void scan_loop (struct loop*, int);
260 static void replace_call_address (rtx, rtx, rtx);
262 static rtx skip_consec_insns (rtx, int);
263 static int libcall_benefit (rtx);
264 static void ignore_some_movables (struct loop_movables *);
265 static void force_movables (struct loop_movables *);
266 static void combine_movables (struct loop_movables *, struct loop_regs *);
267 static int num_unmoved_movables (const struct loop *);
268 static int regs_match_p (rtx, rtx, struct loop_movables *);
269 static int rtx_equal_for_loop_p (rtx, rtx, struct loop_movables *,
271 static void add_label_notes (rtx, rtx);
272 static void move_movables (struct loop *loop, struct loop_movables *, int,
274 static void loop_movables_add (struct loop_movables *, struct movable *);
275 static void loop_movables_free (struct loop_movables *);
276 static int count_nonfixed_reads (const struct loop *, rtx);
277 static void loop_bivs_find (struct loop *);
278 static void loop_bivs_init_find (struct loop *);
279 static void loop_bivs_check (struct loop *);
280 static void loop_givs_find (struct loop *);
281 static void loop_givs_check (struct loop *);
282 static int loop_biv_eliminable_p (struct loop *, struct iv_class *, int, int);
283 static int loop_giv_reduce_benefit (struct loop *, struct iv_class *,
284 struct induction *, rtx);
285 static void loop_givs_dead_check (struct loop *, struct iv_class *);
286 static void loop_givs_reduce (struct loop *, struct iv_class *);
287 static void loop_givs_rescan (struct loop *, struct iv_class *, rtx *);
288 static void loop_ivs_free (struct loop *);
289 static void strength_reduce (struct loop *, int);
290 static void find_single_use_in_loop (struct loop_regs *, rtx, rtx);
291 static int valid_initial_value_p (rtx, rtx, int, rtx);
292 static void find_mem_givs (const struct loop *, rtx, rtx, int, int);
293 static void record_biv (struct loop *, struct induction *, rtx, rtx, rtx,
294 rtx, rtx *, int, int);
295 static void check_final_value (const struct loop *, struct induction *);
296 static void loop_ivs_dump (const struct loop *, FILE *, int);
297 static void loop_iv_class_dump (const struct iv_class *, FILE *, int);
298 static void loop_biv_dump (const struct induction *, FILE *, int);
299 static void loop_giv_dump (const struct induction *, FILE *, int);
300 static void record_giv (const struct loop *, struct induction *, rtx, rtx,
301 rtx, rtx, rtx, rtx, int, enum g_types, int, int,
303 static void update_giv_derive (const struct loop *, rtx);
304 static void check_ext_dependent_givs (const struct loop *, struct iv_class *);
305 static int basic_induction_var (const struct loop *, rtx, enum machine_mode,
306 rtx, rtx, rtx *, rtx *, rtx **);
307 static rtx simplify_giv_expr (const struct loop *, rtx, rtx *, int *);
308 static int general_induction_var (const struct loop *loop, rtx, rtx *, rtx *,
309 rtx *, rtx *, int, int *, enum machine_mode);
310 static int consec_sets_giv (const struct loop *, int, rtx, rtx, rtx, rtx *,
311 rtx *, rtx *, rtx *);
312 static int check_dbra_loop (struct loop *, int);
313 static rtx express_from_1 (rtx, rtx, rtx);
314 static rtx combine_givs_p (struct induction *, struct induction *);
315 static int cmp_combine_givs_stats (const void *, const void *);
316 static void combine_givs (struct loop_regs *, struct iv_class *);
317 static int product_cheap_p (rtx, rtx);
318 static int maybe_eliminate_biv (const struct loop *, struct iv_class *, int,
320 static int maybe_eliminate_biv_1 (const struct loop *, rtx, rtx,
321 struct iv_class *, int, basic_block, rtx);
322 static int last_use_this_basic_block (rtx, rtx);
323 static void record_initial (rtx, rtx, void *);
324 static void update_reg_last_use (rtx, rtx);
325 static rtx next_insn_in_loop (const struct loop *, rtx);
326 static void loop_regs_scan (const struct loop *, int);
327 static int count_insns_in_loop (const struct loop *);
328 static int find_mem_in_note_1 (rtx *, void *);
329 static rtx find_mem_in_note (rtx);
330 static void load_mems (const struct loop *);
331 static int insert_loop_mem (rtx *, void *);
332 static int replace_loop_mem (rtx *, void *);
333 static void replace_loop_mems (rtx, rtx, rtx, int);
334 static int replace_loop_reg (rtx *, void *);
335 static void replace_loop_regs (rtx insn, rtx, rtx);
336 static void note_reg_stored (rtx, rtx, void *);
337 static void try_copy_prop (const struct loop *, rtx, unsigned int);
338 static void try_swap_copy_prop (const struct loop *, rtx, unsigned int);
339 static rtx check_insn_for_givs (struct loop *, rtx, int, int);
340 static rtx check_insn_for_bivs (struct loop *, rtx, int, int);
341 static rtx gen_add_mult (rtx, rtx, rtx, rtx);
342 static void loop_regs_update (const struct loop *, rtx);
343 static int iv_add_mult_cost (rtx, rtx, rtx, rtx);
345 static rtx loop_insn_emit_after (const struct loop *, basic_block, rtx, rtx);
346 static rtx loop_call_insn_emit_before (const struct loop *, basic_block,
348 static rtx loop_call_insn_hoist (const struct loop *, rtx);
349 static rtx loop_insn_sink_or_swim (const struct loop *, rtx);
351 static void loop_dump_aux (const struct loop *, FILE *, int);
352 static void loop_delete_insns (rtx, rtx);
353 static HOST_WIDE_INT remove_constant_addition (rtx *);
354 static rtx gen_load_of_final_value (rtx, rtx);
355 void debug_ivs (const struct loop *);
356 void debug_iv_class (const struct iv_class *);
357 void debug_biv (const struct induction *);
358 void debug_giv (const struct induction *);
359 void debug_loop (const struct loop *);
360 void debug_loops (const struct loops *);
362 typedef struct loop_replace_args
369 /* Nonzero iff INSN is between START and END, inclusive. */
370 #define INSN_IN_RANGE_P(INSN, START, END) \
371 (INSN_UID (INSN) < max_uid_for_loop \
372 && INSN_LUID (INSN) >= INSN_LUID (START) \
373 && INSN_LUID (INSN) <= INSN_LUID (END))
375 /* Indirect_jump_in_function is computed once per function. */
376 static int indirect_jump_in_function;
377 static int indirect_jump_in_function_p (rtx);
379 static int compute_luids (rtx, rtx, int);
381 static int biv_elimination_giv_has_0_offset (struct induction *,
382 struct induction *, rtx);
384 /* Benefit penalty, if a giv is not replaceable, i.e. must emit an insn to
385 copy the value of the strength reduced giv to its original register. */
386 static int copy_cost;
388 /* Cost of using a register, to normalize the benefits of a giv. */
389 static int reg_address_cost;
394 rtx reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
396 reg_address_cost = address_cost (reg, SImode);
398 copy_cost = COSTS_N_INSNS (1);
401 /* Compute the mapping from uids to luids.
402 LUIDs are numbers assigned to insns, like uids,
403 except that luids increase monotonically through the code.
404 Start at insn START and stop just before END. Assign LUIDs
405 starting with PREV_LUID + 1. Return the last assigned LUID + 1. */
407 compute_luids (rtx start, rtx end, int prev_luid)
412 for (insn = start, i = prev_luid; insn != end; insn = NEXT_INSN (insn))
414 if (INSN_UID (insn) >= max_uid_for_loop)
416 /* Don't assign luids to line-number NOTEs, so that the distance in
417 luids between two insns is not affected by -g. */
418 if (GET_CODE (insn) != NOTE
419 || NOTE_LINE_NUMBER (insn) <= 0)
420 uid_luid[INSN_UID (insn)] = ++i;
422 /* Give a line number note the same luid as preceding insn. */
423 uid_luid[INSN_UID (insn)] = i;
428 /* Entry point of this file. Perform loop optimization
429 on the current function. F is the first insn of the function
430 and DUMPFILE is a stream for output of a trace of actions taken
431 (or 0 if none should be output). */
434 loop_optimize (rtx f, FILE *dumpfile, int flags)
438 struct loops loops_data;
439 struct loops *loops = &loops_data;
440 struct loop_info *loops_info;
442 loop_dump_stream = dumpfile;
444 init_recog_no_volatile ();
446 max_reg_before_loop = max_reg_num ();
447 loop_max_reg = max_reg_before_loop;
451 /* Count the number of loops. */
454 for (insn = f; insn; insn = NEXT_INSN (insn))
456 if (GET_CODE (insn) == NOTE
457 && NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
461 /* Don't waste time if no loops. */
462 if (max_loop_num == 0)
465 loops->num = max_loop_num;
467 /* Get size to use for tables indexed by uids.
468 Leave some space for labels allocated by find_and_verify_loops. */
469 max_uid_for_loop = get_max_uid () + 1 + max_loop_num * 32;
471 uid_luid = xcalloc (max_uid_for_loop, sizeof (int));
472 uid_loop = xcalloc (max_uid_for_loop, sizeof (struct loop *));
474 /* Allocate storage for array of loops. */
475 loops->array = xcalloc (loops->num, sizeof (struct loop));
477 /* Find and process each loop.
478 First, find them, and record them in order of their beginnings. */
479 find_and_verify_loops (f, loops);
481 /* Allocate and initialize auxiliary loop information. */
482 loops_info = xcalloc (loops->num, sizeof (struct loop_info));
483 for (i = 0; i < (int) loops->num; i++)
484 loops->array[i].aux = loops_info + i;
486 /* Now find all register lifetimes. This must be done after
487 find_and_verify_loops, because it might reorder the insns in the
489 reg_scan (f, max_reg_before_loop, 1);
491 /* This must occur after reg_scan so that registers created by gcse
492 will have entries in the register tables.
494 We could have added a call to reg_scan after gcse_main in toplev.c,
495 but moving this call to init_alias_analysis is more efficient. */
496 init_alias_analysis ();
498 /* See if we went too far. Note that get_max_uid already returns
499 one more that the maximum uid of all insn. */
500 if (get_max_uid () > max_uid_for_loop)
502 /* Now reset it to the actual size we need. See above. */
503 max_uid_for_loop = get_max_uid ();
505 /* find_and_verify_loops has already called compute_luids, but it
506 might have rearranged code afterwards, so we need to recompute
508 compute_luids (f, NULL_RTX, 0);
510 /* Don't leave gaps in uid_luid for insns that have been
511 deleted. It is possible that the first or last insn
512 using some register has been deleted by cross-jumping.
513 Make sure that uid_luid for that former insn's uid
514 points to the general area where that insn used to be. */
515 for (i = 0; i < max_uid_for_loop; i++)
517 uid_luid[0] = uid_luid[i];
518 if (uid_luid[0] != 0)
521 for (i = 0; i < max_uid_for_loop; i++)
522 if (uid_luid[i] == 0)
523 uid_luid[i] = uid_luid[i - 1];
525 /* Determine if the function has indirect jump. On some systems
526 this prevents low overhead loop instructions from being used. */
527 indirect_jump_in_function = indirect_jump_in_function_p (f);
529 /* Now scan the loops, last ones first, since this means inner ones are done
530 before outer ones. */
531 for (i = max_loop_num - 1; i >= 0; i--)
533 struct loop *loop = &loops->array[i];
535 if (! loop->invalid && loop->end)
536 scan_loop (loop, flags);
539 end_alias_analysis ();
548 /* Returns the next insn, in execution order, after INSN. START and
549 END are the NOTE_INSN_LOOP_BEG and NOTE_INSN_LOOP_END for the loop,
550 respectively. LOOP->TOP, if non-NULL, is the top of the loop in the
551 insn-stream; it is used with loops that are entered near the
555 next_insn_in_loop (const struct loop *loop, rtx insn)
557 insn = NEXT_INSN (insn);
559 if (insn == loop->end)
562 /* Go to the top of the loop, and continue there. */
569 if (insn == loop->scan_start)
576 /* Optimize one loop described by LOOP. */
578 /* ??? Could also move memory writes out of loops if the destination address
579 is invariant, the source is invariant, the memory write is not volatile,
580 and if we can prove that no read inside the loop can read this address
581 before the write occurs. If there is a read of this address after the
582 write, then we can also mark the memory read as invariant. */
585 scan_loop (struct loop *loop, int flags)
587 struct loop_info *loop_info = LOOP_INFO (loop);
588 struct loop_regs *regs = LOOP_REGS (loop);
590 rtx loop_start = loop->start;
591 rtx loop_end = loop->end;
593 /* 1 if we are scanning insns that could be executed zero times. */
595 /* 1 if we are scanning insns that might never be executed
596 due to a subroutine call which might exit before they are reached. */
598 /* Number of insns in the loop. */
601 rtx temp, update_start, update_end;
602 /* The SET from an insn, if it is the only SET in the insn. */
604 /* Chain describing insns movable in current loop. */
605 struct loop_movables *movables = LOOP_MOVABLES (loop);
606 /* Ratio of extra register life span we can justify
607 for saving an instruction. More if loop doesn't call subroutines
608 since in that case saving an insn makes more difference
609 and more registers are available. */
611 /* Nonzero if we are scanning instructions in a sub-loop. */
620 /* Determine whether this loop starts with a jump down to a test at
621 the end. This will occur for a small number of loops with a test
622 that is too complex to duplicate in front of the loop.
624 We search for the first insn or label in the loop, skipping NOTEs.
625 However, we must be careful not to skip past a NOTE_INSN_LOOP_BEG
626 (because we might have a loop executed only once that contains a
627 loop which starts with a jump to its exit test) or a NOTE_INSN_LOOP_END
628 (in case we have a degenerate loop).
630 Note that if we mistakenly think that a loop is entered at the top
631 when, in fact, it is entered at the exit test, the only effect will be
632 slightly poorer optimization. Making the opposite error can generate
633 incorrect code. Since very few loops now start with a jump to the
634 exit test, the code here to detect that case is very conservative. */
636 for (p = NEXT_INSN (loop_start);
638 && GET_CODE (p) != CODE_LABEL && ! INSN_P (p)
639 && (GET_CODE (p) != NOTE
640 || (NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_BEG
641 && NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_END));
645 loop->scan_start = p;
647 /* If loop end is the end of the current function, then emit a
648 NOTE_INSN_DELETED after loop_end and set loop->sink to the dummy
649 note insn. This is the position we use when sinking insns out of
651 if (NEXT_INSN (loop->end) != 0)
652 loop->sink = NEXT_INSN (loop->end);
654 loop->sink = emit_note_after (NOTE_INSN_DELETED, loop->end);
656 /* Set up variables describing this loop. */
658 threshold = (loop_info->has_call ? 1 : 2) * (1 + n_non_fixed_regs);
660 /* If loop has a jump before the first label,
661 the true entry is the target of that jump.
662 Start scan from there.
663 But record in LOOP->TOP the place where the end-test jumps
664 back to so we can scan that after the end of the loop. */
665 if (GET_CODE (p) == JUMP_INSN
666 /* Loop entry must be unconditional jump (and not a RETURN) */
667 && any_uncondjump_p (p)
668 && JUMP_LABEL (p) != 0
669 /* Check to see whether the jump actually
670 jumps out of the loop (meaning it's no loop).
671 This case can happen for things like
672 do {..} while (0). If this label was generated previously
673 by loop, we can't tell anything about it and have to reject
675 && INSN_IN_RANGE_P (JUMP_LABEL (p), loop_start, loop_end))
677 loop->top = next_label (loop->scan_start);
678 loop->scan_start = JUMP_LABEL (p);
681 /* If LOOP->SCAN_START was an insn created by loop, we don't know its luid
682 as required by loop_reg_used_before_p. So skip such loops. (This
683 test may never be true, but it's best to play it safe.)
685 Also, skip loops where we do not start scanning at a label. This
686 test also rejects loops starting with a JUMP_INSN that failed the
689 if (INSN_UID (loop->scan_start) >= max_uid_for_loop
690 || GET_CODE (loop->scan_start) != CODE_LABEL)
692 if (loop_dump_stream)
693 fprintf (loop_dump_stream, "\nLoop from %d to %d is phony.\n\n",
694 INSN_UID (loop_start), INSN_UID (loop_end));
698 /* Allocate extra space for REGs that might be created by load_mems.
699 We allocate a little extra slop as well, in the hopes that we
700 won't have to reallocate the regs array. */
701 loop_regs_scan (loop, loop_info->mems_idx + 16);
702 insn_count = count_insns_in_loop (loop);
704 if (loop_dump_stream)
706 fprintf (loop_dump_stream, "\nLoop from %d to %d: %d real insns.\n",
707 INSN_UID (loop_start), INSN_UID (loop_end), insn_count);
709 fprintf (loop_dump_stream, "Continue at insn %d.\n",
710 INSN_UID (loop->cont));
713 /* Scan through the loop finding insns that are safe to move.
714 Set REGS->ARRAY[I].SET_IN_LOOP negative for the reg I being set, so that
715 this reg will be considered invariant for subsequent insns.
716 We consider whether subsequent insns use the reg
717 in deciding whether it is worth actually moving.
719 MAYBE_NEVER is nonzero if we have passed a conditional jump insn
720 and therefore it is possible that the insns we are scanning
721 would never be executed. At such times, we must make sure
722 that it is safe to execute the insn once instead of zero times.
723 When MAYBE_NEVER is 0, all insns will be executed at least once
724 so that is not a problem. */
726 for (in_libcall = 0, p = next_insn_in_loop (loop, loop->scan_start);
728 p = next_insn_in_loop (loop, p))
730 if (in_libcall && INSN_P (p) && find_reg_note (p, REG_RETVAL, NULL_RTX))
732 if (GET_CODE (p) == INSN)
734 temp = find_reg_note (p, REG_LIBCALL, NULL_RTX);
738 && (set = single_set (p))
739 && GET_CODE (SET_DEST (set)) == REG
740 #ifdef PIC_OFFSET_TABLE_REG_CALL_CLOBBERED
741 && SET_DEST (set) != pic_offset_table_rtx
743 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
749 rtx src = SET_SRC (set);
750 rtx dependencies = 0;
752 /* Figure out what to use as a source of this insn. If a
753 REG_EQUIV note is given or if a REG_EQUAL note with a
754 constant operand is specified, use it as the source and
755 mark that we should move this insn by calling
756 emit_move_insn rather that duplicating the insn.
758 Otherwise, only use the REG_EQUAL contents if a REG_RETVAL
760 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
762 src = XEXP (temp, 0), move_insn = 1;
765 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
766 if (temp && CONSTANT_P (XEXP (temp, 0)))
767 src = XEXP (temp, 0), move_insn = 1;
768 if (temp && find_reg_note (p, REG_RETVAL, NULL_RTX))
770 src = XEXP (temp, 0);
771 /* A libcall block can use regs that don't appear in
772 the equivalent expression. To move the libcall,
773 we must move those regs too. */
774 dependencies = libcall_other_reg (p, src);
778 /* For parallels, add any possible uses to the dependencies, as
779 we can't move the insn without resolving them first. */
780 if (GET_CODE (PATTERN (p)) == PARALLEL)
782 for (i = 0; i < XVECLEN (PATTERN (p), 0); i++)
784 rtx x = XVECEXP (PATTERN (p), 0, i);
785 if (GET_CODE (x) == USE)
787 = gen_rtx_EXPR_LIST (VOIDmode, XEXP (x, 0),
792 if (/* The register is used in basic blocks other
793 than the one where it is set (meaning that
794 something after this point in the loop might
795 depend on its value before the set). */
796 ! reg_in_basic_block_p (p, SET_DEST (set))
797 /* And the set is not guaranteed to be executed once
798 the loop starts, or the value before the set is
799 needed before the set occurs...
801 ??? Note we have quadratic behavior here, mitigated
802 by the fact that the previous test will often fail for
803 large loops. Rather than re-scanning the entire loop
804 each time for register usage, we should build tables
805 of the register usage and use them here instead. */
807 || loop_reg_used_before_p (loop, set, p)))
808 /* It is unsafe to move the set. However, it may be OK to
809 move the source into a new pseudo, and substitute a
810 reg-to-reg copy for the original insn.
812 This code used to consider it OK to move a set of a variable
813 which was not created by the user and not used in an exit
815 That behavior is incorrect and was removed. */
818 /* Don't try to optimize a MODE_CC set with a constant
819 source. It probably will be combined with a conditional
821 if (GET_MODE_CLASS (GET_MODE (SET_DEST (set))) == MODE_CC
824 /* Don't try to optimize a register that was made
825 by loop-optimization for an inner loop.
826 We don't know its life-span, so we can't compute
828 else if (REGNO (SET_DEST (set)) >= max_reg_before_loop)
830 /* Don't move the source and add a reg-to-reg copy:
831 - with -Os (this certainly increases size),
832 - if the mode doesn't support copy operations (obviously),
833 - if the source is already a reg (the motion will gain nothing),
834 - if the source is a legitimate constant (likewise). */
837 || ! can_copy_p (GET_MODE (SET_SRC (set)))
838 || GET_CODE (SET_SRC (set)) == REG
839 || (CONSTANT_P (SET_SRC (set))
840 && LEGITIMATE_CONSTANT_P (SET_SRC (set)))))
842 else if ((tem = loop_invariant_p (loop, src))
843 && (dependencies == 0
845 = loop_invariant_p (loop, dependencies)) != 0)
846 && (regs->array[REGNO (SET_DEST (set))].set_in_loop == 1
848 = consec_sets_invariant_p
849 (loop, SET_DEST (set),
850 regs->array[REGNO (SET_DEST (set))].set_in_loop,
852 /* If the insn can cause a trap (such as divide by zero),
853 can't move it unless it's guaranteed to be executed
854 once loop is entered. Even a function call might
855 prevent the trap insn from being reached
856 (since it might exit!) */
857 && ! ((maybe_never || call_passed)
858 && may_trap_p (src)))
861 int regno = REGNO (SET_DEST (set));
863 /* A potential lossage is where we have a case where two insns
864 can be combined as long as they are both in the loop, but
865 we move one of them outside the loop. For large loops,
866 this can lose. The most common case of this is the address
867 of a function being called.
869 Therefore, if this register is marked as being used
870 exactly once if we are in a loop with calls
871 (a "large loop"), see if we can replace the usage of
872 this register with the source of this SET. If we can,
875 Don't do this if P has a REG_RETVAL note or if we have
876 SMALL_REGISTER_CLASSES and SET_SRC is a hard register. */
878 if (loop_info->has_call
879 && regs->array[regno].single_usage != 0
880 && regs->array[regno].single_usage != const0_rtx
881 && REGNO_FIRST_UID (regno) == INSN_UID (p)
882 && (REGNO_LAST_UID (regno)
883 == INSN_UID (regs->array[regno].single_usage))
884 && regs->array[regno].set_in_loop == 1
885 && GET_CODE (SET_SRC (set)) != ASM_OPERANDS
886 && ! side_effects_p (SET_SRC (set))
887 && ! find_reg_note (p, REG_RETVAL, NULL_RTX)
888 && (! SMALL_REGISTER_CLASSES
889 || (! (GET_CODE (SET_SRC (set)) == REG
890 && (REGNO (SET_SRC (set))
891 < FIRST_PSEUDO_REGISTER))))
892 /* This test is not redundant; SET_SRC (set) might be
893 a call-clobbered register and the life of REGNO
894 might span a call. */
895 && ! modified_between_p (SET_SRC (set), p,
896 regs->array[regno].single_usage)
897 && no_labels_between_p (p,
898 regs->array[regno].single_usage)
899 && validate_replace_rtx (SET_DEST (set), SET_SRC (set),
900 regs->array[regno].single_usage))
902 /* Replace any usage in a REG_EQUAL note. Must copy
903 the new source, so that we don't get rtx sharing
904 between the SET_SOURCE and REG_NOTES of insn p. */
905 REG_NOTES (regs->array[regno].single_usage)
907 (REG_NOTES (regs->array[regno].single_usage),
908 SET_DEST (set), copy_rtx (SET_SRC (set))));
911 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
913 regs->array[regno+i].set_in_loop = 0;
917 m = xmalloc (sizeof (struct movable));
921 m->dependencies = dependencies;
922 m->set_dest = SET_DEST (set);
925 = regs->array[REGNO (SET_DEST (set))].set_in_loop - 1;
929 m->move_insn = move_insn;
930 m->move_insn_first = 0;
931 m->insert_temp = insert_temp;
932 m->is_equiv = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
933 m->savemode = VOIDmode;
935 /* Set M->cond if either loop_invariant_p
936 or consec_sets_invariant_p returned 2
937 (only conditionally invariant). */
938 m->cond = ((tem | tem1 | tem2) > 1);
939 m->global = LOOP_REG_GLOBAL_P (loop, regno);
941 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
942 m->savings = regs->array[regno].n_times_set;
943 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
944 m->savings += libcall_benefit (p);
945 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set)); i++)
946 regs->array[regno+i].set_in_loop = move_insn ? -2 : -1;
947 /* Add M to the end of the chain MOVABLES. */
948 loop_movables_add (movables, m);
952 /* It is possible for the first instruction to have a
953 REG_EQUAL note but a non-invariant SET_SRC, so we must
954 remember the status of the first instruction in case
955 the last instruction doesn't have a REG_EQUAL note. */
956 m->move_insn_first = m->move_insn;
958 /* Skip this insn, not checking REG_LIBCALL notes. */
959 p = next_nonnote_insn (p);
960 /* Skip the consecutive insns, if there are any. */
961 p = skip_consec_insns (p, m->consec);
962 /* Back up to the last insn of the consecutive group. */
963 p = prev_nonnote_insn (p);
965 /* We must now reset m->move_insn, m->is_equiv, and
966 possibly m->set_src to correspond to the effects of
968 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
970 m->set_src = XEXP (temp, 0), m->move_insn = 1;
973 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
974 if (temp && CONSTANT_P (XEXP (temp, 0)))
975 m->set_src = XEXP (temp, 0), m->move_insn = 1;
981 = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
984 /* If this register is always set within a STRICT_LOW_PART
985 or set to zero, then its high bytes are constant.
986 So clear them outside the loop and within the loop
987 just load the low bytes.
988 We must check that the machine has an instruction to do so.
989 Also, if the value loaded into the register
990 depends on the same register, this cannot be done. */
991 else if (SET_SRC (set) == const0_rtx
992 && GET_CODE (NEXT_INSN (p)) == INSN
993 && (set1 = single_set (NEXT_INSN (p)))
994 && GET_CODE (set1) == SET
995 && (GET_CODE (SET_DEST (set1)) == STRICT_LOW_PART)
996 && (GET_CODE (XEXP (SET_DEST (set1), 0)) == SUBREG)
997 && (SUBREG_REG (XEXP (SET_DEST (set1), 0))
999 && !reg_mentioned_p (SET_DEST (set), SET_SRC (set1)))
1001 int regno = REGNO (SET_DEST (set));
1002 if (regs->array[regno].set_in_loop == 2)
1005 m = xmalloc (sizeof (struct movable));
1008 m->set_dest = SET_DEST (set);
1009 m->dependencies = 0;
1015 m->move_insn_first = 0;
1016 m->insert_temp = insert_temp;
1018 /* If the insn may not be executed on some cycles,
1019 we can't clear the whole reg; clear just high part.
1020 Not even if the reg is used only within this loop.
1027 Clearing x before the inner loop could clobber a value
1028 being saved from the last time around the outer loop.
1029 However, if the reg is not used outside this loop
1030 and all uses of the register are in the same
1031 basic block as the store, there is no problem.
1033 If this insn was made by loop, we don't know its
1034 INSN_LUID and hence must make a conservative
1036 m->global = (INSN_UID (p) >= max_uid_for_loop
1037 || LOOP_REG_GLOBAL_P (loop, regno)
1038 || (labels_in_range_p
1039 (p, REGNO_FIRST_LUID (regno))));
1040 if (maybe_never && m->global)
1041 m->savemode = GET_MODE (SET_SRC (set1));
1043 m->savemode = VOIDmode;
1047 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
1050 i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
1052 regs->array[regno+i].set_in_loop = -1;
1053 /* Add M to the end of the chain MOVABLES. */
1054 loop_movables_add (movables, m);
1059 /* Past a call insn, we get to insns which might not be executed
1060 because the call might exit. This matters for insns that trap.
1061 Constant and pure call insns always return, so they don't count. */
1062 else if (GET_CODE (p) == CALL_INSN && ! CONST_OR_PURE_CALL_P (p))
1064 /* Past a label or a jump, we get to insns for which we
1065 can't count on whether or how many times they will be
1066 executed during each iteration. Therefore, we can
1067 only move out sets of trivial variables
1068 (those not used after the loop). */
1069 /* Similar code appears twice in strength_reduce. */
1070 else if ((GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN)
1071 /* If we enter the loop in the middle, and scan around to the
1072 beginning, don't set maybe_never for that. This must be an
1073 unconditional jump, otherwise the code at the top of the
1074 loop might never be executed. Unconditional jumps are
1075 followed by a barrier then the loop_end. */
1076 && ! (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == loop->top
1077 && NEXT_INSN (NEXT_INSN (p)) == loop_end
1078 && any_uncondjump_p (p)))
1080 else if (GET_CODE (p) == NOTE)
1082 /* At the virtual top of a converted loop, insns are again known to
1083 be executed: logically, the loop begins here even though the exit
1084 code has been duplicated. */
1085 if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP && loop_depth == 0)
1086 maybe_never = call_passed = 0;
1087 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
1089 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
1094 /* If one movable subsumes another, ignore that other. */
1096 ignore_some_movables (movables);
1098 /* For each movable insn, see if the reg that it loads
1099 leads when it dies right into another conditionally movable insn.
1100 If so, record that the second insn "forces" the first one,
1101 since the second can be moved only if the first is. */
1103 force_movables (movables);
1105 /* See if there are multiple movable insns that load the same value.
1106 If there are, make all but the first point at the first one
1107 through the `match' field, and add the priorities of them
1108 all together as the priority of the first. */
1110 combine_movables (movables, regs);
1112 /* Now consider each movable insn to decide whether it is worth moving.
1113 Store 0 in regs->array[I].set_in_loop for each reg I that is moved.
1115 For machines with few registers this increases code size, so do not
1116 move moveables when optimizing for code size on such machines.
1117 (The 18 below is the value for i386.) */
1120 || (reg_class_size[GENERAL_REGS] > 18 && !loop_info->has_call))
1122 move_movables (loop, movables, threshold, insn_count);
1124 /* Recalculate regs->array if move_movables has created new
1126 if (max_reg_num () > regs->num)
1128 loop_regs_scan (loop, 0);
1129 for (update_start = loop_start;
1130 PREV_INSN (update_start)
1131 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1132 update_start = PREV_INSN (update_start))
1134 update_end = NEXT_INSN (loop_end);
1136 reg_scan_update (update_start, update_end, loop_max_reg);
1137 loop_max_reg = max_reg_num ();
1141 /* Now candidates that still are negative are those not moved.
1142 Change regs->array[I].set_in_loop to indicate that those are not actually
1144 for (i = 0; i < regs->num; i++)
1145 if (regs->array[i].set_in_loop < 0)
1146 regs->array[i].set_in_loop = regs->array[i].n_times_set;
1148 /* Now that we've moved some things out of the loop, we might be able to
1149 hoist even more memory references. */
1152 /* Recalculate regs->array if load_mems has created new registers. */
1153 if (max_reg_num () > regs->num)
1154 loop_regs_scan (loop, 0);
1156 for (update_start = loop_start;
1157 PREV_INSN (update_start)
1158 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1159 update_start = PREV_INSN (update_start))
1161 update_end = NEXT_INSN (loop_end);
1163 reg_scan_update (update_start, update_end, loop_max_reg);
1164 loop_max_reg = max_reg_num ();
1166 if (flag_strength_reduce)
1168 if (update_end && GET_CODE (update_end) == CODE_LABEL)
1169 /* Ensure our label doesn't go away. */
1170 LABEL_NUSES (update_end)++;
1172 strength_reduce (loop, flags);
1174 reg_scan_update (update_start, update_end, loop_max_reg);
1175 loop_max_reg = max_reg_num ();
1177 if (update_end && GET_CODE (update_end) == CODE_LABEL
1178 && --LABEL_NUSES (update_end) == 0)
1179 delete_related_insns (update_end);
1183 /* The movable information is required for strength reduction. */
1184 loop_movables_free (movables);
1191 /* Add elements to *OUTPUT to record all the pseudo-regs
1192 mentioned in IN_THIS but not mentioned in NOT_IN_THIS. */
1195 record_excess_regs (rtx in_this, rtx not_in_this, rtx *output)
1201 code = GET_CODE (in_this);
1215 if (REGNO (in_this) >= FIRST_PSEUDO_REGISTER
1216 && ! reg_mentioned_p (in_this, not_in_this))
1217 *output = gen_rtx_EXPR_LIST (VOIDmode, in_this, *output);
1224 fmt = GET_RTX_FORMAT (code);
1225 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1232 for (j = 0; j < XVECLEN (in_this, i); j++)
1233 record_excess_regs (XVECEXP (in_this, i, j), not_in_this, output);
1237 record_excess_regs (XEXP (in_this, i), not_in_this, output);
1243 /* Check what regs are referred to in the libcall block ending with INSN,
1244 aside from those mentioned in the equivalent value.
1245 If there are none, return 0.
1246 If there are one or more, return an EXPR_LIST containing all of them. */
1249 libcall_other_reg (rtx insn, rtx equiv)
1251 rtx note = find_reg_note (insn, REG_RETVAL, NULL_RTX);
1252 rtx p = XEXP (note, 0);
1255 /* First, find all the regs used in the libcall block
1256 that are not mentioned as inputs to the result. */
1260 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
1261 || GET_CODE (p) == CALL_INSN)
1262 record_excess_regs (PATTERN (p), equiv, &output);
1269 /* Return 1 if all uses of REG
1270 are between INSN and the end of the basic block. */
1273 reg_in_basic_block_p (rtx insn, rtx reg)
1275 int regno = REGNO (reg);
1278 if (REGNO_FIRST_UID (regno) != INSN_UID (insn))
1281 /* Search this basic block for the already recorded last use of the reg. */
1282 for (p = insn; p; p = NEXT_INSN (p))
1284 switch (GET_CODE (p))
1291 /* Ordinary insn: if this is the last use, we win. */
1292 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1297 /* Jump insn: if this is the last use, we win. */
1298 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1300 /* Otherwise, it's the end of the basic block, so we lose. */
1305 /* It's the end of the basic block, so we lose. */
1313 /* The "last use" that was recorded can't be found after the first
1314 use. This can happen when the last use was deleted while
1315 processing an inner loop, this inner loop was then completely
1316 unrolled, and the outer loop is always exited after the inner loop,
1317 so that everything after the first use becomes a single basic block. */
1321 /* Compute the benefit of eliminating the insns in the block whose
1322 last insn is LAST. This may be a group of insns used to compute a
1323 value directly or can contain a library call. */
1326 libcall_benefit (rtx last)
1331 for (insn = XEXP (find_reg_note (last, REG_RETVAL, NULL_RTX), 0);
1332 insn != last; insn = NEXT_INSN (insn))
1334 if (GET_CODE (insn) == CALL_INSN)
1335 benefit += 10; /* Assume at least this many insns in a library
1337 else if (GET_CODE (insn) == INSN
1338 && GET_CODE (PATTERN (insn)) != USE
1339 && GET_CODE (PATTERN (insn)) != CLOBBER)
1346 /* Skip COUNT insns from INSN, counting library calls as 1 insn. */
1349 skip_consec_insns (rtx insn, int count)
1351 for (; count > 0; count--)
1355 /* If first insn of libcall sequence, skip to end. */
1356 /* Do this at start of loop, since INSN is guaranteed to
1358 if (GET_CODE (insn) != NOTE
1359 && (temp = find_reg_note (insn, REG_LIBCALL, NULL_RTX)))
1360 insn = XEXP (temp, 0);
1363 insn = NEXT_INSN (insn);
1364 while (GET_CODE (insn) == NOTE);
1370 /* Ignore any movable whose insn falls within a libcall
1371 which is part of another movable.
1372 We make use of the fact that the movable for the libcall value
1373 was made later and so appears later on the chain. */
1376 ignore_some_movables (struct loop_movables *movables)
1378 struct movable *m, *m1;
1380 for (m = movables->head; m; m = m->next)
1382 /* Is this a movable for the value of a libcall? */
1383 rtx note = find_reg_note (m->insn, REG_RETVAL, NULL_RTX);
1387 /* Check for earlier movables inside that range,
1388 and mark them invalid. We cannot use LUIDs here because
1389 insns created by loop.c for prior loops don't have LUIDs.
1390 Rather than reject all such insns from movables, we just
1391 explicitly check each insn in the libcall (since invariant
1392 libcalls aren't that common). */
1393 for (insn = XEXP (note, 0); insn != m->insn; insn = NEXT_INSN (insn))
1394 for (m1 = movables->head; m1 != m; m1 = m1->next)
1395 if (m1->insn == insn)
1401 /* For each movable insn, see if the reg that it loads
1402 leads when it dies right into another conditionally movable insn.
1403 If so, record that the second insn "forces" the first one,
1404 since the second can be moved only if the first is. */
1407 force_movables (struct loop_movables *movables)
1409 struct movable *m, *m1;
1411 for (m1 = movables->head; m1; m1 = m1->next)
1412 /* Omit this if moving just the (SET (REG) 0) of a zero-extend. */
1413 if (!m1->partial && !m1->done)
1415 int regno = m1->regno;
1416 for (m = m1->next; m; m = m->next)
1417 /* ??? Could this be a bug? What if CSE caused the
1418 register of M1 to be used after this insn?
1419 Since CSE does not update regno_last_uid,
1420 this insn M->insn might not be where it dies.
1421 But very likely this doesn't matter; what matters is
1422 that M's reg is computed from M1's reg. */
1423 if (INSN_UID (m->insn) == REGNO_LAST_UID (regno)
1426 if (m != 0 && m->set_src == m1->set_dest
1427 /* If m->consec, m->set_src isn't valid. */
1431 /* Increase the priority of the moving the first insn
1432 since it permits the second to be moved as well. */
1436 m1->lifetime += m->lifetime;
1437 m1->savings += m->savings;
1442 /* Find invariant expressions that are equal and can be combined into
1446 combine_movables (struct loop_movables *movables, struct loop_regs *regs)
1449 char *matched_regs = xmalloc (regs->num);
1450 enum machine_mode mode;
1452 /* Regs that are set more than once are not allowed to match
1453 or be matched. I'm no longer sure why not. */
1454 /* Only pseudo registers are allowed to match or be matched,
1455 since move_movables does not validate the change. */
1456 /* Perhaps testing m->consec_sets would be more appropriate here? */
1458 for (m = movables->head; m; m = m->next)
1459 if (m->match == 0 && regs->array[m->regno].n_times_set == 1
1460 && m->regno >= FIRST_PSEUDO_REGISTER
1465 int regno = m->regno;
1467 memset (matched_regs, 0, regs->num);
1468 matched_regs[regno] = 1;
1470 /* We want later insns to match the first one. Don't make the first
1471 one match any later ones. So start this loop at m->next. */
1472 for (m1 = m->next; m1; m1 = m1->next)
1473 if (m != m1 && m1->match == 0
1475 && regs->array[m1->regno].n_times_set == 1
1476 && m1->regno >= FIRST_PSEUDO_REGISTER
1477 /* A reg used outside the loop mustn't be eliminated. */
1479 /* A reg used for zero-extending mustn't be eliminated. */
1481 && (matched_regs[m1->regno]
1484 /* Can combine regs with different modes loaded from the
1485 same constant only if the modes are the same or
1486 if both are integer modes with M wider or the same
1487 width as M1. The check for integer is redundant, but
1488 safe, since the only case of differing destination
1489 modes with equal sources is when both sources are
1490 VOIDmode, i.e., CONST_INT. */
1491 (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest)
1492 || (GET_MODE_CLASS (GET_MODE (m->set_dest)) == MODE_INT
1493 && GET_MODE_CLASS (GET_MODE (m1->set_dest)) == MODE_INT
1494 && (GET_MODE_BITSIZE (GET_MODE (m->set_dest))
1495 >= GET_MODE_BITSIZE (GET_MODE (m1->set_dest)))))
1496 /* See if the source of M1 says it matches M. */
1497 && ((GET_CODE (m1->set_src) == REG
1498 && matched_regs[REGNO (m1->set_src)])
1499 || rtx_equal_for_loop_p (m->set_src, m1->set_src,
1501 && ((m->dependencies == m1->dependencies)
1502 || rtx_equal_p (m->dependencies, m1->dependencies)))
1504 m->lifetime += m1->lifetime;
1505 m->savings += m1->savings;
1508 matched_regs[m1->regno] = 1;
1512 /* Now combine the regs used for zero-extension.
1513 This can be done for those not marked `global'
1514 provided their lives don't overlap. */
1516 for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
1517 mode = GET_MODE_WIDER_MODE (mode))
1519 struct movable *m0 = 0;
1521 /* Combine all the registers for extension from mode MODE.
1522 Don't combine any that are used outside this loop. */
1523 for (m = movables->head; m; m = m->next)
1524 if (m->partial && ! m->global
1525 && mode == GET_MODE (SET_SRC (PATTERN (NEXT_INSN (m->insn)))))
1529 int first = REGNO_FIRST_LUID (m->regno);
1530 int last = REGNO_LAST_LUID (m->regno);
1534 /* First one: don't check for overlap, just record it. */
1539 /* Make sure they extend to the same mode.
1540 (Almost always true.) */
1541 if (GET_MODE (m->set_dest) != GET_MODE (m0->set_dest))
1544 /* We already have one: check for overlap with those
1545 already combined together. */
1546 for (m1 = movables->head; m1 != m; m1 = m1->next)
1547 if (m1 == m0 || (m1->partial && m1->match == m0))
1548 if (! (REGNO_FIRST_LUID (m1->regno) > last
1549 || REGNO_LAST_LUID (m1->regno) < first))
1552 /* No overlap: we can combine this with the others. */
1553 m0->lifetime += m->lifetime;
1554 m0->savings += m->savings;
1564 free (matched_regs);
1567 /* Returns the number of movable instructions in LOOP that were not
1568 moved outside the loop. */
1571 num_unmoved_movables (const struct loop *loop)
1576 for (m = LOOP_MOVABLES (loop)->head; m; m = m->next)
1584 /* Return 1 if regs X and Y will become the same if moved. */
1587 regs_match_p (rtx x, rtx y, struct loop_movables *movables)
1589 unsigned int xn = REGNO (x);
1590 unsigned int yn = REGNO (y);
1591 struct movable *mx, *my;
1593 for (mx = movables->head; mx; mx = mx->next)
1594 if (mx->regno == xn)
1597 for (my = movables->head; my; my = my->next)
1598 if (my->regno == yn)
1602 && ((mx->match == my->match && mx->match != 0)
1604 || mx == my->match));
1607 /* Return 1 if X and Y are identical-looking rtx's.
1608 This is the Lisp function EQUAL for rtx arguments.
1610 If two registers are matching movables or a movable register and an
1611 equivalent constant, consider them equal. */
1614 rtx_equal_for_loop_p (rtx x, rtx y, struct loop_movables *movables,
1615 struct loop_regs *regs)
1625 if (x == 0 || y == 0)
1628 code = GET_CODE (x);
1630 /* If we have a register and a constant, they may sometimes be
1632 if (GET_CODE (x) == REG && regs->array[REGNO (x)].set_in_loop == -2
1635 for (m = movables->head; m; m = m->next)
1636 if (m->move_insn && m->regno == REGNO (x)
1637 && rtx_equal_p (m->set_src, y))
1640 else if (GET_CODE (y) == REG && regs->array[REGNO (y)].set_in_loop == -2
1643 for (m = movables->head; m; m = m->next)
1644 if (m->move_insn && m->regno == REGNO (y)
1645 && rtx_equal_p (m->set_src, x))
1649 /* Otherwise, rtx's of different codes cannot be equal. */
1650 if (code != GET_CODE (y))
1653 /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent.
1654 (REG:SI x) and (REG:HI x) are NOT equivalent. */
1656 if (GET_MODE (x) != GET_MODE (y))
1659 /* These three types of rtx's can be compared nonrecursively. */
1661 return (REGNO (x) == REGNO (y) || regs_match_p (x, y, movables));
1663 if (code == LABEL_REF)
1664 return XEXP (x, 0) == XEXP (y, 0);
1665 if (code == SYMBOL_REF)
1666 return XSTR (x, 0) == XSTR (y, 0);
1668 /* Compare the elements. If any pair of corresponding elements
1669 fail to match, return 0 for the whole things. */
1671 fmt = GET_RTX_FORMAT (code);
1672 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1677 if (XWINT (x, i) != XWINT (y, i))
1682 if (XINT (x, i) != XINT (y, i))
1687 /* Two vectors must have the same length. */
1688 if (XVECLEN (x, i) != XVECLEN (y, i))
1691 /* And the corresponding elements must match. */
1692 for (j = 0; j < XVECLEN (x, i); j++)
1693 if (rtx_equal_for_loop_p (XVECEXP (x, i, j), XVECEXP (y, i, j),
1694 movables, regs) == 0)
1699 if (rtx_equal_for_loop_p (XEXP (x, i), XEXP (y, i), movables, regs)
1705 if (strcmp (XSTR (x, i), XSTR (y, i)))
1710 /* These are just backpointers, so they don't matter. */
1716 /* It is believed that rtx's at this level will never
1717 contain anything but integers and other rtx's,
1718 except for within LABEL_REFs and SYMBOL_REFs. */
1726 /* If X contains any LABEL_REF's, add REG_LABEL notes for them to all
1727 insns in INSNS which use the reference. LABEL_NUSES for CODE_LABEL
1728 references is incremented once for each added note. */
1731 add_label_notes (rtx x, rtx insns)
1733 enum rtx_code code = GET_CODE (x);
1738 if (code == LABEL_REF && !LABEL_REF_NONLOCAL_P (x))
1740 /* This code used to ignore labels that referred to dispatch tables to
1741 avoid flow generating (slightly) worse code.
1743 We no longer ignore such label references (see LABEL_REF handling in
1744 mark_jump_label for additional information). */
1745 for (insn = insns; insn; insn = NEXT_INSN (insn))
1746 if (reg_mentioned_p (XEXP (x, 0), insn))
1748 REG_NOTES (insn) = gen_rtx_INSN_LIST (REG_LABEL, XEXP (x, 0),
1750 if (LABEL_P (XEXP (x, 0)))
1751 LABEL_NUSES (XEXP (x, 0))++;
1755 fmt = GET_RTX_FORMAT (code);
1756 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1759 add_label_notes (XEXP (x, i), insns);
1760 else if (fmt[i] == 'E')
1761 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1762 add_label_notes (XVECEXP (x, i, j), insns);
1766 /* Scan MOVABLES, and move the insns that deserve to be moved.
1767 If two matching movables are combined, replace one reg with the
1768 other throughout. */
1771 move_movables (struct loop *loop, struct loop_movables *movables,
1772 int threshold, int insn_count)
1774 struct loop_regs *regs = LOOP_REGS (loop);
1775 int nregs = regs->num;
1779 rtx loop_start = loop->start;
1780 rtx loop_end = loop->end;
1781 /* Map of pseudo-register replacements to handle combining
1782 when we move several insns that load the same value
1783 into different pseudo-registers. */
1784 rtx *reg_map = xcalloc (nregs, sizeof (rtx));
1785 char *already_moved = xcalloc (nregs, sizeof (char));
1787 for (m = movables->head; m; m = m->next)
1789 /* Describe this movable insn. */
1791 if (loop_dump_stream)
1793 fprintf (loop_dump_stream, "Insn %d: regno %d (life %d), ",
1794 INSN_UID (m->insn), m->regno, m->lifetime);
1796 fprintf (loop_dump_stream, "consec %d, ", m->consec);
1798 fprintf (loop_dump_stream, "cond ");
1800 fprintf (loop_dump_stream, "force ");
1802 fprintf (loop_dump_stream, "global ");
1804 fprintf (loop_dump_stream, "done ");
1806 fprintf (loop_dump_stream, "move-insn ");
1808 fprintf (loop_dump_stream, "matches %d ",
1809 INSN_UID (m->match->insn));
1811 fprintf (loop_dump_stream, "forces %d ",
1812 INSN_UID (m->forces->insn));
1815 /* Ignore the insn if it's already done (it matched something else).
1816 Otherwise, see if it is now safe to move. */
1820 || (1 == loop_invariant_p (loop, m->set_src)
1821 && (m->dependencies == 0
1822 || 1 == loop_invariant_p (loop, m->dependencies))
1824 || 1 == consec_sets_invariant_p (loop, m->set_dest,
1827 && (! m->forces || m->forces->done))
1831 int savings = m->savings;
1833 /* We have an insn that is safe to move.
1834 Compute its desirability. */
1839 if (loop_dump_stream)
1840 fprintf (loop_dump_stream, "savings %d ", savings);
1842 if (regs->array[regno].moved_once && loop_dump_stream)
1843 fprintf (loop_dump_stream, "halved since already moved ");
1845 /* An insn MUST be moved if we already moved something else
1846 which is safe only if this one is moved too: that is,
1847 if already_moved[REGNO] is nonzero. */
1849 /* An insn is desirable to move if the new lifetime of the
1850 register is no more than THRESHOLD times the old lifetime.
1851 If it's not desirable, it means the loop is so big
1852 that moving won't speed things up much,
1853 and it is liable to make register usage worse. */
1855 /* It is also desirable to move if it can be moved at no
1856 extra cost because something else was already moved. */
1858 if (already_moved[regno]
1859 || flag_move_all_movables
1860 || (threshold * savings * m->lifetime) >=
1861 (regs->array[regno].moved_once ? insn_count * 2 : insn_count)
1862 || (m->forces && m->forces->done
1863 && regs->array[m->forces->regno].n_times_set == 1))
1867 rtx first = NULL_RTX;
1868 rtx newreg = NULL_RTX;
1871 newreg = gen_reg_rtx (GET_MODE (m->set_dest));
1873 /* Now move the insns that set the reg. */
1875 if (m->partial && m->match)
1879 /* Find the end of this chain of matching regs.
1880 Thus, we load each reg in the chain from that one reg.
1881 And that reg is loaded with 0 directly,
1882 since it has ->match == 0. */
1883 for (m1 = m; m1->match; m1 = m1->match);
1884 newpat = gen_move_insn (SET_DEST (PATTERN (m->insn)),
1885 SET_DEST (PATTERN (m1->insn)));
1886 i1 = loop_insn_hoist (loop, newpat);
1888 /* Mark the moved, invariant reg as being allowed to
1889 share a hard reg with the other matching invariant. */
1890 REG_NOTES (i1) = REG_NOTES (m->insn);
1891 r1 = SET_DEST (PATTERN (m->insn));
1892 r2 = SET_DEST (PATTERN (m1->insn));
1894 = gen_rtx_EXPR_LIST (VOIDmode, r1,
1895 gen_rtx_EXPR_LIST (VOIDmode, r2,
1897 delete_insn (m->insn);
1902 if (loop_dump_stream)
1903 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
1905 /* If we are to re-generate the item being moved with a
1906 new move insn, first delete what we have and then emit
1907 the move insn before the loop. */
1908 else if (m->move_insn)
1912 for (count = m->consec; count >= 0; count--)
1914 /* If this is the first insn of a library call sequence,
1915 something is very wrong. */
1916 if (GET_CODE (p) != NOTE
1917 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
1920 /* If this is the last insn of a libcall sequence, then
1921 delete every insn in the sequence except the last.
1922 The last insn is handled in the normal manner. */
1923 if (GET_CODE (p) != NOTE
1924 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
1926 temp = XEXP (temp, 0);
1928 temp = delete_insn (temp);
1932 p = delete_insn (p);
1934 /* simplify_giv_expr expects that it can walk the insns
1935 at m->insn forwards and see this old sequence we are
1936 tossing here. delete_insn does preserve the next
1937 pointers, but when we skip over a NOTE we must fix
1938 it up. Otherwise that code walks into the non-deleted
1940 while (p && GET_CODE (p) == NOTE)
1941 p = NEXT_INSN (temp) = NEXT_INSN (p);
1945 /* Replace the original insn with a move from
1946 our newly created temp. */
1948 emit_move_insn (m->set_dest, newreg);
1951 emit_insn_before (seq, p);
1956 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
1961 add_label_notes (m->set_src, seq);
1963 i1 = loop_insn_hoist (loop, seq);
1964 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
1965 set_unique_reg_note (i1,
1966 m->is_equiv ? REG_EQUIV : REG_EQUAL,
1969 if (loop_dump_stream)
1970 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
1972 /* The more regs we move, the less we like moving them. */
1977 for (count = m->consec; count >= 0; count--)
1981 /* If first insn of libcall sequence, skip to end. */
1982 /* Do this at start of loop, since p is guaranteed to
1984 if (GET_CODE (p) != NOTE
1985 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
1988 /* If last insn of libcall sequence, move all
1989 insns except the last before the loop. The last
1990 insn is handled in the normal manner. */
1991 if (GET_CODE (p) != NOTE
1992 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
1996 rtx fn_address_insn = 0;
1999 for (temp = XEXP (temp, 0); temp != p;
2000 temp = NEXT_INSN (temp))
2006 if (GET_CODE (temp) == NOTE)
2009 body = PATTERN (temp);
2011 /* Find the next insn after TEMP,
2012 not counting USE or NOTE insns. */
2013 for (next = NEXT_INSN (temp); next != p;
2014 next = NEXT_INSN (next))
2015 if (! (GET_CODE (next) == INSN
2016 && GET_CODE (PATTERN (next)) == USE)
2017 && GET_CODE (next) != NOTE)
2020 /* If that is the call, this may be the insn
2021 that loads the function address.
2023 Extract the function address from the insn
2024 that loads it into a register.
2025 If this insn was cse'd, we get incorrect code.
2027 So emit a new move insn that copies the
2028 function address into the register that the
2029 call insn will use. flow.c will delete any
2030 redundant stores that we have created. */
2031 if (GET_CODE (next) == CALL_INSN
2032 && GET_CODE (body) == SET
2033 && GET_CODE (SET_DEST (body)) == REG
2034 && (n = find_reg_note (temp, REG_EQUAL,
2037 fn_reg = SET_SRC (body);
2038 if (GET_CODE (fn_reg) != REG)
2039 fn_reg = SET_DEST (body);
2040 fn_address = XEXP (n, 0);
2041 fn_address_insn = temp;
2043 /* We have the call insn.
2044 If it uses the register we suspect it might,
2045 load it with the correct address directly. */
2046 if (GET_CODE (temp) == CALL_INSN
2048 && reg_referenced_p (fn_reg, body))
2049 loop_insn_emit_after (loop, 0, fn_address_insn,
2051 (fn_reg, fn_address));
2053 if (GET_CODE (temp) == CALL_INSN)
2055 i1 = loop_call_insn_hoist (loop, body);
2056 /* Because the USAGE information potentially
2057 contains objects other than hard registers
2058 we need to copy it. */
2059 if (CALL_INSN_FUNCTION_USAGE (temp))
2060 CALL_INSN_FUNCTION_USAGE (i1)
2061 = copy_rtx (CALL_INSN_FUNCTION_USAGE (temp));
2064 i1 = loop_insn_hoist (loop, body);
2067 if (temp == fn_address_insn)
2068 fn_address_insn = i1;
2069 REG_NOTES (i1) = REG_NOTES (temp);
2070 REG_NOTES (temp) = NULL;
2076 if (m->savemode != VOIDmode)
2078 /* P sets REG to zero; but we should clear only
2079 the bits that are not covered by the mode
2081 rtx reg = m->set_dest;
2086 tem = expand_simple_binop
2087 (GET_MODE (reg), AND, reg,
2088 GEN_INT ((((HOST_WIDE_INT) 1
2089 << GET_MODE_BITSIZE (m->savemode)))
2091 reg, 1, OPTAB_LIB_WIDEN);
2095 emit_move_insn (reg, tem);
2096 sequence = get_insns ();
2098 i1 = loop_insn_hoist (loop, sequence);
2100 else if (GET_CODE (p) == CALL_INSN)
2102 i1 = loop_call_insn_hoist (loop, PATTERN (p));
2103 /* Because the USAGE information potentially
2104 contains objects other than hard registers
2105 we need to copy it. */
2106 if (CALL_INSN_FUNCTION_USAGE (p))
2107 CALL_INSN_FUNCTION_USAGE (i1)
2108 = copy_rtx (CALL_INSN_FUNCTION_USAGE (p));
2110 else if (count == m->consec && m->move_insn_first)
2113 /* The SET_SRC might not be invariant, so we must
2114 use the REG_EQUAL note. */
2116 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
2121 add_label_notes (m->set_src, seq);
2123 i1 = loop_insn_hoist (loop, seq);
2124 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2125 set_unique_reg_note (i1, m->is_equiv ? REG_EQUIV
2126 : REG_EQUAL, m->set_src);
2128 else if (m->insert_temp)
2130 rtx *reg_map2 = xcalloc (REGNO (newreg),
2132 reg_map2 [m->regno] = newreg;
2134 i1 = loop_insn_hoist (loop, copy_rtx (PATTERN (p)));
2135 replace_regs (i1, reg_map2, REGNO (newreg), 1);
2139 i1 = loop_insn_hoist (loop, PATTERN (p));
2141 if (REG_NOTES (i1) == 0)
2143 REG_NOTES (i1) = REG_NOTES (p);
2144 REG_NOTES (p) = NULL;
2146 /* If there is a REG_EQUAL note present whose value
2147 is not loop invariant, then delete it, since it
2148 may cause problems with later optimization passes.
2149 It is possible for cse to create such notes
2150 like this as a result of record_jump_cond. */
2152 if ((temp = find_reg_note (i1, REG_EQUAL, NULL_RTX))
2153 && ! loop_invariant_p (loop, XEXP (temp, 0)))
2154 remove_note (i1, temp);
2160 if (loop_dump_stream)
2161 fprintf (loop_dump_stream, " moved to %d",
2164 /* If library call, now fix the REG_NOTES that contain
2165 insn pointers, namely REG_LIBCALL on FIRST
2166 and REG_RETVAL on I1. */
2167 if ((temp = find_reg_note (i1, REG_RETVAL, NULL_RTX)))
2169 XEXP (temp, 0) = first;
2170 temp = find_reg_note (first, REG_LIBCALL, NULL_RTX);
2171 XEXP (temp, 0) = i1;
2178 /* simplify_giv_expr expects that it can walk the insns
2179 at m->insn forwards and see this old sequence we are
2180 tossing here. delete_insn does preserve the next
2181 pointers, but when we skip over a NOTE we must fix
2182 it up. Otherwise that code walks into the non-deleted
2184 while (p && GET_CODE (p) == NOTE)
2185 p = NEXT_INSN (temp) = NEXT_INSN (p);
2190 /* Replace the original insn with a move from
2191 our newly created temp. */
2193 emit_move_insn (m->set_dest, newreg);
2196 emit_insn_before (seq, p);
2200 /* The more regs we move, the less we like moving them. */
2206 if (!m->insert_temp)
2208 /* Any other movable that loads the same register
2210 already_moved[regno] = 1;
2212 /* This reg has been moved out of one loop. */
2213 regs->array[regno].moved_once = 1;
2215 /* The reg set here is now invariant. */
2219 for (i = 0; i < LOOP_REGNO_NREGS (regno, m->set_dest); i++)
2220 regs->array[regno+i].set_in_loop = 0;
2223 /* Change the length-of-life info for the register
2224 to say it lives at least the full length of this loop.
2225 This will help guide optimizations in outer loops. */
2227 if (REGNO_FIRST_LUID (regno) > INSN_LUID (loop_start))
2228 /* This is the old insn before all the moved insns.
2229 We can't use the moved insn because it is out of range
2230 in uid_luid. Only the old insns have luids. */
2231 REGNO_FIRST_UID (regno) = INSN_UID (loop_start);
2232 if (REGNO_LAST_LUID (regno) < INSN_LUID (loop_end))
2233 REGNO_LAST_UID (regno) = INSN_UID (loop_end);
2236 /* Combine with this moved insn any other matching movables. */
2239 for (m1 = movables->head; m1; m1 = m1->next)
2244 /* Schedule the reg loaded by M1
2245 for replacement so that shares the reg of M.
2246 If the modes differ (only possible in restricted
2247 circumstances, make a SUBREG.
2249 Note this assumes that the target dependent files
2250 treat REG and SUBREG equally, including within
2251 GO_IF_LEGITIMATE_ADDRESS and in all the
2252 predicates since we never verify that replacing the
2253 original register with a SUBREG results in a
2254 recognizable insn. */
2255 if (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest))
2256 reg_map[m1->regno] = m->set_dest;
2259 = gen_lowpart_common (GET_MODE (m1->set_dest),
2262 /* Get rid of the matching insn
2263 and prevent further processing of it. */
2266 /* If library call, delete all insns. */
2267 if ((temp = find_reg_note (m1->insn, REG_RETVAL,
2269 delete_insn_chain (XEXP (temp, 0), m1->insn);
2271 delete_insn (m1->insn);
2273 /* Any other movable that loads the same register
2275 already_moved[m1->regno] = 1;
2277 /* The reg merged here is now invariant,
2278 if the reg it matches is invariant. */
2283 i < LOOP_REGNO_NREGS (regno, m1->set_dest);
2285 regs->array[m1->regno+i].set_in_loop = 0;
2289 else if (loop_dump_stream)
2290 fprintf (loop_dump_stream, "not desirable");
2292 else if (loop_dump_stream && !m->match)
2293 fprintf (loop_dump_stream, "not safe");
2295 if (loop_dump_stream)
2296 fprintf (loop_dump_stream, "\n");
2300 new_start = loop_start;
2302 /* Go through all the instructions in the loop, making
2303 all the register substitutions scheduled in REG_MAP. */
2304 for (p = new_start; p != loop_end; p = NEXT_INSN (p))
2305 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
2306 || GET_CODE (p) == CALL_INSN)
2308 replace_regs (PATTERN (p), reg_map, nregs, 0);
2309 replace_regs (REG_NOTES (p), reg_map, nregs, 0);
2315 free (already_moved);
2320 loop_movables_add (struct loop_movables *movables, struct movable *m)
2322 if (movables->head == 0)
2325 movables->last->next = m;
2331 loop_movables_free (struct loop_movables *movables)
2334 struct movable *m_next;
2336 for (m = movables->head; m; m = m_next)
2344 /* Scan X and replace the address of any MEM in it with ADDR.
2345 REG is the address that MEM should have before the replacement. */
2348 replace_call_address (rtx x, rtx reg, rtx addr)
2356 code = GET_CODE (x);
2370 /* Short cut for very common case. */
2371 replace_call_address (XEXP (x, 1), reg, addr);
2375 /* Short cut for very common case. */
2376 replace_call_address (XEXP (x, 0), reg, addr);
2380 /* If this MEM uses a reg other than the one we expected,
2381 something is wrong. */
2382 if (XEXP (x, 0) != reg)
2391 fmt = GET_RTX_FORMAT (code);
2392 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2395 replace_call_address (XEXP (x, i), reg, addr);
2396 else if (fmt[i] == 'E')
2399 for (j = 0; j < XVECLEN (x, i); j++)
2400 replace_call_address (XVECEXP (x, i, j), reg, addr);
2406 /* Return the number of memory refs to addresses that vary
2410 count_nonfixed_reads (const struct loop *loop, rtx x)
2420 code = GET_CODE (x);
2434 return ((loop_invariant_p (loop, XEXP (x, 0)) != 1)
2435 + count_nonfixed_reads (loop, XEXP (x, 0)));
2442 fmt = GET_RTX_FORMAT (code);
2443 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2446 value += count_nonfixed_reads (loop, XEXP (x, i));
2450 for (j = 0; j < XVECLEN (x, i); j++)
2451 value += count_nonfixed_reads (loop, XVECEXP (x, i, j));
2457 /* Scan a loop setting the elements `cont', `vtop', `loops_enclosed',
2458 `has_call', `has_nonconst_call', `has_volatile', `has_tablejump',
2459 `unknown_address_altered', `unknown_constant_address_altered', and
2460 `num_mem_sets' in LOOP. Also, fill in the array `mems' and the
2461 list `store_mems' in LOOP. */
2464 prescan_loop (struct loop *loop)
2468 struct loop_info *loop_info = LOOP_INFO (loop);
2469 rtx start = loop->start;
2470 rtx end = loop->end;
2471 /* The label after END. Jumping here is just like falling off the
2472 end of the loop. We use next_nonnote_insn instead of next_label
2473 as a hedge against the (pathological) case where some actual insn
2474 might end up between the two. */
2475 rtx exit_target = next_nonnote_insn (end);
2477 loop_info->has_indirect_jump = indirect_jump_in_function;
2478 loop_info->pre_header_has_call = 0;
2479 loop_info->has_call = 0;
2480 loop_info->has_nonconst_call = 0;
2481 loop_info->has_prefetch = 0;
2482 loop_info->has_volatile = 0;
2483 loop_info->has_tablejump = 0;
2484 loop_info->has_multiple_exit_targets = 0;
2487 loop_info->unknown_address_altered = 0;
2488 loop_info->unknown_constant_address_altered = 0;
2489 loop_info->store_mems = NULL_RTX;
2490 loop_info->first_loop_store_insn = NULL_RTX;
2491 loop_info->mems_idx = 0;
2492 loop_info->num_mem_sets = 0;
2493 /* If loop opts run twice, this was set on 1st pass for 2nd. */
2494 loop_info->preconditioned = NOTE_PRECONDITIONED (end);
2496 for (insn = start; insn && GET_CODE (insn) != CODE_LABEL;
2497 insn = PREV_INSN (insn))
2499 if (GET_CODE (insn) == CALL_INSN)
2501 loop_info->pre_header_has_call = 1;
2506 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2507 insn = NEXT_INSN (insn))
2509 switch (GET_CODE (insn))
2512 if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
2515 /* Count number of loops contained in this one. */
2518 else if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_END)
2523 if (! CONST_OR_PURE_CALL_P (insn))
2525 loop_info->unknown_address_altered = 1;
2526 loop_info->has_nonconst_call = 1;
2528 else if (pure_call_p (insn))
2529 loop_info->has_nonconst_call = 1;
2530 loop_info->has_call = 1;
2531 if (can_throw_internal (insn))
2532 loop_info->has_multiple_exit_targets = 1;
2534 /* Calls initializing constant objects have CLOBBER of MEM /u in the
2535 attached FUNCTION_USAGE expression list, not accounted for by the
2536 code above. We should note these to avoid missing dependencies in
2537 later references. */
2541 for (fusage_entry = CALL_INSN_FUNCTION_USAGE (insn);
2542 fusage_entry; fusage_entry = XEXP (fusage_entry, 1))
2544 rtx fusage = XEXP (fusage_entry, 0);
2546 if (GET_CODE (fusage) == CLOBBER
2547 && GET_CODE (XEXP (fusage, 0)) == MEM
2548 && RTX_UNCHANGING_P (XEXP (fusage, 0)))
2550 note_stores (fusage, note_addr_stored, loop_info);
2551 if (! loop_info->first_loop_store_insn
2552 && loop_info->store_mems)
2553 loop_info->first_loop_store_insn = insn;
2560 if (! loop_info->has_multiple_exit_targets)
2562 rtx set = pc_set (insn);
2566 rtx src = SET_SRC (set);
2569 if (GET_CODE (src) == IF_THEN_ELSE)
2571 label1 = XEXP (src, 1);
2572 label2 = XEXP (src, 2);
2582 if (label1 && label1 != pc_rtx)
2584 if (GET_CODE (label1) != LABEL_REF)
2586 /* Something tricky. */
2587 loop_info->has_multiple_exit_targets = 1;
2590 else if (XEXP (label1, 0) != exit_target
2591 && LABEL_OUTSIDE_LOOP_P (label1))
2593 /* A jump outside the current loop. */
2594 loop_info->has_multiple_exit_targets = 1;
2606 /* A return, or something tricky. */
2607 loop_info->has_multiple_exit_targets = 1;
2613 if (volatile_refs_p (PATTERN (insn)))
2614 loop_info->has_volatile = 1;
2616 if (GET_CODE (insn) == JUMP_INSN
2617 && (GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC
2618 || GET_CODE (PATTERN (insn)) == ADDR_VEC))
2619 loop_info->has_tablejump = 1;
2621 note_stores (PATTERN (insn), note_addr_stored, loop_info);
2622 if (! loop_info->first_loop_store_insn && loop_info->store_mems)
2623 loop_info->first_loop_store_insn = insn;
2625 if (flag_non_call_exceptions && can_throw_internal (insn))
2626 loop_info->has_multiple_exit_targets = 1;
2634 /* Now, rescan the loop, setting up the LOOP_MEMS array. */
2635 if (/* An exception thrown by a called function might land us
2637 ! loop_info->has_nonconst_call
2638 /* We don't want loads for MEMs moved to a location before the
2639 one at which their stack memory becomes allocated. (Note
2640 that this is not a problem for malloc, etc., since those
2641 require actual function calls. */
2642 && ! current_function_calls_alloca
2643 /* There are ways to leave the loop other than falling off the
2645 && ! loop_info->has_multiple_exit_targets)
2646 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2647 insn = NEXT_INSN (insn))
2648 for_each_rtx (&insn, insert_loop_mem, loop_info);
2650 /* BLKmode MEMs are added to LOOP_STORE_MEM as necessary so
2651 that loop_invariant_p and load_mems can use true_dependence
2652 to determine what is really clobbered. */
2653 if (loop_info->unknown_address_altered)
2655 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2657 loop_info->store_mems
2658 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2660 if (loop_info->unknown_constant_address_altered)
2662 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2664 RTX_UNCHANGING_P (mem) = 1;
2665 loop_info->store_mems
2666 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2670 /* Invalidate all loops containing LABEL. */
2673 invalidate_loops_containing_label (rtx label)
2676 for (loop = uid_loop[INSN_UID (label)]; loop; loop = loop->outer)
2680 /* Scan the function looking for loops. Record the start and end of each loop.
2681 Also mark as invalid loops any loops that contain a setjmp or are branched
2682 to from outside the loop. */
2685 find_and_verify_loops (rtx f, struct loops *loops)
2690 struct loop *current_loop;
2691 struct loop *next_loop;
2694 num_loops = loops->num;
2696 compute_luids (f, NULL_RTX, 0);
2698 /* If there are jumps to undefined labels,
2699 treat them as jumps out of any/all loops.
2700 This also avoids writing past end of tables when there are no loops. */
2703 /* Find boundaries of loops, mark which loops are contained within
2704 loops, and invalidate loops that have setjmp. */
2707 current_loop = NULL;
2708 for (insn = f; insn; insn = NEXT_INSN (insn))
2710 if (GET_CODE (insn) == NOTE)
2711 switch (NOTE_LINE_NUMBER (insn))
2713 case NOTE_INSN_LOOP_BEG:
2714 next_loop = loops->array + num_loops;
2715 next_loop->num = num_loops;
2717 next_loop->start = insn;
2718 next_loop->outer = current_loop;
2719 current_loop = next_loop;
2722 case NOTE_INSN_LOOP_CONT:
2723 current_loop->cont = insn;
2726 case NOTE_INSN_LOOP_VTOP:
2727 current_loop->vtop = insn;
2730 case NOTE_INSN_LOOP_END:
2734 current_loop->end = insn;
2735 current_loop = current_loop->outer;
2742 if (GET_CODE (insn) == CALL_INSN
2743 && find_reg_note (insn, REG_SETJMP, NULL))
2745 /* In this case, we must invalidate our current loop and any
2747 for (loop = current_loop; loop; loop = loop->outer)
2750 if (loop_dump_stream)
2751 fprintf (loop_dump_stream,
2752 "\nLoop at %d ignored due to setjmp.\n",
2753 INSN_UID (loop->start));
2757 /* Note that this will mark the NOTE_INSN_LOOP_END note as being in the
2758 enclosing loop, but this doesn't matter. */
2759 uid_loop[INSN_UID (insn)] = current_loop;
2762 /* Any loop containing a label used in an initializer must be invalidated,
2763 because it can be jumped into from anywhere. */
2764 for (label = forced_labels; label; label = XEXP (label, 1))
2765 invalidate_loops_containing_label (XEXP (label, 0));
2767 /* Any loop containing a label used for an exception handler must be
2768 invalidated, because it can be jumped into from anywhere. */
2769 for_each_eh_label (invalidate_loops_containing_label);
2771 /* Now scan all insn's in the function. If any JUMP_INSN branches into a
2772 loop that it is not contained within, that loop is marked invalid.
2773 If any INSN or CALL_INSN uses a label's address, then the loop containing
2774 that label is marked invalid, because it could be jumped into from
2777 Also look for blocks of code ending in an unconditional branch that
2778 exits the loop. If such a block is surrounded by a conditional
2779 branch around the block, move the block elsewhere (see below) and
2780 invert the jump to point to the code block. This may eliminate a
2781 label in our loop and will simplify processing by both us and a
2782 possible second cse pass. */
2784 for (insn = f; insn; insn = NEXT_INSN (insn))
2787 struct loop *this_loop = uid_loop[INSN_UID (insn)];
2789 if (GET_CODE (insn) == INSN || GET_CODE (insn) == CALL_INSN)
2791 rtx note = find_reg_note (insn, REG_LABEL, NULL_RTX);
2793 invalidate_loops_containing_label (XEXP (note, 0));
2796 if (GET_CODE (insn) != JUMP_INSN)
2799 mark_loop_jump (PATTERN (insn), this_loop);
2801 /* See if this is an unconditional branch outside the loop. */
2803 && (GET_CODE (PATTERN (insn)) == RETURN
2804 || (any_uncondjump_p (insn)
2805 && onlyjump_p (insn)
2806 && (uid_loop[INSN_UID (JUMP_LABEL (insn))]
2808 && get_max_uid () < max_uid_for_loop)
2811 rtx our_next = next_real_insn (insn);
2812 rtx last_insn_to_move = NEXT_INSN (insn);
2813 struct loop *dest_loop;
2814 struct loop *outer_loop = NULL;
2816 /* Go backwards until we reach the start of the loop, a label,
2818 for (p = PREV_INSN (insn);
2819 GET_CODE (p) != CODE_LABEL
2820 && ! (GET_CODE (p) == NOTE
2821 && NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
2822 && GET_CODE (p) != JUMP_INSN;
2826 /* Check for the case where we have a jump to an inner nested
2827 loop, and do not perform the optimization in that case. */
2829 if (JUMP_LABEL (insn))
2831 dest_loop = uid_loop[INSN_UID (JUMP_LABEL (insn))];
2834 for (outer_loop = dest_loop; outer_loop;
2835 outer_loop = outer_loop->outer)
2836 if (outer_loop == this_loop)
2841 /* Make sure that the target of P is within the current loop. */
2843 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
2844 && uid_loop[INSN_UID (JUMP_LABEL (p))] != this_loop)
2845 outer_loop = this_loop;
2847 /* If we stopped on a JUMP_INSN to the next insn after INSN,
2848 we have a block of code to try to move.
2850 We look backward and then forward from the target of INSN
2851 to find a BARRIER at the same loop depth as the target.
2852 If we find such a BARRIER, we make a new label for the start
2853 of the block, invert the jump in P and point it to that label,
2854 and move the block of code to the spot we found. */
2857 && GET_CODE (p) == JUMP_INSN
2858 && JUMP_LABEL (p) != 0
2859 /* Just ignore jumps to labels that were never emitted.
2860 These always indicate compilation errors. */
2861 && INSN_UID (JUMP_LABEL (p)) != 0
2862 && any_condjump_p (p) && onlyjump_p (p)
2863 && next_real_insn (JUMP_LABEL (p)) == our_next
2864 /* If it's not safe to move the sequence, then we
2866 && insns_safe_to_move_p (p, NEXT_INSN (insn),
2867 &last_insn_to_move))
2870 = JUMP_LABEL (insn) ? JUMP_LABEL (insn) : get_last_insn ();
2871 struct loop *target_loop = uid_loop[INSN_UID (target)];
2875 /* Search for possible garbage past the conditional jumps
2876 and look for the last barrier. */
2877 for (tmp = last_insn_to_move;
2878 tmp && GET_CODE (tmp) != CODE_LABEL; tmp = NEXT_INSN (tmp))
2879 if (GET_CODE (tmp) == BARRIER)
2880 last_insn_to_move = tmp;
2882 for (loc = target; loc; loc = PREV_INSN (loc))
2883 if (GET_CODE (loc) == BARRIER
2884 /* Don't move things inside a tablejump. */
2885 && ((loc2 = next_nonnote_insn (loc)) == 0
2886 || GET_CODE (loc2) != CODE_LABEL
2887 || (loc2 = next_nonnote_insn (loc2)) == 0
2888 || GET_CODE (loc2) != JUMP_INSN
2889 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2890 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2891 && uid_loop[INSN_UID (loc)] == target_loop)
2895 for (loc = target; loc; loc = NEXT_INSN (loc))
2896 if (GET_CODE (loc) == BARRIER
2897 /* Don't move things inside a tablejump. */
2898 && ((loc2 = next_nonnote_insn (loc)) == 0
2899 || GET_CODE (loc2) != CODE_LABEL
2900 || (loc2 = next_nonnote_insn (loc2)) == 0
2901 || GET_CODE (loc2) != JUMP_INSN
2902 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2903 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2904 && uid_loop[INSN_UID (loc)] == target_loop)
2909 rtx cond_label = JUMP_LABEL (p);
2910 rtx new_label = get_label_after (p);
2912 /* Ensure our label doesn't go away. */
2913 LABEL_NUSES (cond_label)++;
2915 /* Verify that uid_loop is large enough and that
2917 if (invert_jump (p, new_label, 1))
2921 /* If no suitable BARRIER was found, create a suitable
2922 one before TARGET. Since TARGET is a fall through
2923 path, we'll need to insert a jump around our block
2924 and add a BARRIER before TARGET.
2926 This creates an extra unconditional jump outside
2927 the loop. However, the benefits of removing rarely
2928 executed instructions from inside the loop usually
2929 outweighs the cost of the extra unconditional jump
2930 outside the loop. */
2935 temp = gen_jump (JUMP_LABEL (insn));
2936 temp = emit_jump_insn_before (temp, target);
2937 JUMP_LABEL (temp) = JUMP_LABEL (insn);
2938 LABEL_NUSES (JUMP_LABEL (insn))++;
2939 loc = emit_barrier_before (target);
2942 /* Include the BARRIER after INSN and copy the
2944 if (squeeze_notes (&new_label, &last_insn_to_move))
2946 reorder_insns (new_label, last_insn_to_move, loc);
2948 /* All those insns are now in TARGET_LOOP. */
2950 q != NEXT_INSN (last_insn_to_move);
2952 uid_loop[INSN_UID (q)] = target_loop;
2954 /* The label jumped to by INSN is no longer a loop
2955 exit. Unless INSN does not have a label (e.g.,
2956 it is a RETURN insn), search loop->exit_labels
2957 to find its label_ref, and remove it. Also turn
2958 off LABEL_OUTSIDE_LOOP_P bit. */
2959 if (JUMP_LABEL (insn))
2961 for (q = 0, r = this_loop->exit_labels;
2963 q = r, r = LABEL_NEXTREF (r))
2964 if (XEXP (r, 0) == JUMP_LABEL (insn))
2966 LABEL_OUTSIDE_LOOP_P (r) = 0;
2968 LABEL_NEXTREF (q) = LABEL_NEXTREF (r);
2970 this_loop->exit_labels = LABEL_NEXTREF (r);
2974 for (loop = this_loop; loop && loop != target_loop;
2978 /* If we didn't find it, then something is
2984 /* P is now a jump outside the loop, so it must be put
2985 in loop->exit_labels, and marked as such.
2986 The easiest way to do this is to just call
2987 mark_loop_jump again for P. */
2988 mark_loop_jump (PATTERN (p), this_loop);
2990 /* If INSN now jumps to the insn after it,
2992 if (JUMP_LABEL (insn) != 0
2993 && (next_real_insn (JUMP_LABEL (insn))
2994 == next_real_insn (insn)))
2995 delete_related_insns (insn);
2998 /* Continue the loop after where the conditional
2999 branch used to jump, since the only branch insn
3000 in the block (if it still remains) is an inter-loop
3001 branch and hence needs no processing. */
3002 insn = NEXT_INSN (cond_label);
3004 if (--LABEL_NUSES (cond_label) == 0)
3005 delete_related_insns (cond_label);
3007 /* This loop will be continued with NEXT_INSN (insn). */
3008 insn = PREV_INSN (insn);
3015 /* If any label in X jumps to a loop different from LOOP_NUM and any of the
3016 loops it is contained in, mark the target loop invalid.
3018 For speed, we assume that X is part of a pattern of a JUMP_INSN. */
3021 mark_loop_jump (rtx x, struct loop *loop)
3023 struct loop *dest_loop;
3024 struct loop *outer_loop;
3027 switch (GET_CODE (x))
3040 /* There could be a label reference in here. */
3041 mark_loop_jump (XEXP (x, 0), loop);
3047 mark_loop_jump (XEXP (x, 0), loop);
3048 mark_loop_jump (XEXP (x, 1), loop);
3052 /* This may refer to a LABEL_REF or SYMBOL_REF. */
3053 mark_loop_jump (XEXP (x, 1), loop);
3058 mark_loop_jump (XEXP (x, 0), loop);
3062 dest_loop = uid_loop[INSN_UID (XEXP (x, 0))];
3064 /* Link together all labels that branch outside the loop. This
3065 is used by final_[bg]iv_value and the loop unrolling code. Also
3066 mark this LABEL_REF so we know that this branch should predict
3069 /* A check to make sure the label is not in an inner nested loop,
3070 since this does not count as a loop exit. */
3073 for (outer_loop = dest_loop; outer_loop;
3074 outer_loop = outer_loop->outer)
3075 if (outer_loop == loop)
3081 if (loop && ! outer_loop)
3083 LABEL_OUTSIDE_LOOP_P (x) = 1;
3084 LABEL_NEXTREF (x) = loop->exit_labels;
3085 loop->exit_labels = x;
3087 for (outer_loop = loop;
3088 outer_loop && outer_loop != dest_loop;
3089 outer_loop = outer_loop->outer)
3090 outer_loop->exit_count++;
3093 /* If this is inside a loop, but not in the current loop or one enclosed
3094 by it, it invalidates at least one loop. */
3099 /* We must invalidate every nested loop containing the target of this
3100 label, except those that also contain the jump insn. */
3102 for (; dest_loop; dest_loop = dest_loop->outer)
3104 /* Stop when we reach a loop that also contains the jump insn. */
3105 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3106 if (dest_loop == outer_loop)
3109 /* If we get here, we know we need to invalidate a loop. */
3110 if (loop_dump_stream && ! dest_loop->invalid)
3111 fprintf (loop_dump_stream,
3112 "\nLoop at %d ignored due to multiple entry points.\n",
3113 INSN_UID (dest_loop->start));
3115 dest_loop->invalid = 1;
3120 /* If this is not setting pc, ignore. */
3121 if (SET_DEST (x) == pc_rtx)
3122 mark_loop_jump (SET_SRC (x), loop);
3126 mark_loop_jump (XEXP (x, 1), loop);
3127 mark_loop_jump (XEXP (x, 2), loop);
3132 for (i = 0; i < XVECLEN (x, 0); i++)
3133 mark_loop_jump (XVECEXP (x, 0, i), loop);
3137 for (i = 0; i < XVECLEN (x, 1); i++)
3138 mark_loop_jump (XVECEXP (x, 1, i), loop);
3142 /* Strictly speaking this is not a jump into the loop, only a possible
3143 jump out of the loop. However, we have no way to link the destination
3144 of this jump onto the list of exit labels. To be safe we mark this
3145 loop and any containing loops as invalid. */
3148 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3150 if (loop_dump_stream && ! outer_loop->invalid)
3151 fprintf (loop_dump_stream,
3152 "\nLoop at %d ignored due to unknown exit jump.\n",
3153 INSN_UID (outer_loop->start));
3154 outer_loop->invalid = 1;
3161 /* Return nonzero if there is a label in the range from
3162 insn INSN to and including the insn whose luid is END
3163 INSN must have an assigned luid (i.e., it must not have
3164 been previously created by loop.c). */
3167 labels_in_range_p (rtx insn, int end)
3169 while (insn && INSN_LUID (insn) <= end)
3171 if (GET_CODE (insn) == CODE_LABEL)
3173 insn = NEXT_INSN (insn);
3179 /* Record that a memory reference X is being set. */
3182 note_addr_stored (rtx x, rtx y ATTRIBUTE_UNUSED,
3183 void *data ATTRIBUTE_UNUSED)
3185 struct loop_info *loop_info = data;
3187 if (x == 0 || GET_CODE (x) != MEM)
3190 /* Count number of memory writes.
3191 This affects heuristics in strength_reduce. */
3192 loop_info->num_mem_sets++;
3194 /* BLKmode MEM means all memory is clobbered. */
3195 if (GET_MODE (x) == BLKmode)
3197 if (RTX_UNCHANGING_P (x))
3198 loop_info->unknown_constant_address_altered = 1;
3200 loop_info->unknown_address_altered = 1;
3205 loop_info->store_mems = gen_rtx_EXPR_LIST (VOIDmode, x,
3206 loop_info->store_mems);
3209 /* X is a value modified by an INSN that references a biv inside a loop
3210 exit test (ie, X is somehow related to the value of the biv). If X
3211 is a pseudo that is used more than once, then the biv is (effectively)
3212 used more than once. DATA is a pointer to a loop_regs structure. */
3215 note_set_pseudo_multiple_uses (rtx x, rtx y ATTRIBUTE_UNUSED, void *data)
3217 struct loop_regs *regs = (struct loop_regs *) data;
3222 while (GET_CODE (x) == STRICT_LOW_PART
3223 || GET_CODE (x) == SIGN_EXTRACT
3224 || GET_CODE (x) == ZERO_EXTRACT
3225 || GET_CODE (x) == SUBREG)
3228 if (GET_CODE (x) != REG || REGNO (x) < FIRST_PSEUDO_REGISTER)
3231 /* If we do not have usage information, or if we know the register
3232 is used more than once, note that fact for check_dbra_loop. */
3233 if (REGNO (x) >= max_reg_before_loop
3234 || ! regs->array[REGNO (x)].single_usage
3235 || regs->array[REGNO (x)].single_usage == const0_rtx)
3236 regs->multiple_uses = 1;
3239 /* Return nonzero if the rtx X is invariant over the current loop.
3241 The value is 2 if we refer to something only conditionally invariant.
3243 A memory ref is invariant if it is not volatile and does not conflict
3244 with anything stored in `loop_info->store_mems'. */
3247 loop_invariant_p (const struct loop *loop, rtx x)
3249 struct loop_info *loop_info = LOOP_INFO (loop);
3250 struct loop_regs *regs = LOOP_REGS (loop);
3254 int conditional = 0;
3259 code = GET_CODE (x);
3269 /* A LABEL_REF is normally invariant, however, if we are unrolling
3270 loops, and this label is inside the loop, then it isn't invariant.
3271 This is because each unrolled copy of the loop body will have
3272 a copy of this label. If this was invariant, then an insn loading
3273 the address of this label into a register might get moved outside
3274 the loop, and then each loop body would end up using the same label.
3276 We don't know the loop bounds here though, so just fail for all
3278 if (flag_old_unroll_loops)
3285 case UNSPEC_VOLATILE:
3289 /* We used to check RTX_UNCHANGING_P (x) here, but that is invalid
3290 since the reg might be set by initialization within the loop. */
3292 if ((x == frame_pointer_rtx || x == hard_frame_pointer_rtx
3293 || x == arg_pointer_rtx || x == pic_offset_table_rtx)
3294 && ! current_function_has_nonlocal_goto)
3297 if (LOOP_INFO (loop)->has_call
3298 && REGNO (x) < FIRST_PSEUDO_REGISTER && call_used_regs[REGNO (x)])
3301 /* Out-of-range regs can occur when we are called from unrolling.
3302 These registers created by the unroller are set in the loop,
3303 hence are never invariant.
3304 Other out-of-range regs can be generated by load_mems; those that
3305 are written to in the loop are not invariant, while those that are
3306 not written to are invariant. It would be easy for load_mems
3307 to set n_times_set correctly for these registers, however, there
3308 is no easy way to distinguish them from registers created by the
3311 if (REGNO (x) >= (unsigned) regs->num)
3314 if (regs->array[REGNO (x)].set_in_loop < 0)
3317 return regs->array[REGNO (x)].set_in_loop == 0;
3320 /* Volatile memory references must be rejected. Do this before
3321 checking for read-only items, so that volatile read-only items
3322 will be rejected also. */
3323 if (MEM_VOLATILE_P (x))
3326 /* See if there is any dependence between a store and this load. */
3327 mem_list_entry = loop_info->store_mems;
3328 while (mem_list_entry)
3330 if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
3334 mem_list_entry = XEXP (mem_list_entry, 1);
3337 /* It's not invalidated by a store in memory
3338 but we must still verify the address is invariant. */
3342 /* Don't mess with insns declared volatile. */
3343 if (MEM_VOLATILE_P (x))
3351 fmt = GET_RTX_FORMAT (code);
3352 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3356 int tem = loop_invariant_p (loop, XEXP (x, i));
3362 else if (fmt[i] == 'E')
3365 for (j = 0; j < XVECLEN (x, i); j++)
3367 int tem = loop_invariant_p (loop, XVECEXP (x, i, j));
3377 return 1 + conditional;
3380 /* Return nonzero if all the insns in the loop that set REG
3381 are INSN and the immediately following insns,
3382 and if each of those insns sets REG in an invariant way
3383 (not counting uses of REG in them).
3385 The value is 2 if some of these insns are only conditionally invariant.
3387 We assume that INSN itself is the first set of REG
3388 and that its source is invariant. */
3391 consec_sets_invariant_p (const struct loop *loop, rtx reg, int n_sets,
3394 struct loop_regs *regs = LOOP_REGS (loop);
3396 unsigned int regno = REGNO (reg);
3398 /* Number of sets we have to insist on finding after INSN. */
3399 int count = n_sets - 1;
3400 int old = regs->array[regno].set_in_loop;
3404 /* If N_SETS hit the limit, we can't rely on its value. */
3408 regs->array[regno].set_in_loop = 0;
3416 code = GET_CODE (p);
3418 /* If library call, skip to end of it. */
3419 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
3424 && (set = single_set (p))
3425 && GET_CODE (SET_DEST (set)) == REG
3426 && REGNO (SET_DEST (set)) == regno)
3428 this = loop_invariant_p (loop, SET_SRC (set));
3431 else if ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX)))
3433 /* If this is a libcall, then any invariant REG_EQUAL note is OK.
3434 If this is an ordinary insn, then only CONSTANT_P REG_EQUAL
3436 this = (CONSTANT_P (XEXP (temp, 0))
3437 || (find_reg_note (p, REG_RETVAL, NULL_RTX)
3438 && loop_invariant_p (loop, XEXP (temp, 0))));
3445 else if (code != NOTE)
3447 regs->array[regno].set_in_loop = old;
3452 regs->array[regno].set_in_loop = old;
3453 /* If loop_invariant_p ever returned 2, we return 2. */
3454 return 1 + (value & 2);
3458 /* I don't think this condition is sufficient to allow INSN
3459 to be moved, so we no longer test it. */
3461 /* Return 1 if all insns in the basic block of INSN and following INSN
3462 that set REG are invariant according to TABLE. */
3465 all_sets_invariant_p (rtx reg, rtx insn, short *table)
3468 int regno = REGNO (reg);
3474 code = GET_CODE (p);
3475 if (code == CODE_LABEL || code == JUMP_INSN)
3477 if (code == INSN && GET_CODE (PATTERN (p)) == SET
3478 && GET_CODE (SET_DEST (PATTERN (p))) == REG
3479 && REGNO (SET_DEST (PATTERN (p))) == regno)
3481 if (! loop_invariant_p (loop, SET_SRC (PATTERN (p)), table))
3488 /* Look at all uses (not sets) of registers in X. For each, if it is
3489 the single use, set USAGE[REGNO] to INSN; if there was a previous use in
3490 a different insn, set USAGE[REGNO] to const0_rtx. */
3493 find_single_use_in_loop (struct loop_regs *regs, rtx insn, rtx x)
3495 enum rtx_code code = GET_CODE (x);
3496 const char *fmt = GET_RTX_FORMAT (code);
3500 regs->array[REGNO (x)].single_usage
3501 = (regs->array[REGNO (x)].single_usage != 0
3502 && regs->array[REGNO (x)].single_usage != insn)
3503 ? const0_rtx : insn;
3505 else if (code == SET)
3507 /* Don't count SET_DEST if it is a REG; otherwise count things
3508 in SET_DEST because if a register is partially modified, it won't
3509 show up as a potential movable so we don't care how USAGE is set
3511 if (GET_CODE (SET_DEST (x)) != REG)
3512 find_single_use_in_loop (regs, insn, SET_DEST (x));
3513 find_single_use_in_loop (regs, insn, SET_SRC (x));
3516 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3518 if (fmt[i] == 'e' && XEXP (x, i) != 0)
3519 find_single_use_in_loop (regs, insn, XEXP (x, i));
3520 else if (fmt[i] == 'E')
3521 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
3522 find_single_use_in_loop (regs, insn, XVECEXP (x, i, j));
3526 /* Count and record any set in X which is contained in INSN. Update
3527 REGS->array[I].MAY_NOT_OPTIMIZE and LAST_SET for any register I set
3531 count_one_set (struct loop_regs *regs, rtx insn, rtx x, rtx *last_set)
3533 if (GET_CODE (x) == CLOBBER && GET_CODE (XEXP (x, 0)) == REG)
3534 /* Don't move a reg that has an explicit clobber.
3535 It's not worth the pain to try to do it correctly. */
3536 regs->array[REGNO (XEXP (x, 0))].may_not_optimize = 1;
3538 if (GET_CODE (x) == SET || GET_CODE (x) == CLOBBER)
3540 rtx dest = SET_DEST (x);
3541 while (GET_CODE (dest) == SUBREG
3542 || GET_CODE (dest) == ZERO_EXTRACT
3543 || GET_CODE (dest) == SIGN_EXTRACT
3544 || GET_CODE (dest) == STRICT_LOW_PART)
3545 dest = XEXP (dest, 0);
3546 if (GET_CODE (dest) == REG)
3549 int regno = REGNO (dest);
3550 for (i = 0; i < LOOP_REGNO_NREGS (regno, dest); i++)
3552 /* If this is the first setting of this reg
3553 in current basic block, and it was set before,
3554 it must be set in two basic blocks, so it cannot
3555 be moved out of the loop. */
3556 if (regs->array[regno].set_in_loop > 0
3557 && last_set[regno] == 0)
3558 regs->array[regno+i].may_not_optimize = 1;
3559 /* If this is not first setting in current basic block,
3560 see if reg was used in between previous one and this.
3561 If so, neither one can be moved. */
3562 if (last_set[regno] != 0
3563 && reg_used_between_p (dest, last_set[regno], insn))
3564 regs->array[regno+i].may_not_optimize = 1;
3565 if (regs->array[regno+i].set_in_loop < 127)
3566 ++regs->array[regno+i].set_in_loop;
3567 last_set[regno+i] = insn;
3573 /* Given a loop that is bounded by LOOP->START and LOOP->END and that
3574 is entered at LOOP->SCAN_START, return 1 if the register set in SET
3575 contained in insn INSN is used by any insn that precedes INSN in
3576 cyclic order starting from the loop entry point.
3578 We don't want to use INSN_LUID here because if we restrict INSN to those
3579 that have a valid INSN_LUID, it means we cannot move an invariant out
3580 from an inner loop past two loops. */
3583 loop_reg_used_before_p (const struct loop *loop, rtx set, rtx insn)
3585 rtx reg = SET_DEST (set);
3588 /* Scan forward checking for register usage. If we hit INSN, we
3589 are done. Otherwise, if we hit LOOP->END, wrap around to LOOP->START. */
3590 for (p = loop->scan_start; p != insn; p = NEXT_INSN (p))
3592 if (INSN_P (p) && reg_overlap_mentioned_p (reg, PATTERN (p)))
3603 /* Information we collect about arrays that we might want to prefetch. */
3604 struct prefetch_info
3606 struct iv_class *class; /* Class this prefetch is based on. */
3607 struct induction *giv; /* GIV this prefetch is based on. */
3608 rtx base_address; /* Start prefetching from this address plus
3610 HOST_WIDE_INT index;
3611 HOST_WIDE_INT stride; /* Prefetch stride in bytes in each
3613 unsigned int bytes_accessed; /* Sum of sizes of all accesses to this
3614 prefetch area in one iteration. */
3615 unsigned int total_bytes; /* Total bytes loop will access in this block.
3616 This is set only for loops with known
3617 iteration counts and is 0xffffffff
3619 int prefetch_in_loop; /* Number of prefetch insns in loop. */
3620 int prefetch_before_loop; /* Number of prefetch insns before loop. */
3621 unsigned int write : 1; /* 1 for read/write prefetches. */
3624 /* Data used by check_store function. */
3625 struct check_store_data
3631 static void check_store (rtx, rtx, void *);
3632 static void emit_prefetch_instructions (struct loop *);
3633 static int rtx_equal_for_prefetch_p (rtx, rtx);
3635 /* Set mem_write when mem_address is found. Used as callback to
3638 check_store (rtx x, rtx pat ATTRIBUTE_UNUSED, void *data)
3640 struct check_store_data *d = (struct check_store_data *) data;
3642 if ((GET_CODE (x) == MEM) && rtx_equal_p (d->mem_address, XEXP (x, 0)))
3646 /* Like rtx_equal_p, but attempts to swap commutative operands. This is
3647 important to get some addresses combined. Later more sophisticated
3648 transformations can be added when necessary.
3650 ??? Same trick with swapping operand is done at several other places.
3651 It can be nice to develop some common way to handle this. */
3654 rtx_equal_for_prefetch_p (rtx x, rtx y)
3658 enum rtx_code code = GET_CODE (x);
3663 if (code != GET_CODE (y))
3666 code = GET_CODE (x);
3668 if (GET_RTX_CLASS (code) == 'c')
3670 return ((rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 0))
3671 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 1)))
3672 || (rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 1))
3673 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 0))));
3675 /* Compare the elements. If any pair of corresponding elements fails to
3676 match, return 0 for the whole thing. */
3678 fmt = GET_RTX_FORMAT (code);
3679 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3684 if (XWINT (x, i) != XWINT (y, i))
3689 if (XINT (x, i) != XINT (y, i))
3694 /* Two vectors must have the same length. */
3695 if (XVECLEN (x, i) != XVECLEN (y, i))
3698 /* And the corresponding elements must match. */
3699 for (j = 0; j < XVECLEN (x, i); j++)
3700 if (rtx_equal_for_prefetch_p (XVECEXP (x, i, j),
3701 XVECEXP (y, i, j)) == 0)
3706 if (rtx_equal_for_prefetch_p (XEXP (x, i), XEXP (y, i)) == 0)
3711 if (strcmp (XSTR (x, i), XSTR (y, i)))
3716 /* These are just backpointers, so they don't matter. */
3722 /* It is believed that rtx's at this level will never
3723 contain anything but integers and other rtx's,
3724 except for within LABEL_REFs and SYMBOL_REFs. */
3732 /* Remove constant addition value from the expression X (when present)
3735 static HOST_WIDE_INT
3736 remove_constant_addition (rtx *x)
3738 HOST_WIDE_INT addval = 0;
3741 /* Avoid clobbering a shared CONST expression. */
3742 if (GET_CODE (exp) == CONST)
3744 if (GET_CODE (XEXP (exp, 0)) == PLUS
3745 && GET_CODE (XEXP (XEXP (exp, 0), 0)) == SYMBOL_REF
3746 && GET_CODE (XEXP (XEXP (exp, 0), 1)) == CONST_INT)
3748 *x = XEXP (XEXP (exp, 0), 0);
3749 return INTVAL (XEXP (XEXP (exp, 0), 1));
3754 if (GET_CODE (exp) == CONST_INT)
3756 addval = INTVAL (exp);
3760 /* For plus expression recurse on ourself. */
3761 else if (GET_CODE (exp) == PLUS)
3763 addval += remove_constant_addition (&XEXP (exp, 0));
3764 addval += remove_constant_addition (&XEXP (exp, 1));
3766 /* In case our parameter was constant, remove extra zero from the
3768 if (XEXP (exp, 0) == const0_rtx)
3770 else if (XEXP (exp, 1) == const0_rtx)
3777 /* Attempt to identify accesses to arrays that are most likely to cause cache
3778 misses, and emit prefetch instructions a few prefetch blocks forward.
3780 To detect the arrays we use the GIV information that was collected by the
3781 strength reduction pass.
3783 The prefetch instructions are generated after the GIV information is done
3784 and before the strength reduction process. The new GIVs are injected into
3785 the strength reduction tables, so the prefetch addresses are optimized as
3788 GIVs are split into base address, stride, and constant addition values.
3789 GIVs with the same address, stride and close addition values are combined
3790 into a single prefetch. Also writes to GIVs are detected, so that prefetch
3791 for write instructions can be used for the block we write to, on machines
3792 that support write prefetches.
3794 Several heuristics are used to determine when to prefetch. They are
3795 controlled by defined symbols that can be overridden for each target. */
3798 emit_prefetch_instructions (struct loop *loop)
3800 int num_prefetches = 0;
3801 int num_real_prefetches = 0;
3802 int num_real_write_prefetches = 0;
3803 int num_prefetches_before = 0;
3804 int num_write_prefetches_before = 0;
3807 struct iv_class *bl;
3808 struct induction *iv;
3809 struct prefetch_info info[MAX_PREFETCHES];
3810 struct loop_ivs *ivs = LOOP_IVS (loop);
3815 /* Consider only loops w/o calls. When a call is done, the loop is probably
3816 slow enough to read the memory. */
3817 if (PREFETCH_NO_CALL && LOOP_INFO (loop)->has_call)
3819 if (loop_dump_stream)
3820 fprintf (loop_dump_stream, "Prefetch: ignoring loop: has call.\n");
3825 /* Don't prefetch in loops known to have few iterations. */
3826 if (PREFETCH_NO_LOW_LOOPCNT
3827 && LOOP_INFO (loop)->n_iterations
3828 && LOOP_INFO (loop)->n_iterations <= PREFETCH_LOW_LOOPCNT)
3830 if (loop_dump_stream)
3831 fprintf (loop_dump_stream,
3832 "Prefetch: ignoring loop: not enough iterations.\n");
3836 /* Search all induction variables and pick those interesting for the prefetch
3838 for (bl = ivs->list; bl; bl = bl->next)
3840 struct induction *biv = bl->biv, *biv1;
3845 /* Expect all BIVs to be executed in each iteration. This makes our
3846 analysis more conservative. */
3849 /* Discard non-constant additions that we can't handle well yet, and
3850 BIVs that are executed multiple times; such BIVs ought to be
3851 handled in the nested loop. We accept not_every_iteration BIVs,
3852 since these only result in larger strides and make our
3853 heuristics more conservative. */
3854 if (GET_CODE (biv->add_val) != CONST_INT)
3856 if (loop_dump_stream)
3858 fprintf (loop_dump_stream,
3859 "Prefetch: ignoring biv %d: non-constant addition at insn %d:",
3860 REGNO (biv->src_reg), INSN_UID (biv->insn));
3861 print_rtl (loop_dump_stream, biv->add_val);
3862 fprintf (loop_dump_stream, "\n");
3867 if (biv->maybe_multiple)
3869 if (loop_dump_stream)
3871 fprintf (loop_dump_stream,
3872 "Prefetch: ignoring biv %d: maybe_multiple at insn %i:",
3873 REGNO (biv->src_reg), INSN_UID (biv->insn));
3874 print_rtl (loop_dump_stream, biv->add_val);
3875 fprintf (loop_dump_stream, "\n");
3880 basestride += INTVAL (biv1->add_val);
3881 biv1 = biv1->next_iv;
3884 if (biv1 || !basestride)
3887 for (iv = bl->giv; iv; iv = iv->next_iv)
3891 HOST_WIDE_INT index = 0;
3893 HOST_WIDE_INT stride = 0;
3894 int stride_sign = 1;
3895 struct check_store_data d;
3896 const char *ignore_reason = NULL;
3897 int size = GET_MODE_SIZE (GET_MODE (iv));
3899 /* See whether an induction variable is interesting to us and if
3900 not, report the reason. */
3901 if (iv->giv_type != DEST_ADDR)
3902 ignore_reason = "giv is not a destination address";
3904 /* We are interested only in constant stride memory references
3905 in order to be able to compute density easily. */
3906 else if (GET_CODE (iv->mult_val) != CONST_INT)
3907 ignore_reason = "stride is not constant";
3911 stride = INTVAL (iv->mult_val) * basestride;
3918 /* On some targets, reversed order prefetches are not
3920 if (PREFETCH_NO_REVERSE_ORDER && stride_sign < 0)
3921 ignore_reason = "reversed order stride";
3923 /* Prefetch of accesses with an extreme stride might not be
3924 worthwhile, either. */
3925 else if (PREFETCH_NO_EXTREME_STRIDE
3926 && stride > PREFETCH_EXTREME_STRIDE)
3927 ignore_reason = "extreme stride";
3929 /* Ignore GIVs with varying add values; we can't predict the
3930 value for the next iteration. */
3931 else if (!loop_invariant_p (loop, iv->add_val))
3932 ignore_reason = "giv has varying add value";
3934 /* Ignore GIVs in the nested loops; they ought to have been
3936 else if (iv->maybe_multiple)
3937 ignore_reason = "giv is in nested loop";
3940 if (ignore_reason != NULL)
3942 if (loop_dump_stream)
3943 fprintf (loop_dump_stream,
3944 "Prefetch: ignoring giv at %d: %s.\n",
3945 INSN_UID (iv->insn), ignore_reason);
3949 /* Determine the pointer to the basic array we are examining. It is
3950 the sum of the BIV's initial value and the GIV's add_val. */
3951 address = copy_rtx (iv->add_val);
3952 temp = copy_rtx (bl->initial_value);
3954 address = simplify_gen_binary (PLUS, Pmode, temp, address);
3955 index = remove_constant_addition (&address);
3958 d.mem_address = *iv->location;
3960 /* When the GIV is not always executed, we might be better off by
3961 not dirtying the cache pages. */
3962 if (PREFETCH_CONDITIONAL || iv->always_executed)
3963 note_stores (PATTERN (iv->insn), check_store, &d);
3966 if (loop_dump_stream)
3967 fprintf (loop_dump_stream, "Prefetch: Ignoring giv at %d: %s\n",
3968 INSN_UID (iv->insn), "in conditional code.");
3972 /* Attempt to find another prefetch to the same array and see if we
3973 can merge this one. */
3974 for (i = 0; i < num_prefetches; i++)
3975 if (rtx_equal_for_prefetch_p (address, info[i].base_address)
3976 && stride == info[i].stride)
3978 /* In case both access same array (same location
3979 just with small difference in constant indexes), merge
3980 the prefetches. Just do the later and the earlier will
3981 get prefetched from previous iteration.
3982 The artificial threshold should not be too small,
3983 but also not bigger than small portion of memory usually
3984 traversed by single loop. */
3985 if (index >= info[i].index
3986 && index - info[i].index < PREFETCH_EXTREME_DIFFERENCE)
3988 info[i].write |= d.mem_write;
3989 info[i].bytes_accessed += size;
3990 info[i].index = index;
3993 info[num_prefetches].base_address = address;
3998 if (index < info[i].index
3999 && info[i].index - index < PREFETCH_EXTREME_DIFFERENCE)
4001 info[i].write |= d.mem_write;
4002 info[i].bytes_accessed += size;
4008 /* Merging failed. */
4011 info[num_prefetches].giv = iv;
4012 info[num_prefetches].class = bl;
4013 info[num_prefetches].index = index;
4014 info[num_prefetches].stride = stride;
4015 info[num_prefetches].base_address = address;
4016 info[num_prefetches].write = d.mem_write;
4017 info[num_prefetches].bytes_accessed = size;
4019 if (num_prefetches >= MAX_PREFETCHES)
4021 if (loop_dump_stream)
4022 fprintf (loop_dump_stream,
4023 "Maximal number of prefetches exceeded.\n");
4030 for (i = 0; i < num_prefetches; i++)
4034 /* Attempt to calculate the total number of bytes fetched by all
4035 iterations of the loop. Avoid overflow. */
4036 if (LOOP_INFO (loop)->n_iterations
4037 && ((unsigned HOST_WIDE_INT) (0xffffffff / info[i].stride)
4038 >= LOOP_INFO (loop)->n_iterations))
4039 info[i].total_bytes = info[i].stride * LOOP_INFO (loop)->n_iterations;
4041 info[i].total_bytes = 0xffffffff;
4043 density = info[i].bytes_accessed * 100 / info[i].stride;
4045 /* Prefetch might be worthwhile only when the loads/stores are dense. */
4046 if (PREFETCH_ONLY_DENSE_MEM)
4047 if (density * 256 > PREFETCH_DENSE_MEM * 100
4048 && (info[i].total_bytes / PREFETCH_BLOCK
4049 >= PREFETCH_BLOCKS_BEFORE_LOOP_MIN))
4051 info[i].prefetch_before_loop = 1;
4052 info[i].prefetch_in_loop
4053 = (info[i].total_bytes / PREFETCH_BLOCK
4054 > PREFETCH_BLOCKS_BEFORE_LOOP_MAX);
4058 info[i].prefetch_in_loop = 0, info[i].prefetch_before_loop = 0;
4059 if (loop_dump_stream)
4060 fprintf (loop_dump_stream,
4061 "Prefetch: ignoring giv at %d: %d%% density is too low.\n",
4062 INSN_UID (info[i].giv->insn), density);
4065 info[i].prefetch_in_loop = 1, info[i].prefetch_before_loop = 1;
4067 /* Find how many prefetch instructions we'll use within the loop. */
4068 if (info[i].prefetch_in_loop != 0)
4070 info[i].prefetch_in_loop = ((info[i].stride + PREFETCH_BLOCK - 1)
4072 num_real_prefetches += info[i].prefetch_in_loop;
4074 num_real_write_prefetches += info[i].prefetch_in_loop;
4078 /* Determine how many iterations ahead to prefetch within the loop, based
4079 on how many prefetches we currently expect to do within the loop. */
4080 if (num_real_prefetches != 0)
4082 if ((ahead = SIMULTANEOUS_PREFETCHES / num_real_prefetches) == 0)
4084 if (loop_dump_stream)
4085 fprintf (loop_dump_stream,
4086 "Prefetch: ignoring prefetches within loop: ahead is zero; %d < %d\n",
4087 SIMULTANEOUS_PREFETCHES, num_real_prefetches);
4088 num_real_prefetches = 0, num_real_write_prefetches = 0;
4091 /* We'll also use AHEAD to determine how many prefetch instructions to
4092 emit before a loop, so don't leave it zero. */
4094 ahead = PREFETCH_BLOCKS_BEFORE_LOOP_MAX;
4096 for (i = 0; i < num_prefetches; i++)
4098 /* Update if we've decided not to prefetch anything within the loop. */
4099 if (num_real_prefetches == 0)
4100 info[i].prefetch_in_loop = 0;
4102 /* Find how many prefetch instructions we'll use before the loop. */
4103 if (info[i].prefetch_before_loop != 0)
4105 int n = info[i].total_bytes / PREFETCH_BLOCK;
4108 info[i].prefetch_before_loop = n;
4109 num_prefetches_before += n;
4111 num_write_prefetches_before += n;
4114 if (loop_dump_stream)
4116 if (info[i].prefetch_in_loop == 0
4117 && info[i].prefetch_before_loop == 0)
4119 fprintf (loop_dump_stream, "Prefetch insn: %d",
4120 INSN_UID (info[i].giv->insn));
4121 fprintf (loop_dump_stream,
4122 "; in loop: %d; before: %d; %s\n",
4123 info[i].prefetch_in_loop,
4124 info[i].prefetch_before_loop,
4125 info[i].write ? "read/write" : "read only");
4126 fprintf (loop_dump_stream,
4127 " density: %d%%; bytes_accessed: %u; total_bytes: %u\n",
4128 (int) (info[i].bytes_accessed * 100 / info[i].stride),
4129 info[i].bytes_accessed, info[i].total_bytes);
4130 fprintf (loop_dump_stream, " index: " HOST_WIDE_INT_PRINT_DEC
4131 "; stride: " HOST_WIDE_INT_PRINT_DEC "; address: ",
4132 info[i].index, info[i].stride);
4133 print_rtl (loop_dump_stream, info[i].base_address);
4134 fprintf (loop_dump_stream, "\n");
4138 if (num_real_prefetches + num_prefetches_before > 0)
4140 /* Record that this loop uses prefetch instructions. */
4141 LOOP_INFO (loop)->has_prefetch = 1;
4143 if (loop_dump_stream)
4145 fprintf (loop_dump_stream, "Real prefetches needed within loop: %d (write: %d)\n",
4146 num_real_prefetches, num_real_write_prefetches);
4147 fprintf (loop_dump_stream, "Real prefetches needed before loop: %d (write: %d)\n",
4148 num_prefetches_before, num_write_prefetches_before);
4152 for (i = 0; i < num_prefetches; i++)
4156 for (y = 0; y < info[i].prefetch_in_loop; y++)
4158 rtx loc = copy_rtx (*info[i].giv->location);
4160 int bytes_ahead = PREFETCH_BLOCK * (ahead + y);
4161 rtx before_insn = info[i].giv->insn;
4162 rtx prev_insn = PREV_INSN (info[i].giv->insn);
4165 /* We can save some effort by offsetting the address on
4166 architectures with offsettable memory references. */
4167 if (offsettable_address_p (0, VOIDmode, loc))
4168 loc = plus_constant (loc, bytes_ahead);
4171 rtx reg = gen_reg_rtx (Pmode);
4172 loop_iv_add_mult_emit_before (loop, loc, const1_rtx,
4173 GEN_INT (bytes_ahead), reg,
4179 /* Make sure the address operand is valid for prefetch. */
4180 if (! (*insn_data[(int)CODE_FOR_prefetch].operand[0].predicate)
4181 (loc, insn_data[(int)CODE_FOR_prefetch].operand[0].mode))
4182 loc = force_reg (Pmode, loc);
4183 emit_insn (gen_prefetch (loc, GEN_INT (info[i].write),
4187 emit_insn_before (seq, before_insn);
4189 /* Check all insns emitted and record the new GIV
4191 insn = NEXT_INSN (prev_insn);
4192 while (insn != before_insn)
4194 insn = check_insn_for_givs (loop, insn,
4195 info[i].giv->always_executed,
4196 info[i].giv->maybe_multiple);
4197 insn = NEXT_INSN (insn);
4201 if (PREFETCH_BEFORE_LOOP)
4203 /* Emit insns before the loop to fetch the first cache lines or,
4204 if we're not prefetching within the loop, everything we expect
4206 for (y = 0; y < info[i].prefetch_before_loop; y++)
4208 rtx reg = gen_reg_rtx (Pmode);
4209 rtx loop_start = loop->start;
4210 rtx init_val = info[i].class->initial_value;
4211 rtx add_val = simplify_gen_binary (PLUS, Pmode,
4212 info[i].giv->add_val,
4213 GEN_INT (y * PREFETCH_BLOCK));
4215 /* Functions called by LOOP_IV_ADD_EMIT_BEFORE expect a
4216 non-constant INIT_VAL to have the same mode as REG, which
4217 in this case we know to be Pmode. */
4218 if (GET_MODE (init_val) != Pmode && !CONSTANT_P (init_val))
4223 init_val = convert_to_mode (Pmode, init_val, 0);
4226 loop_insn_emit_before (loop, 0, loop_start, seq);
4228 loop_iv_add_mult_emit_before (loop, init_val,
4229 info[i].giv->mult_val,
4230 add_val, reg, 0, loop_start);
4231 emit_insn_before (gen_prefetch (reg, GEN_INT (info[i].write),
4241 /* Communication with routines called via `note_stores'. */
4243 static rtx note_insn;
4245 /* Dummy register to have nonzero DEST_REG for DEST_ADDR type givs. */
4247 static rtx addr_placeholder;
4249 /* ??? Unfinished optimizations, and possible future optimizations,
4250 for the strength reduction code. */
4252 /* ??? The interaction of biv elimination, and recognition of 'constant'
4253 bivs, may cause problems. */
4255 /* ??? Add heuristics so that DEST_ADDR strength reduction does not cause
4256 performance problems.
4258 Perhaps don't eliminate things that can be combined with an addressing
4259 mode. Find all givs that have the same biv, mult_val, and add_val;
4260 then for each giv, check to see if its only use dies in a following
4261 memory address. If so, generate a new memory address and check to see
4262 if it is valid. If it is valid, then store the modified memory address,
4263 otherwise, mark the giv as not done so that it will get its own iv. */
4265 /* ??? Could try to optimize branches when it is known that a biv is always
4268 /* ??? When replace a biv in a compare insn, we should replace with closest
4269 giv so that an optimized branch can still be recognized by the combiner,
4270 e.g. the VAX acb insn. */
4272 /* ??? Many of the checks involving uid_luid could be simplified if regscan
4273 was rerun in loop_optimize whenever a register was added or moved.
4274 Also, some of the optimizations could be a little less conservative. */
4276 /* Scan the loop body and call FNCALL for each insn. In the addition to the
4277 LOOP and INSN parameters pass MAYBE_MULTIPLE and NOT_EVERY_ITERATION to the
4280 NOT_EVERY_ITERATION is 1 if current insn is not known to be executed at
4281 least once for every loop iteration except for the last one.
4283 MAYBE_MULTIPLE is 1 if current insn may be executed more than once for every
4287 for_each_insn_in_loop (struct loop *loop, loop_insn_callback fncall)
4289 int not_every_iteration = 0;
4290 int maybe_multiple = 0;
4291 int past_loop_latch = 0;
4295 /* If loop_scan_start points to the loop exit test, we have to be wary of
4296 subversive use of gotos inside expression statements. */
4297 if (prev_nonnote_insn (loop->scan_start) != prev_nonnote_insn (loop->start))
4298 maybe_multiple = back_branch_in_range_p (loop, loop->scan_start);
4300 /* Scan through loop and update NOT_EVERY_ITERATION and MAYBE_MULTIPLE. */
4301 for (p = next_insn_in_loop (loop, loop->scan_start);
4303 p = next_insn_in_loop (loop, p))
4305 p = fncall (loop, p, not_every_iteration, maybe_multiple);
4307 /* Past CODE_LABEL, we get to insns that may be executed multiple
4308 times. The only way we can be sure that they can't is if every
4309 jump insn between here and the end of the loop either
4310 returns, exits the loop, is a jump to a location that is still
4311 behind the label, or is a jump to the loop start. */
4313 if (GET_CODE (p) == CODE_LABEL)
4321 insn = NEXT_INSN (insn);
4322 if (insn == loop->scan_start)
4324 if (insn == loop->end)
4330 if (insn == loop->scan_start)
4334 if (GET_CODE (insn) == JUMP_INSN
4335 && GET_CODE (PATTERN (insn)) != RETURN
4336 && (!any_condjump_p (insn)
4337 || (JUMP_LABEL (insn) != 0
4338 && JUMP_LABEL (insn) != loop->scan_start
4339 && !loop_insn_first_p (p, JUMP_LABEL (insn)))))
4347 /* Past a jump, we get to insns for which we can't count
4348 on whether they will be executed during each iteration. */
4349 /* This code appears twice in strength_reduce. There is also similar
4350 code in scan_loop. */
4351 if (GET_CODE (p) == JUMP_INSN
4352 /* If we enter the loop in the middle, and scan around to the
4353 beginning, don't set not_every_iteration for that.
4354 This can be any kind of jump, since we want to know if insns
4355 will be executed if the loop is executed. */
4356 && !(JUMP_LABEL (p) == loop->top
4357 && ((NEXT_INSN (NEXT_INSN (p)) == loop->end
4358 && any_uncondjump_p (p))
4359 || (NEXT_INSN (p) == loop->end && any_condjump_p (p)))))
4363 /* If this is a jump outside the loop, then it also doesn't
4364 matter. Check to see if the target of this branch is on the
4365 loop->exits_labels list. */
4367 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
4368 if (XEXP (label, 0) == JUMP_LABEL (p))
4372 not_every_iteration = 1;
4375 else if (GET_CODE (p) == NOTE)
4377 /* At the virtual top of a converted loop, insns are again known to
4378 be executed each iteration: logically, the loop begins here
4379 even though the exit code has been duplicated.
4381 Insns are also again known to be executed each iteration at
4382 the LOOP_CONT note. */
4383 if ((NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP
4384 || NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_CONT)
4386 not_every_iteration = 0;
4387 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
4389 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
4393 /* Note if we pass a loop latch. If we do, then we can not clear
4394 NOT_EVERY_ITERATION below when we pass the last CODE_LABEL in
4395 a loop since a jump before the last CODE_LABEL may have started
4396 a new loop iteration.
4398 Note that LOOP_TOP is only set for rotated loops and we need
4399 this check for all loops, so compare against the CODE_LABEL
4400 which immediately follows LOOP_START. */
4401 if (GET_CODE (p) == JUMP_INSN
4402 && JUMP_LABEL (p) == NEXT_INSN (loop->start))
4403 past_loop_latch = 1;
4405 /* Unlike in the code motion pass where MAYBE_NEVER indicates that
4406 an insn may never be executed, NOT_EVERY_ITERATION indicates whether
4407 or not an insn is known to be executed each iteration of the
4408 loop, whether or not any iterations are known to occur.
4410 Therefore, if we have just passed a label and have no more labels
4411 between here and the test insn of the loop, and we have not passed
4412 a jump to the top of the loop, then we know these insns will be
4413 executed each iteration. */
4415 if (not_every_iteration
4417 && GET_CODE (p) == CODE_LABEL
4418 && no_labels_between_p (p, loop->end)
4419 && loop_insn_first_p (p, loop->cont))
4420 not_every_iteration = 0;
4425 loop_bivs_find (struct loop *loop)
4427 struct loop_regs *regs = LOOP_REGS (loop);
4428 struct loop_ivs *ivs = LOOP_IVS (loop);
4429 /* Temporary list pointers for traversing ivs->list. */
4430 struct iv_class *bl, **backbl;
4434 for_each_insn_in_loop (loop, check_insn_for_bivs);
4436 /* Scan ivs->list to remove all regs that proved not to be bivs.
4437 Make a sanity check against regs->n_times_set. */
4438 for (backbl = &ivs->list, bl = *backbl; bl; bl = bl->next)
4440 if (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4441 /* Above happens if register modified by subreg, etc. */
4442 /* Make sure it is not recognized as a basic induction var: */
4443 || regs->array[bl->regno].n_times_set != bl->biv_count
4444 /* If never incremented, it is invariant that we decided not to
4445 move. So leave it alone. */
4446 || ! bl->incremented)
4448 if (loop_dump_stream)
4449 fprintf (loop_dump_stream, "Biv %d: discarded, %s\n",
4451 (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4452 ? "not induction variable"
4453 : (! bl->incremented ? "never incremented"
4456 REG_IV_TYPE (ivs, bl->regno) = NOT_BASIC_INDUCT;
4463 if (loop_dump_stream)
4464 fprintf (loop_dump_stream, "Biv %d: verified\n", bl->regno);
4470 /* Determine how BIVS are initialized by looking through pre-header
4471 extended basic block. */
4473 loop_bivs_init_find (struct loop *loop)
4475 struct loop_ivs *ivs = LOOP_IVS (loop);
4476 /* Temporary list pointers for traversing ivs->list. */
4477 struct iv_class *bl;
4481 /* Find initial value for each biv by searching backwards from loop_start,
4482 halting at first label. Also record any test condition. */
4485 for (p = loop->start; p && GET_CODE (p) != CODE_LABEL; p = PREV_INSN (p))
4491 if (GET_CODE (p) == CALL_INSN)
4495 note_stores (PATTERN (p), record_initial, ivs);
4497 /* Record any test of a biv that branches around the loop if no store
4498 between it and the start of loop. We only care about tests with
4499 constants and registers and only certain of those. */
4500 if (GET_CODE (p) == JUMP_INSN
4501 && JUMP_LABEL (p) != 0
4502 && next_real_insn (JUMP_LABEL (p)) == next_real_insn (loop->end)
4503 && (test = get_condition_for_loop (loop, p)) != 0
4504 && GET_CODE (XEXP (test, 0)) == REG
4505 && REGNO (XEXP (test, 0)) < max_reg_before_loop
4506 && (bl = REG_IV_CLASS (ivs, REGNO (XEXP (test, 0)))) != 0
4507 && valid_initial_value_p (XEXP (test, 1), p, call_seen, loop->start)
4508 && bl->init_insn == 0)
4510 /* If an NE test, we have an initial value! */
4511 if (GET_CODE (test) == NE)
4514 bl->init_set = gen_rtx_SET (VOIDmode,
4515 XEXP (test, 0), XEXP (test, 1));
4518 bl->initial_test = test;
4524 /* Look at the each biv and see if we can say anything better about its
4525 initial value from any initializing insns set up above. (This is done
4526 in two passes to avoid missing SETs in a PARALLEL.) */
4528 loop_bivs_check (struct loop *loop)
4530 struct loop_ivs *ivs = LOOP_IVS (loop);
4531 /* Temporary list pointers for traversing ivs->list. */
4532 struct iv_class *bl;
4533 struct iv_class **backbl;
4535 for (backbl = &ivs->list; (bl = *backbl); backbl = &bl->next)
4540 if (! bl->init_insn)
4543 /* IF INIT_INSN has a REG_EQUAL or REG_EQUIV note and the value
4544 is a constant, use the value of that. */
4545 if (((note = find_reg_note (bl->init_insn, REG_EQUAL, 0)) != NULL
4546 && CONSTANT_P (XEXP (note, 0)))
4547 || ((note = find_reg_note (bl->init_insn, REG_EQUIV, 0)) != NULL
4548 && CONSTANT_P (XEXP (note, 0))))
4549 src = XEXP (note, 0);
4551 src = SET_SRC (bl->init_set);
4553 if (loop_dump_stream)
4554 fprintf (loop_dump_stream,
4555 "Biv %d: initialized at insn %d: initial value ",
4556 bl->regno, INSN_UID (bl->init_insn));
4558 if ((GET_MODE (src) == GET_MODE (regno_reg_rtx[bl->regno])
4559 || GET_MODE (src) == VOIDmode)
4560 && valid_initial_value_p (src, bl->init_insn,
4561 LOOP_INFO (loop)->pre_header_has_call,
4564 bl->initial_value = src;
4566 if (loop_dump_stream)
4568 print_simple_rtl (loop_dump_stream, src);
4569 fputc ('\n', loop_dump_stream);
4572 /* If we can't make it a giv,
4573 let biv keep initial value of "itself". */
4574 else if (loop_dump_stream)
4575 fprintf (loop_dump_stream, "is complex\n");
4580 /* Search the loop for general induction variables. */
4583 loop_givs_find (struct loop* loop)
4585 for_each_insn_in_loop (loop, check_insn_for_givs);
4589 /* For each giv for which we still don't know whether or not it is
4590 replaceable, check to see if it is replaceable because its final value
4591 can be calculated. */
4594 loop_givs_check (struct loop *loop)
4596 struct loop_ivs *ivs = LOOP_IVS (loop);
4597 struct iv_class *bl;
4599 for (bl = ivs->list; bl; bl = bl->next)
4601 struct induction *v;
4603 for (v = bl->giv; v; v = v->next_iv)
4604 if (! v->replaceable && ! v->not_replaceable)
4605 check_final_value (loop, v);
4610 /* Return nonzero if it is possible to eliminate the biv BL provided
4611 all givs are reduced. This is possible if either the reg is not
4612 used outside the loop, or we can compute what its final value will
4616 loop_biv_eliminable_p (struct loop *loop, struct iv_class *bl,
4617 int threshold, int insn_count)
4619 /* For architectures with a decrement_and_branch_until_zero insn,
4620 don't do this if we put a REG_NONNEG note on the endtest for this
4623 #ifdef HAVE_decrement_and_branch_until_zero
4626 if (loop_dump_stream)
4627 fprintf (loop_dump_stream,
4628 "Cannot eliminate nonneg biv %d.\n", bl->regno);
4633 /* Check that biv is used outside loop or if it has a final value.
4634 Compare against bl->init_insn rather than loop->start. We aren't
4635 concerned with any uses of the biv between init_insn and
4636 loop->start since these won't be affected by the value of the biv
4637 elsewhere in the function, so long as init_insn doesn't use the
4640 if ((REGNO_LAST_LUID (bl->regno) < INSN_LUID (loop->end)
4642 && INSN_UID (bl->init_insn) < max_uid_for_loop
4643 && REGNO_FIRST_LUID (bl->regno) >= INSN_LUID (bl->init_insn)
4644 && ! reg_mentioned_p (bl->biv->dest_reg, SET_SRC (bl->init_set)))
4645 || (bl->final_value = final_biv_value (loop, bl)))
4646 return maybe_eliminate_biv (loop, bl, 0, threshold, insn_count);
4648 if (loop_dump_stream)
4650 fprintf (loop_dump_stream,
4651 "Cannot eliminate biv %d.\n",
4653 fprintf (loop_dump_stream,
4654 "First use: insn %d, last use: insn %d.\n",
4655 REGNO_FIRST_UID (bl->regno),
4656 REGNO_LAST_UID (bl->regno));
4662 /* Reduce each giv of BL that we have decided to reduce. */
4665 loop_givs_reduce (struct loop *loop, struct iv_class *bl)
4667 struct induction *v;
4669 for (v = bl->giv; v; v = v->next_iv)
4671 struct induction *tv;
4672 if (! v->ignore && v->same == 0)
4674 int auto_inc_opt = 0;
4676 /* If the code for derived givs immediately below has already
4677 allocated a new_reg, we must keep it. */
4679 v->new_reg = gen_reg_rtx (v->mode);
4682 /* If the target has auto-increment addressing modes, and
4683 this is an address giv, then try to put the increment
4684 immediately after its use, so that flow can create an
4685 auto-increment addressing mode. */
4686 if (v->giv_type == DEST_ADDR && bl->biv_count == 1
4687 && bl->biv->always_executed && ! bl->biv->maybe_multiple
4688 /* We don't handle reversed biv's because bl->biv->insn
4689 does not have a valid INSN_LUID. */
4691 && v->always_executed && ! v->maybe_multiple
4692 && INSN_UID (v->insn) < max_uid_for_loop)
4694 /* If other giv's have been combined with this one, then
4695 this will work only if all uses of the other giv's occur
4696 before this giv's insn. This is difficult to check.
4698 We simplify this by looking for the common case where
4699 there is one DEST_REG giv, and this giv's insn is the
4700 last use of the dest_reg of that DEST_REG giv. If the
4701 increment occurs after the address giv, then we can
4702 perform the optimization. (Otherwise, the increment
4703 would have to go before other_giv, and we would not be
4704 able to combine it with the address giv to get an
4705 auto-inc address.) */
4706 if (v->combined_with)
4708 struct induction *other_giv = 0;
4710 for (tv = bl->giv; tv; tv = tv->next_iv)
4718 if (! tv && other_giv
4719 && REGNO (other_giv->dest_reg) < max_reg_before_loop
4720 && (REGNO_LAST_UID (REGNO (other_giv->dest_reg))
4721 == INSN_UID (v->insn))
4722 && INSN_LUID (v->insn) < INSN_LUID (bl->biv->insn))
4725 /* Check for case where increment is before the address
4726 giv. Do this test in "loop order". */
4727 else if ((INSN_LUID (v->insn) > INSN_LUID (bl->biv->insn)
4728 && (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4729 || (INSN_LUID (bl->biv->insn)
4730 > INSN_LUID (loop->scan_start))))
4731 || (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4732 && (INSN_LUID (loop->scan_start)
4733 < INSN_LUID (bl->biv->insn))))
4742 /* We can't put an insn immediately after one setting
4743 cc0, or immediately before one using cc0. */
4744 if ((auto_inc_opt == 1 && sets_cc0_p (PATTERN (v->insn)))
4745 || (auto_inc_opt == -1
4746 && (prev = prev_nonnote_insn (v->insn)) != 0
4748 && sets_cc0_p (PATTERN (prev))))
4754 v->auto_inc_opt = 1;
4758 /* For each place where the biv is incremented, add an insn
4759 to increment the new, reduced reg for the giv. */
4760 for (tv = bl->biv; tv; tv = tv->next_iv)
4764 /* Skip if location is the same as a previous one. */
4768 insert_before = NEXT_INSN (tv->insn);
4769 else if (auto_inc_opt == 1)
4770 insert_before = NEXT_INSN (v->insn);
4772 insert_before = v->insn;
4774 if (tv->mult_val == const1_rtx)
4775 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4776 v->new_reg, v->new_reg,
4778 else /* tv->mult_val == const0_rtx */
4779 /* A multiply is acceptable here
4780 since this is presumed to be seldom executed. */
4781 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4782 v->add_val, v->new_reg,
4786 /* Add code at loop start to initialize giv's reduced reg. */
4788 loop_iv_add_mult_hoist (loop,
4789 extend_value_for_giv (v, bl->initial_value),
4790 v->mult_val, v->add_val, v->new_reg);
4796 /* Check for givs whose first use is their definition and whose
4797 last use is the definition of another giv. If so, it is likely
4798 dead and should not be used to derive another giv nor to
4802 loop_givs_dead_check (struct loop *loop ATTRIBUTE_UNUSED, struct iv_class *bl)
4804 struct induction *v;
4806 for (v = bl->giv; v; v = v->next_iv)
4809 || (v->same && v->same->ignore))
4812 if (v->giv_type == DEST_REG
4813 && REGNO_FIRST_UID (REGNO (v->dest_reg)) == INSN_UID (v->insn))
4815 struct induction *v1;
4817 for (v1 = bl->giv; v1; v1 = v1->next_iv)
4818 if (REGNO_LAST_UID (REGNO (v->dest_reg)) == INSN_UID (v1->insn))
4826 loop_givs_rescan (struct loop *loop, struct iv_class *bl, rtx *reg_map)
4828 struct induction *v;
4830 for (v = bl->giv; v; v = v->next_iv)
4832 if (v->same && v->same->ignore)
4838 /* Update expression if this was combined, in case other giv was
4841 v->new_reg = replace_rtx (v->new_reg,
4842 v->same->dest_reg, v->same->new_reg);
4844 /* See if this register is known to be a pointer to something. If
4845 so, see if we can find the alignment. First see if there is a
4846 destination register that is a pointer. If so, this shares the
4847 alignment too. Next see if we can deduce anything from the
4848 computational information. If not, and this is a DEST_ADDR
4849 giv, at least we know that it's a pointer, though we don't know
4851 if (GET_CODE (v->new_reg) == REG
4852 && v->giv_type == DEST_REG
4853 && REG_POINTER (v->dest_reg))
4854 mark_reg_pointer (v->new_reg,
4855 REGNO_POINTER_ALIGN (REGNO (v->dest_reg)));
4856 else if (GET_CODE (v->new_reg) == REG
4857 && REG_POINTER (v->src_reg))
4859 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->src_reg));
4862 || GET_CODE (v->add_val) != CONST_INT
4863 || INTVAL (v->add_val) % (align / BITS_PER_UNIT) != 0)
4866 mark_reg_pointer (v->new_reg, align);
4868 else if (GET_CODE (v->new_reg) == REG
4869 && GET_CODE (v->add_val) == REG
4870 && REG_POINTER (v->add_val))
4872 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->add_val));
4874 if (align == 0 || GET_CODE (v->mult_val) != CONST_INT
4875 || INTVAL (v->mult_val) % (align / BITS_PER_UNIT) != 0)
4878 mark_reg_pointer (v->new_reg, align);
4880 else if (GET_CODE (v->new_reg) == REG && v->giv_type == DEST_ADDR)
4881 mark_reg_pointer (v->new_reg, 0);
4883 if (v->giv_type == DEST_ADDR)
4884 /* Store reduced reg as the address in the memref where we found
4886 validate_change (v->insn, v->location, v->new_reg, 0);
4887 else if (v->replaceable)
4889 reg_map[REGNO (v->dest_reg)] = v->new_reg;
4893 rtx original_insn = v->insn;
4896 /* Not replaceable; emit an insn to set the original giv reg from
4897 the reduced giv, same as above. */
4898 v->insn = loop_insn_emit_after (loop, 0, original_insn,
4899 gen_move_insn (v->dest_reg,
4902 /* The original insn may have a REG_EQUAL note. This note is
4903 now incorrect and may result in invalid substitutions later.
4904 The original insn is dead, but may be part of a libcall
4905 sequence, which doesn't seem worth the bother of handling. */
4906 note = find_reg_note (original_insn, REG_EQUAL, NULL_RTX);
4908 remove_note (original_insn, note);
4911 /* When a loop is reversed, givs which depend on the reversed
4912 biv, and which are live outside the loop, must be set to their
4913 correct final value. This insn is only needed if the giv is
4914 not replaceable. The correct final value is the same as the
4915 value that the giv starts the reversed loop with. */
4916 if (bl->reversed && ! v->replaceable)
4917 loop_iv_add_mult_sink (loop,
4918 extend_value_for_giv (v, bl->initial_value),
4919 v->mult_val, v->add_val, v->dest_reg);
4920 else if (v->final_value)
4921 loop_insn_sink_or_swim (loop,
4922 gen_load_of_final_value (v->dest_reg,
4925 if (loop_dump_stream)
4927 fprintf (loop_dump_stream, "giv at %d reduced to ",
4928 INSN_UID (v->insn));
4929 print_simple_rtl (loop_dump_stream, v->new_reg);
4930 fprintf (loop_dump_stream, "\n");
4937 loop_giv_reduce_benefit (struct loop *loop ATTRIBUTE_UNUSED,
4938 struct iv_class *bl, struct induction *v,
4944 benefit = v->benefit;
4945 PUT_MODE (test_reg, v->mode);
4946 add_cost = iv_add_mult_cost (bl->biv->add_val, v->mult_val,
4947 test_reg, test_reg);
4949 /* Reduce benefit if not replaceable, since we will insert a
4950 move-insn to replace the insn that calculates this giv. Don't do
4951 this unless the giv is a user variable, since it will often be
4952 marked non-replaceable because of the duplication of the exit
4953 code outside the loop. In such a case, the copies we insert are
4954 dead and will be deleted. So they don't have a cost. Similar
4955 situations exist. */
4956 /* ??? The new final_[bg]iv_value code does a much better job of
4957 finding replaceable giv's, and hence this code may no longer be
4959 if (! v->replaceable && ! bl->eliminable
4960 && REG_USERVAR_P (v->dest_reg))
4961 benefit -= copy_cost;
4963 /* Decrease the benefit to count the add-insns that we will insert
4964 to increment the reduced reg for the giv. ??? This can
4965 overestimate the run-time cost of the additional insns, e.g. if
4966 there are multiple basic blocks that increment the biv, but only
4967 one of these blocks is executed during each iteration. There is
4968 no good way to detect cases like this with the current structure
4969 of the loop optimizer. This code is more accurate for
4970 determining code size than run-time benefits. */
4971 benefit -= add_cost * bl->biv_count;
4973 /* Decide whether to strength-reduce this giv or to leave the code
4974 unchanged (recompute it from the biv each time it is used). This
4975 decision can be made independently for each giv. */
4978 /* Attempt to guess whether autoincrement will handle some of the
4979 new add insns; if so, increase BENEFIT (undo the subtraction of
4980 add_cost that was done above). */
4981 if (v->giv_type == DEST_ADDR
4982 /* Increasing the benefit is risky, since this is only a guess.
4983 Avoid increasing register pressure in cases where there would
4984 be no other benefit from reducing this giv. */
4986 && GET_CODE (v->mult_val) == CONST_INT)
4988 int size = GET_MODE_SIZE (GET_MODE (v->mem));
4990 if (HAVE_POST_INCREMENT
4991 && INTVAL (v->mult_val) == size)
4992 benefit += add_cost * bl->biv_count;
4993 else if (HAVE_PRE_INCREMENT
4994 && INTVAL (v->mult_val) == size)
4995 benefit += add_cost * bl->biv_count;
4996 else if (HAVE_POST_DECREMENT
4997 && -INTVAL (v->mult_val) == size)
4998 benefit += add_cost * bl->biv_count;
4999 else if (HAVE_PRE_DECREMENT
5000 && -INTVAL (v->mult_val) == size)
5001 benefit += add_cost * bl->biv_count;
5009 /* Free IV structures for LOOP. */
5012 loop_ivs_free (struct loop *loop)
5014 struct loop_ivs *ivs = LOOP_IVS (loop);
5015 struct iv_class *iv = ivs->list;
5021 struct iv_class *next = iv->next;
5022 struct induction *induction;
5023 struct induction *next_induction;
5025 for (induction = iv->biv; induction; induction = next_induction)
5027 next_induction = induction->next_iv;
5030 for (induction = iv->giv; induction; induction = next_induction)
5032 next_induction = induction->next_iv;
5042 /* Perform strength reduction and induction variable elimination.
5044 Pseudo registers created during this function will be beyond the
5045 last valid index in several tables including
5046 REGS->ARRAY[I].N_TIMES_SET and REGNO_LAST_UID. This does not cause a
5047 problem here, because the added registers cannot be givs outside of
5048 their loop, and hence will never be reconsidered. But scan_loop
5049 must check regnos to make sure they are in bounds. */
5052 strength_reduce (struct loop *loop, int flags)
5054 struct loop_info *loop_info = LOOP_INFO (loop);
5055 struct loop_regs *regs = LOOP_REGS (loop);
5056 struct loop_ivs *ivs = LOOP_IVS (loop);
5058 /* Temporary list pointer for traversing ivs->list. */
5059 struct iv_class *bl;
5060 /* Ratio of extra register life span we can justify
5061 for saving an instruction. More if loop doesn't call subroutines
5062 since in that case saving an insn makes more difference
5063 and more registers are available. */
5064 /* ??? could set this to last value of threshold in move_movables */
5065 int threshold = (loop_info->has_call ? 1 : 2) * (3 + n_non_fixed_regs);
5066 /* Map of pseudo-register replacements. */
5067 rtx *reg_map = NULL;
5069 int unrolled_insn_copies = 0;
5070 rtx test_reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
5071 int insn_count = count_insns_in_loop (loop);
5073 addr_placeholder = gen_reg_rtx (Pmode);
5075 ivs->n_regs = max_reg_before_loop;
5076 ivs->regs = xcalloc (ivs->n_regs, sizeof (struct iv));
5078 /* Find all BIVs in loop. */
5079 loop_bivs_find (loop);
5081 /* Exit if there are no bivs. */
5084 /* Can still unroll the loop anyways, but indicate that there is no
5085 strength reduction info available. */
5086 if (flags & LOOP_UNROLL)
5087 unroll_loop (loop, insn_count, 0);
5089 loop_ivs_free (loop);
5093 /* Determine how BIVS are initialized by looking through pre-header
5094 extended basic block. */
5095 loop_bivs_init_find (loop);
5097 /* Look at the each biv and see if we can say anything better about its
5098 initial value from any initializing insns set up above. */
5099 loop_bivs_check (loop);
5101 /* Search the loop for general induction variables. */
5102 loop_givs_find (loop);
5104 /* Try to calculate and save the number of loop iterations. This is
5105 set to zero if the actual number can not be calculated. This must
5106 be called after all giv's have been identified, since otherwise it may
5107 fail if the iteration variable is a giv. */
5108 loop_iterations (loop);
5110 #ifdef HAVE_prefetch
5111 if (flags & LOOP_PREFETCH)
5112 emit_prefetch_instructions (loop);
5115 /* Now for each giv for which we still don't know whether or not it is
5116 replaceable, check to see if it is replaceable because its final value
5117 can be calculated. This must be done after loop_iterations is called,
5118 so that final_giv_value will work correctly. */
5119 loop_givs_check (loop);
5121 /* Try to prove that the loop counter variable (if any) is always
5122 nonnegative; if so, record that fact with a REG_NONNEG note
5123 so that "decrement and branch until zero" insn can be used. */
5124 check_dbra_loop (loop, insn_count);
5126 /* Create reg_map to hold substitutions for replaceable giv regs.
5127 Some givs might have been made from biv increments, so look at
5128 ivs->reg_iv_type for a suitable size. */
5129 reg_map_size = ivs->n_regs;
5130 reg_map = xcalloc (reg_map_size, sizeof (rtx));
5132 /* Examine each iv class for feasibility of strength reduction/induction
5133 variable elimination. */
5135 for (bl = ivs->list; bl; bl = bl->next)
5137 struct induction *v;
5140 /* Test whether it will be possible to eliminate this biv
5141 provided all givs are reduced. */
5142 bl->eliminable = loop_biv_eliminable_p (loop, bl, threshold, insn_count);
5144 /* This will be true at the end, if all givs which depend on this
5145 biv have been strength reduced.
5146 We can't (currently) eliminate the biv unless this is so. */
5147 bl->all_reduced = 1;
5149 /* Check each extension dependent giv in this class to see if its
5150 root biv is safe from wrapping in the interior mode. */
5151 check_ext_dependent_givs (loop, bl);
5153 /* Combine all giv's for this iv_class. */
5154 combine_givs (regs, bl);
5156 for (v = bl->giv; v; v = v->next_iv)
5158 struct induction *tv;
5160 if (v->ignore || v->same)
5163 benefit = loop_giv_reduce_benefit (loop, bl, v, test_reg);
5165 /* If an insn is not to be strength reduced, then set its ignore
5166 flag, and clear bl->all_reduced. */
5168 /* A giv that depends on a reversed biv must be reduced if it is
5169 used after the loop exit, otherwise, it would have the wrong
5170 value after the loop exit. To make it simple, just reduce all
5171 of such giv's whether or not we know they are used after the loop
5174 if (! flag_reduce_all_givs
5175 && v->lifetime * threshold * benefit < insn_count
5178 if (loop_dump_stream)
5179 fprintf (loop_dump_stream,
5180 "giv of insn %d not worth while, %d vs %d.\n",
5182 v->lifetime * threshold * benefit, insn_count);
5184 bl->all_reduced = 0;
5188 /* Check that we can increment the reduced giv without a
5189 multiply insn. If not, reject it. */
5191 for (tv = bl->biv; tv; tv = tv->next_iv)
5192 if (tv->mult_val == const1_rtx
5193 && ! product_cheap_p (tv->add_val, v->mult_val))
5195 if (loop_dump_stream)
5196 fprintf (loop_dump_stream,
5197 "giv of insn %d: would need a multiply.\n",
5198 INSN_UID (v->insn));
5200 bl->all_reduced = 0;
5206 /* Check for givs whose first use is their definition and whose
5207 last use is the definition of another giv. If so, it is likely
5208 dead and should not be used to derive another giv nor to
5210 loop_givs_dead_check (loop, bl);
5212 /* Reduce each giv that we decided to reduce. */
5213 loop_givs_reduce (loop, bl);
5215 /* Rescan all givs. If a giv is the same as a giv not reduced, mark it
5218 For each giv register that can be reduced now: if replaceable,
5219 substitute reduced reg wherever the old giv occurs;
5220 else add new move insn "giv_reg = reduced_reg". */
5221 loop_givs_rescan (loop, bl, reg_map);
5223 /* All the givs based on the biv bl have been reduced if they
5226 /* For each giv not marked as maybe dead that has been combined with a
5227 second giv, clear any "maybe dead" mark on that second giv.
5228 v->new_reg will either be or refer to the register of the giv it
5231 Doing this clearing avoids problems in biv elimination where
5232 a giv's new_reg is a complex value that can't be put in the
5233 insn but the giv combined with (with a reg as new_reg) is
5234 marked maybe_dead. Since the register will be used in either
5235 case, we'd prefer it be used from the simpler giv. */
5237 for (v = bl->giv; v; v = v->next_iv)
5238 if (! v->maybe_dead && v->same)
5239 v->same->maybe_dead = 0;
5241 /* Try to eliminate the biv, if it is a candidate.
5242 This won't work if ! bl->all_reduced,
5243 since the givs we planned to use might not have been reduced.
5245 We have to be careful that we didn't initially think we could
5246 eliminate this biv because of a giv that we now think may be
5247 dead and shouldn't be used as a biv replacement.
5249 Also, there is the possibility that we may have a giv that looks
5250 like it can be used to eliminate a biv, but the resulting insn
5251 isn't valid. This can happen, for example, on the 88k, where a
5252 JUMP_INSN can compare a register only with zero. Attempts to
5253 replace it with a compare with a constant will fail.
5255 Note that in cases where this call fails, we may have replaced some
5256 of the occurrences of the biv with a giv, but no harm was done in
5257 doing so in the rare cases where it can occur. */
5259 if (bl->all_reduced == 1 && bl->eliminable
5260 && maybe_eliminate_biv (loop, bl, 1, threshold, insn_count))
5262 /* ?? If we created a new test to bypass the loop entirely,
5263 or otherwise drop straight in, based on this test, then
5264 we might want to rewrite it also. This way some later
5265 pass has more hope of removing the initialization of this
5268 /* If final_value != 0, then the biv may be used after loop end
5269 and we must emit an insn to set it just in case.
5271 Reversed bivs already have an insn after the loop setting their
5272 value, so we don't need another one. We can't calculate the
5273 proper final value for such a biv here anyways. */
5274 if (bl->final_value && ! bl->reversed)
5275 loop_insn_sink_or_swim (loop,
5276 gen_load_of_final_value (bl->biv->dest_reg,
5279 if (loop_dump_stream)
5280 fprintf (loop_dump_stream, "Reg %d: biv eliminated\n",
5283 /* See above note wrt final_value. But since we couldn't eliminate
5284 the biv, we must set the value after the loop instead of before. */
5285 else if (bl->final_value && ! bl->reversed)
5286 loop_insn_sink (loop, gen_load_of_final_value (bl->biv->dest_reg,
5290 /* Go through all the instructions in the loop, making all the
5291 register substitutions scheduled in REG_MAP. */
5293 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
5294 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5295 || GET_CODE (p) == CALL_INSN)
5297 replace_regs (PATTERN (p), reg_map, reg_map_size, 0);
5298 replace_regs (REG_NOTES (p), reg_map, reg_map_size, 0);
5302 if (loop_info->n_iterations > 0)
5304 /* When we completely unroll a loop we will likely not need the increment
5305 of the loop BIV and we will not need the conditional branch at the
5307 unrolled_insn_copies = insn_count - 2;
5310 /* When we completely unroll a loop on a HAVE_cc0 machine we will not
5311 need the comparison before the conditional branch at the end of the
5313 unrolled_insn_copies -= 1;
5316 /* We'll need one copy for each loop iteration. */
5317 unrolled_insn_copies *= loop_info->n_iterations;
5319 /* A little slop to account for the ability to remove initialization
5320 code, better CSE, and other secondary benefits of completely
5321 unrolling some loops. */
5322 unrolled_insn_copies -= 1;
5324 /* Clamp the value. */
5325 if (unrolled_insn_copies < 0)
5326 unrolled_insn_copies = 0;
5329 /* Unroll loops from within strength reduction so that we can use the
5330 induction variable information that strength_reduce has already
5331 collected. Always unroll loops that would be as small or smaller
5332 unrolled than when rolled. */
5333 if ((flags & LOOP_UNROLL)
5334 || ((flags & LOOP_AUTO_UNROLL)
5335 && loop_info->n_iterations > 0
5336 && unrolled_insn_copies <= insn_count))
5337 unroll_loop (loop, insn_count, 1);
5339 #ifdef HAVE_doloop_end
5340 if (HAVE_doloop_end && (flags & LOOP_BCT) && flag_branch_on_count_reg)
5341 doloop_optimize (loop);
5342 #endif /* HAVE_doloop_end */
5344 /* In case number of iterations is known, drop branch prediction note
5345 in the branch. Do that only in second loop pass, as loop unrolling
5346 may change the number of iterations performed. */
5347 if (flags & LOOP_BCT)
5349 unsigned HOST_WIDE_INT n
5350 = loop_info->n_iterations / loop_info->unroll_number;
5352 predict_insn (prev_nonnote_insn (loop->end), PRED_LOOP_ITERATIONS,
5353 REG_BR_PROB_BASE - REG_BR_PROB_BASE / n);
5356 if (loop_dump_stream)
5357 fprintf (loop_dump_stream, "\n");
5359 loop_ivs_free (loop);
5364 /*Record all basic induction variables calculated in the insn. */
5366 check_insn_for_bivs (struct loop *loop, rtx p, int not_every_iteration,
5369 struct loop_ivs *ivs = LOOP_IVS (loop);
5376 if (GET_CODE (p) == INSN
5377 && (set = single_set (p))
5378 && GET_CODE (SET_DEST (set)) == REG)
5380 dest_reg = SET_DEST (set);
5381 if (REGNO (dest_reg) < max_reg_before_loop
5382 && REGNO (dest_reg) >= FIRST_PSEUDO_REGISTER
5383 && REG_IV_TYPE (ivs, REGNO (dest_reg)) != NOT_BASIC_INDUCT)
5385 if (basic_induction_var (loop, SET_SRC (set),
5386 GET_MODE (SET_SRC (set)),
5387 dest_reg, p, &inc_val, &mult_val,
5390 /* It is a possible basic induction variable.
5391 Create and initialize an induction structure for it. */
5393 struct induction *v = xmalloc (sizeof (struct induction));
5395 record_biv (loop, v, p, dest_reg, inc_val, mult_val, location,
5396 not_every_iteration, maybe_multiple);
5397 REG_IV_TYPE (ivs, REGNO (dest_reg)) = BASIC_INDUCT;
5399 else if (REGNO (dest_reg) < ivs->n_regs)
5400 REG_IV_TYPE (ivs, REGNO (dest_reg)) = NOT_BASIC_INDUCT;
5406 /* Record all givs calculated in the insn.
5407 A register is a giv if: it is only set once, it is a function of a
5408 biv and a constant (or invariant), and it is not a biv. */
5410 check_insn_for_givs (struct loop *loop, rtx p, int not_every_iteration,
5413 struct loop_regs *regs = LOOP_REGS (loop);
5416 /* Look for a general induction variable in a register. */
5417 if (GET_CODE (p) == INSN
5418 && (set = single_set (p))
5419 && GET_CODE (SET_DEST (set)) == REG
5420 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
5429 rtx last_consec_insn;
5431 dest_reg = SET_DEST (set);
5432 if (REGNO (dest_reg) < FIRST_PSEUDO_REGISTER)
5435 if (/* SET_SRC is a giv. */
5436 (general_induction_var (loop, SET_SRC (set), &src_reg, &add_val,
5437 &mult_val, &ext_val, 0, &benefit, VOIDmode)
5438 /* Equivalent expression is a giv. */
5439 || ((regnote = find_reg_note (p, REG_EQUAL, NULL_RTX))
5440 && general_induction_var (loop, XEXP (regnote, 0), &src_reg,
5441 &add_val, &mult_val, &ext_val, 0,
5442 &benefit, VOIDmode)))
5443 /* Don't try to handle any regs made by loop optimization.
5444 We have nothing on them in regno_first_uid, etc. */
5445 && REGNO (dest_reg) < max_reg_before_loop
5446 /* Don't recognize a BASIC_INDUCT_VAR here. */
5447 && dest_reg != src_reg
5448 /* This must be the only place where the register is set. */
5449 && (regs->array[REGNO (dest_reg)].n_times_set == 1
5450 /* or all sets must be consecutive and make a giv. */
5451 || (benefit = consec_sets_giv (loop, benefit, p,
5453 &add_val, &mult_val, &ext_val,
5454 &last_consec_insn))))
5456 struct induction *v = xmalloc (sizeof (struct induction));
5458 /* If this is a library call, increase benefit. */
5459 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
5460 benefit += libcall_benefit (p);
5462 /* Skip the consecutive insns, if there are any. */
5463 if (regs->array[REGNO (dest_reg)].n_times_set != 1)
5464 p = last_consec_insn;
5466 record_giv (loop, v, p, src_reg, dest_reg, mult_val, add_val,
5467 ext_val, benefit, DEST_REG, not_every_iteration,
5468 maybe_multiple, (rtx*) 0);
5473 /* Look for givs which are memory addresses. */
5474 if (GET_CODE (p) == INSN)
5475 find_mem_givs (loop, PATTERN (p), p, not_every_iteration,
5478 /* Update the status of whether giv can derive other givs. This can
5479 change when we pass a label or an insn that updates a biv. */
5480 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5481 || GET_CODE (p) == CODE_LABEL)
5482 update_giv_derive (loop, p);
5486 /* Return 1 if X is a valid source for an initial value (or as value being
5487 compared against in an initial test).
5489 X must be either a register or constant and must not be clobbered between
5490 the current insn and the start of the loop.
5492 INSN is the insn containing X. */
5495 valid_initial_value_p (rtx x, rtx insn, int call_seen, rtx loop_start)
5500 /* Only consider pseudos we know about initialized in insns whose luids
5502 if (GET_CODE (x) != REG
5503 || REGNO (x) >= max_reg_before_loop)
5506 /* Don't use call-clobbered registers across a call which clobbers it. On
5507 some machines, don't use any hard registers at all. */
5508 if (REGNO (x) < FIRST_PSEUDO_REGISTER
5509 && (SMALL_REGISTER_CLASSES
5510 || (call_used_regs[REGNO (x)] && call_seen)))
5513 /* Don't use registers that have been clobbered before the start of the
5515 if (reg_set_between_p (x, insn, loop_start))
5521 /* Scan X for memory refs and check each memory address
5522 as a possible giv. INSN is the insn whose pattern X comes from.
5523 NOT_EVERY_ITERATION is 1 if the insn might not be executed during
5524 every loop iteration. MAYBE_MULTIPLE is 1 if the insn might be executed
5525 more than once in each loop iteration. */
5528 find_mem_givs (const struct loop *loop, rtx x, rtx insn,
5529 int not_every_iteration, int maybe_multiple)
5538 code = GET_CODE (x);
5563 /* This code used to disable creating GIVs with mult_val == 1 and
5564 add_val == 0. However, this leads to lost optimizations when
5565 it comes time to combine a set of related DEST_ADDR GIVs, since
5566 this one would not be seen. */
5568 if (general_induction_var (loop, XEXP (x, 0), &src_reg, &add_val,
5569 &mult_val, &ext_val, 1, &benefit,
5572 /* Found one; record it. */
5573 struct induction *v = xmalloc (sizeof (struct induction));
5575 record_giv (loop, v, insn, src_reg, addr_placeholder, mult_val,
5576 add_val, ext_val, benefit, DEST_ADDR,
5577 not_every_iteration, maybe_multiple, &XEXP (x, 0));
5588 /* Recursively scan the subexpressions for other mem refs. */
5590 fmt = GET_RTX_FORMAT (code);
5591 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5593 find_mem_givs (loop, XEXP (x, i), insn, not_every_iteration,
5595 else if (fmt[i] == 'E')
5596 for (j = 0; j < XVECLEN (x, i); j++)
5597 find_mem_givs (loop, XVECEXP (x, i, j), insn, not_every_iteration,
5601 /* Fill in the data about one biv update.
5602 V is the `struct induction' in which we record the biv. (It is
5603 allocated by the caller, with alloca.)
5604 INSN is the insn that sets it.
5605 DEST_REG is the biv's reg.
5607 MULT_VAL is const1_rtx if the biv is being incremented here, in which case
5608 INC_VAL is the increment. Otherwise, MULT_VAL is const0_rtx and the biv is
5609 being set to INC_VAL.
5611 NOT_EVERY_ITERATION is nonzero if this biv update is not know to be
5612 executed every iteration; MAYBE_MULTIPLE is nonzero if this biv update
5613 can be executed more than once per iteration. If MAYBE_MULTIPLE
5614 and NOT_EVERY_ITERATION are both zero, we know that the biv update is
5615 executed exactly once per iteration. */
5618 record_biv (struct loop *loop, struct induction *v, rtx insn, rtx dest_reg,
5619 rtx inc_val, rtx mult_val, rtx *location,
5620 int not_every_iteration, int maybe_multiple)
5622 struct loop_ivs *ivs = LOOP_IVS (loop);
5623 struct iv_class *bl;
5626 v->src_reg = dest_reg;
5627 v->dest_reg = dest_reg;
5628 v->mult_val = mult_val;
5629 v->add_val = inc_val;
5630 v->ext_dependent = NULL_RTX;
5631 v->location = location;
5632 v->mode = GET_MODE (dest_reg);
5633 v->always_computable = ! not_every_iteration;
5634 v->always_executed = ! not_every_iteration;
5635 v->maybe_multiple = maybe_multiple;
5638 /* Add this to the reg's iv_class, creating a class
5639 if this is the first incrementation of the reg. */
5641 bl = REG_IV_CLASS (ivs, REGNO (dest_reg));
5644 /* Create and initialize new iv_class. */
5646 bl = xmalloc (sizeof (struct iv_class));
5648 bl->regno = REGNO (dest_reg);
5654 /* Set initial value to the reg itself. */
5655 bl->initial_value = dest_reg;
5656 bl->final_value = 0;
5657 /* We haven't seen the initializing insn yet. */
5660 bl->initial_test = 0;
5661 bl->incremented = 0;
5665 bl->total_benefit = 0;
5667 /* Add this class to ivs->list. */
5668 bl->next = ivs->list;
5671 /* Put it in the array of biv register classes. */
5672 REG_IV_CLASS (ivs, REGNO (dest_reg)) = bl;
5676 /* Check if location is the same as a previous one. */
5677 struct induction *induction;
5678 for (induction = bl->biv; induction; induction = induction->next_iv)
5679 if (location == induction->location)
5681 v->same = induction;
5686 /* Update IV_CLASS entry for this biv. */
5687 v->next_iv = bl->biv;
5690 if (mult_val == const1_rtx)
5691 bl->incremented = 1;
5693 if (loop_dump_stream)
5694 loop_biv_dump (v, loop_dump_stream, 0);
5697 /* Fill in the data about one giv.
5698 V is the `struct induction' in which we record the giv. (It is
5699 allocated by the caller, with alloca.)
5700 INSN is the insn that sets it.
5701 BENEFIT estimates the savings from deleting this insn.
5702 TYPE is DEST_REG or DEST_ADDR; it says whether the giv is computed
5703 into a register or is used as a memory address.
5705 SRC_REG is the biv reg which the giv is computed from.
5706 DEST_REG is the giv's reg (if the giv is stored in a reg).
5707 MULT_VAL and ADD_VAL are the coefficients used to compute the giv.
5708 LOCATION points to the place where this giv's value appears in INSN. */
5711 record_giv (const struct loop *loop, struct induction *v, rtx insn,
5712 rtx src_reg, rtx dest_reg, rtx mult_val, rtx add_val,
5713 rtx ext_val, int benefit, enum g_types type,
5714 int not_every_iteration, int maybe_multiple, rtx *location)
5716 struct loop_ivs *ivs = LOOP_IVS (loop);
5717 struct induction *b;
5718 struct iv_class *bl;
5719 rtx set = single_set (insn);
5722 /* Attempt to prove constantness of the values. Don't let simplify_rtx
5723 undo the MULT canonicalization that we performed earlier. */
5724 temp = simplify_rtx (add_val);
5726 && ! (GET_CODE (add_val) == MULT
5727 && GET_CODE (temp) == ASHIFT))
5731 v->src_reg = src_reg;
5733 v->dest_reg = dest_reg;
5734 v->mult_val = mult_val;
5735 v->add_val = add_val;
5736 v->ext_dependent = ext_val;
5737 v->benefit = benefit;
5738 v->location = location;
5740 v->combined_with = 0;
5741 v->maybe_multiple = maybe_multiple;
5743 v->derive_adjustment = 0;
5749 v->auto_inc_opt = 0;
5753 /* The v->always_computable field is used in update_giv_derive, to
5754 determine whether a giv can be used to derive another giv. For a
5755 DEST_REG giv, INSN computes a new value for the giv, so its value
5756 isn't computable if INSN insn't executed every iteration.
5757 However, for a DEST_ADDR giv, INSN merely uses the value of the giv;
5758 it does not compute a new value. Hence the value is always computable
5759 regardless of whether INSN is executed each iteration. */
5761 if (type == DEST_ADDR)
5762 v->always_computable = 1;
5764 v->always_computable = ! not_every_iteration;
5766 v->always_executed = ! not_every_iteration;
5768 if (type == DEST_ADDR)
5770 v->mode = GET_MODE (*location);
5773 else /* type == DEST_REG */
5775 v->mode = GET_MODE (SET_DEST (set));
5777 v->lifetime = LOOP_REG_LIFETIME (loop, REGNO (dest_reg));
5779 /* If the lifetime is zero, it means that this register is
5780 really a dead store. So mark this as a giv that can be
5781 ignored. This will not prevent the biv from being eliminated. */
5782 if (v->lifetime == 0)
5785 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
5786 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
5789 /* Add the giv to the class of givs computed from one biv. */
5791 bl = REG_IV_CLASS (ivs, REGNO (src_reg));
5794 v->next_iv = bl->giv;
5796 /* Don't count DEST_ADDR. This is supposed to count the number of
5797 insns that calculate givs. */
5798 if (type == DEST_REG)
5800 bl->total_benefit += benefit;
5803 /* Fatal error, biv missing for this giv? */
5806 if (type == DEST_ADDR)
5809 v->not_replaceable = 0;
5813 /* The giv can be replaced outright by the reduced register only if all
5814 of the following conditions are true:
5815 - the insn that sets the giv is always executed on any iteration
5816 on which the giv is used at all
5817 (there are two ways to deduce this:
5818 either the insn is executed on every iteration,
5819 or all uses follow that insn in the same basic block),
5820 - the giv is not used outside the loop
5821 - no assignments to the biv occur during the giv's lifetime. */
5823 if (REGNO_FIRST_UID (REGNO (dest_reg)) == INSN_UID (insn)
5824 /* Previous line always fails if INSN was moved by loop opt. */
5825 && REGNO_LAST_LUID (REGNO (dest_reg))
5826 < INSN_LUID (loop->end)
5827 && (! not_every_iteration
5828 || last_use_this_basic_block (dest_reg, insn)))
5830 /* Now check that there are no assignments to the biv within the
5831 giv's lifetime. This requires two separate checks. */
5833 /* Check each biv update, and fail if any are between the first
5834 and last use of the giv.
5836 If this loop contains an inner loop that was unrolled, then
5837 the insn modifying the biv may have been emitted by the loop
5838 unrolling code, and hence does not have a valid luid. Just
5839 mark the biv as not replaceable in this case. It is not very
5840 useful as a biv, because it is used in two different loops.
5841 It is very unlikely that we would be able to optimize the giv
5842 using this biv anyways. */
5845 v->not_replaceable = 0;
5846 for (b = bl->biv; b; b = b->next_iv)
5848 if (INSN_UID (b->insn) >= max_uid_for_loop
5849 || ((INSN_LUID (b->insn)
5850 >= REGNO_FIRST_LUID (REGNO (dest_reg)))
5851 && (INSN_LUID (b->insn)
5852 <= REGNO_LAST_LUID (REGNO (dest_reg)))))
5855 v->not_replaceable = 1;
5860 /* If there are any backwards branches that go from after the
5861 biv update to before it, then this giv is not replaceable. */
5863 for (b = bl->biv; b; b = b->next_iv)
5864 if (back_branch_in_range_p (loop, b->insn))
5867 v->not_replaceable = 1;
5873 /* May still be replaceable, we don't have enough info here to
5876 v->not_replaceable = 0;
5880 /* Record whether the add_val contains a const_int, for later use by
5885 v->no_const_addval = 1;
5886 if (tem == const0_rtx)
5888 else if (CONSTANT_P (add_val))
5889 v->no_const_addval = 0;
5890 if (GET_CODE (tem) == PLUS)
5894 if (GET_CODE (XEXP (tem, 0)) == PLUS)
5895 tem = XEXP (tem, 0);
5896 else if (GET_CODE (XEXP (tem, 1)) == PLUS)
5897 tem = XEXP (tem, 1);
5901 if (CONSTANT_P (XEXP (tem, 1)))
5902 v->no_const_addval = 0;
5906 if (loop_dump_stream)
5907 loop_giv_dump (v, loop_dump_stream, 0);
5910 /* All this does is determine whether a giv can be made replaceable because
5911 its final value can be calculated. This code can not be part of record_giv
5912 above, because final_giv_value requires that the number of loop iterations
5913 be known, and that can not be accurately calculated until after all givs
5914 have been identified. */
5917 check_final_value (const struct loop *loop, struct induction *v)
5919 rtx final_value = 0;
5921 /* DEST_ADDR givs will never reach here, because they are always marked
5922 replaceable above in record_giv. */
5924 /* The giv can be replaced outright by the reduced register only if all
5925 of the following conditions are true:
5926 - the insn that sets the giv is always executed on any iteration
5927 on which the giv is used at all
5928 (there are two ways to deduce this:
5929 either the insn is executed on every iteration,
5930 or all uses follow that insn in the same basic block),
5931 - its final value can be calculated (this condition is different
5932 than the one above in record_giv)
5933 - it's not used before the it's set
5934 - no assignments to the biv occur during the giv's lifetime. */
5937 /* This is only called now when replaceable is known to be false. */
5938 /* Clear replaceable, so that it won't confuse final_giv_value. */
5942 if ((final_value = final_giv_value (loop, v))
5943 && (v->always_executed
5944 || last_use_this_basic_block (v->dest_reg, v->insn)))
5946 int biv_increment_seen = 0, before_giv_insn = 0;
5951 v->not_replaceable = 0;
5953 /* When trying to determine whether or not a biv increment occurs
5954 during the lifetime of the giv, we can ignore uses of the variable
5955 outside the loop because final_value is true. Hence we can not
5956 use regno_last_uid and regno_first_uid as above in record_giv. */
5958 /* Search the loop to determine whether any assignments to the
5959 biv occur during the giv's lifetime. Start with the insn
5960 that sets the giv, and search around the loop until we come
5961 back to that insn again.
5963 Also fail if there is a jump within the giv's lifetime that jumps
5964 to somewhere outside the lifetime but still within the loop. This
5965 catches spaghetti code where the execution order is not linear, and
5966 hence the above test fails. Here we assume that the giv lifetime
5967 does not extend from one iteration of the loop to the next, so as
5968 to make the test easier. Since the lifetime isn't known yet,
5969 this requires two loops. See also record_giv above. */
5971 last_giv_use = v->insn;
5978 before_giv_insn = 1;
5979 p = NEXT_INSN (loop->start);
5984 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5985 || GET_CODE (p) == CALL_INSN)
5987 /* It is possible for the BIV increment to use the GIV if we
5988 have a cycle. Thus we must be sure to check each insn for
5989 both BIV and GIV uses, and we must check for BIV uses
5992 if (! biv_increment_seen
5993 && reg_set_p (v->src_reg, PATTERN (p)))
5994 biv_increment_seen = 1;
5996 if (reg_mentioned_p (v->dest_reg, PATTERN (p)))
5998 if (biv_increment_seen || before_giv_insn)
6001 v->not_replaceable = 1;
6009 /* Now that the lifetime of the giv is known, check for branches
6010 from within the lifetime to outside the lifetime if it is still
6020 p = NEXT_INSN (loop->start);
6021 if (p == last_giv_use)
6024 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
6025 && LABEL_NAME (JUMP_LABEL (p))
6026 && ((loop_insn_first_p (JUMP_LABEL (p), v->insn)
6027 && loop_insn_first_p (loop->start, JUMP_LABEL (p)))
6028 || (loop_insn_first_p (last_giv_use, JUMP_LABEL (p))
6029 && loop_insn_first_p (JUMP_LABEL (p), loop->end))))
6032 v->not_replaceable = 1;
6034 if (loop_dump_stream)
6035 fprintf (loop_dump_stream,
6036 "Found branch outside giv lifetime.\n");
6043 /* If it is replaceable, then save the final value. */
6045 v->final_value = final_value;
6048 if (loop_dump_stream && v->replaceable)
6049 fprintf (loop_dump_stream, "Insn %d: giv reg %d final_value replaceable\n",
6050 INSN_UID (v->insn), REGNO (v->dest_reg));
6053 /* Update the status of whether a giv can derive other givs.
6055 We need to do something special if there is or may be an update to the biv
6056 between the time the giv is defined and the time it is used to derive
6059 In addition, a giv that is only conditionally set is not allowed to
6060 derive another giv once a label has been passed.
6062 The cases we look at are when a label or an update to a biv is passed. */
6065 update_giv_derive (const struct loop *loop, rtx p)
6067 struct loop_ivs *ivs = LOOP_IVS (loop);
6068 struct iv_class *bl;
6069 struct induction *biv, *giv;
6073 /* Search all IV classes, then all bivs, and finally all givs.
6075 There are three cases we are concerned with. First we have the situation
6076 of a giv that is only updated conditionally. In that case, it may not
6077 derive any givs after a label is passed.
6079 The second case is when a biv update occurs, or may occur, after the
6080 definition of a giv. For certain biv updates (see below) that are
6081 known to occur between the giv definition and use, we can adjust the
6082 giv definition. For others, or when the biv update is conditional,
6083 we must prevent the giv from deriving any other givs. There are two
6084 sub-cases within this case.
6086 If this is a label, we are concerned with any biv update that is done
6087 conditionally, since it may be done after the giv is defined followed by
6088 a branch here (actually, we need to pass both a jump and a label, but
6089 this extra tracking doesn't seem worth it).
6091 If this is a jump, we are concerned about any biv update that may be
6092 executed multiple times. We are actually only concerned about
6093 backward jumps, but it is probably not worth performing the test
6094 on the jump again here.
6096 If this is a biv update, we must adjust the giv status to show that a
6097 subsequent biv update was performed. If this adjustment cannot be done,
6098 the giv cannot derive further givs. */
6100 for (bl = ivs->list; bl; bl = bl->next)
6101 for (biv = bl->biv; biv; biv = biv->next_iv)
6102 if (GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN
6105 /* Skip if location is the same as a previous one. */
6109 for (giv = bl->giv; giv; giv = giv->next_iv)
6111 /* If cant_derive is already true, there is no point in
6112 checking all of these conditions again. */
6113 if (giv->cant_derive)
6116 /* If this giv is conditionally set and we have passed a label,
6117 it cannot derive anything. */
6118 if (GET_CODE (p) == CODE_LABEL && ! giv->always_computable)
6119 giv->cant_derive = 1;
6121 /* Skip givs that have mult_val == 0, since
6122 they are really invariants. Also skip those that are
6123 replaceable, since we know their lifetime doesn't contain
6125 else if (giv->mult_val == const0_rtx || giv->replaceable)
6128 /* The only way we can allow this giv to derive another
6129 is if this is a biv increment and we can form the product
6130 of biv->add_val and giv->mult_val. In this case, we will
6131 be able to compute a compensation. */
6132 else if (biv->insn == p)
6137 if (biv->mult_val == const1_rtx)
6138 tem = simplify_giv_expr (loop,
6139 gen_rtx_MULT (giv->mode,
6142 &ext_val_dummy, &dummy);
6144 if (tem && giv->derive_adjustment)
6145 tem = simplify_giv_expr
6147 gen_rtx_PLUS (giv->mode, tem, giv->derive_adjustment),
6148 &ext_val_dummy, &dummy);
6151 giv->derive_adjustment = tem;
6153 giv->cant_derive = 1;
6155 else if ((GET_CODE (p) == CODE_LABEL && ! biv->always_computable)
6156 || (GET_CODE (p) == JUMP_INSN && biv->maybe_multiple))
6157 giv->cant_derive = 1;
6162 /* Check whether an insn is an increment legitimate for a basic induction var.
6163 X is the source of insn P, or a part of it.
6164 MODE is the mode in which X should be interpreted.
6166 DEST_REG is the putative biv, also the destination of the insn.
6167 We accept patterns of these forms:
6168 REG = REG + INVARIANT (includes REG = REG - CONSTANT)
6169 REG = INVARIANT + REG
6171 If X is suitable, we return 1, set *MULT_VAL to CONST1_RTX,
6172 store the additive term into *INC_VAL, and store the place where
6173 we found the additive term into *LOCATION.
6175 If X is an assignment of an invariant into DEST_REG, we set
6176 *MULT_VAL to CONST0_RTX, and store the invariant into *INC_VAL.
6178 We also want to detect a BIV when it corresponds to a variable
6179 whose mode was promoted via PROMOTED_MODE. In that case, an increment
6180 of the variable may be a PLUS that adds a SUBREG of that variable to
6181 an invariant and then sign- or zero-extends the result of the PLUS
6184 Most GIVs in such cases will be in the promoted mode, since that is the
6185 probably the natural computation mode (and almost certainly the mode
6186 used for addresses) on the machine. So we view the pseudo-reg containing
6187 the variable as the BIV, as if it were simply incremented.
6189 Note that treating the entire pseudo as a BIV will result in making
6190 simple increments to any GIVs based on it. However, if the variable
6191 overflows in its declared mode but not its promoted mode, the result will
6192 be incorrect. This is acceptable if the variable is signed, since
6193 overflows in such cases are undefined, but not if it is unsigned, since
6194 those overflows are defined. So we only check for SIGN_EXTEND and
6197 If we cannot find a biv, we return 0. */
6200 basic_induction_var (const struct loop *loop, rtx x, enum machine_mode mode,
6201 rtx dest_reg, rtx p, rtx *inc_val, rtx *mult_val,
6206 rtx insn, set = 0, last, inc;
6208 code = GET_CODE (x);
6213 if (rtx_equal_p (XEXP (x, 0), dest_reg)
6214 || (GET_CODE (XEXP (x, 0)) == SUBREG
6215 && SUBREG_PROMOTED_VAR_P (XEXP (x, 0))
6216 && SUBREG_REG (XEXP (x, 0)) == dest_reg))
6218 argp = &XEXP (x, 1);
6220 else if (rtx_equal_p (XEXP (x, 1), dest_reg)
6221 || (GET_CODE (XEXP (x, 1)) == SUBREG
6222 && SUBREG_PROMOTED_VAR_P (XEXP (x, 1))
6223 && SUBREG_REG (XEXP (x, 1)) == dest_reg))
6225 argp = &XEXP (x, 0);
6231 if (loop_invariant_p (loop, arg) != 1)
6234 /* convert_modes can emit new instructions, e.g. when arg is a loop
6235 invariant MEM and dest_reg has a different mode.
6236 These instructions would be emitted after the end of the function
6237 and then *inc_val would be an uninitialized pseudo.
6238 Detect this and bail in this case.
6239 Other alternatives to solve this can be introducing a convert_modes
6240 variant which is allowed to fail but not allowed to emit new
6241 instructions, emit these instructions before loop start and let
6242 it be garbage collected if *inc_val is never used or saving the
6243 *inc_val initialization sequence generated here and when *inc_val
6244 is going to be actually used, emit it at some suitable place. */
6245 last = get_last_insn ();
6246 inc = convert_modes (GET_MODE (dest_reg), GET_MODE (x), arg, 0);
6247 if (get_last_insn () != last)
6249 delete_insns_since (last);
6254 *mult_val = const1_rtx;
6259 /* If what's inside the SUBREG is a BIV, then the SUBREG. This will
6260 handle addition of promoted variables.
6261 ??? The comment at the start of this function is wrong: promoted
6262 variable increments don't look like it says they do. */
6263 return basic_induction_var (loop, SUBREG_REG (x),
6264 GET_MODE (SUBREG_REG (x)),
6265 dest_reg, p, inc_val, mult_val, location);
6268 /* If this register is assigned in a previous insn, look at its
6269 source, but don't go outside the loop or past a label. */
6271 /* If this sets a register to itself, we would repeat any previous
6272 biv increment if we applied this strategy blindly. */
6273 if (rtx_equal_p (dest_reg, x))
6282 insn = PREV_INSN (insn);
6284 while (insn && GET_CODE (insn) == NOTE
6285 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6289 set = single_set (insn);
6292 dest = SET_DEST (set);
6294 || (GET_CODE (dest) == SUBREG
6295 && (GET_MODE_SIZE (GET_MODE (dest)) <= UNITS_PER_WORD)
6296 && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_INT)
6297 && SUBREG_REG (dest) == x))
6298 return basic_induction_var (loop, SET_SRC (set),
6299 (GET_MODE (SET_SRC (set)) == VOIDmode
6301 : GET_MODE (SET_SRC (set))),
6303 inc_val, mult_val, location);
6305 while (GET_CODE (dest) == SIGN_EXTRACT
6306 || GET_CODE (dest) == ZERO_EXTRACT
6307 || GET_CODE (dest) == SUBREG
6308 || GET_CODE (dest) == STRICT_LOW_PART)
6309 dest = XEXP (dest, 0);
6315 /* Can accept constant setting of biv only when inside inner most loop.
6316 Otherwise, a biv of an inner loop may be incorrectly recognized
6317 as a biv of the outer loop,
6318 causing code to be moved INTO the inner loop. */
6320 if (loop_invariant_p (loop, x) != 1)
6325 /* convert_modes aborts if we try to convert to or from CCmode, so just
6326 exclude that case. It is very unlikely that a condition code value
6327 would be a useful iterator anyways. convert_modes aborts if we try to
6328 convert a float mode to non-float or vice versa too. */
6329 if (loop->level == 1
6330 && GET_MODE_CLASS (mode) == GET_MODE_CLASS (GET_MODE (dest_reg))
6331 && GET_MODE_CLASS (mode) != MODE_CC)
6333 /* Possible bug here? Perhaps we don't know the mode of X. */
6334 last = get_last_insn ();
6335 inc = convert_modes (GET_MODE (dest_reg), mode, x, 0);
6336 if (get_last_insn () != last)
6338 delete_insns_since (last);
6343 *mult_val = const0_rtx;
6350 /* Ignore this BIV if signed arithmetic overflow is defined. */
6353 return basic_induction_var (loop, XEXP (x, 0), GET_MODE (XEXP (x, 0)),
6354 dest_reg, p, inc_val, mult_val, location);
6357 /* Similar, since this can be a sign extension. */
6358 for (insn = PREV_INSN (p);
6359 (insn && GET_CODE (insn) == NOTE
6360 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6361 insn = PREV_INSN (insn))
6365 set = single_set (insn);
6367 if (! rtx_equal_p (dest_reg, XEXP (x, 0))
6368 && set && SET_DEST (set) == XEXP (x, 0)
6369 && GET_CODE (XEXP (x, 1)) == CONST_INT
6370 && INTVAL (XEXP (x, 1)) >= 0
6371 && GET_CODE (SET_SRC (set)) == ASHIFT
6372 && XEXP (x, 1) == XEXP (SET_SRC (set), 1))
6373 return basic_induction_var (loop, XEXP (SET_SRC (set), 0),
6374 GET_MODE (XEXP (x, 0)),
6375 dest_reg, insn, inc_val, mult_val,
6384 /* A general induction variable (giv) is any quantity that is a linear
6385 function of a basic induction variable,
6386 i.e. giv = biv * mult_val + add_val.
6387 The coefficients can be any loop invariant quantity.
6388 A giv need not be computed directly from the biv;
6389 it can be computed by way of other givs. */
6391 /* Determine whether X computes a giv.
6392 If it does, return a nonzero value
6393 which is the benefit from eliminating the computation of X;
6394 set *SRC_REG to the register of the biv that it is computed from;
6395 set *ADD_VAL and *MULT_VAL to the coefficients,
6396 such that the value of X is biv * mult + add; */
6399 general_induction_var (const struct loop *loop, rtx x, rtx *src_reg,
6400 rtx *add_val, rtx *mult_val, rtx *ext_val,
6401 int is_addr, int *pbenefit,
6402 enum machine_mode addr_mode)
6404 struct loop_ivs *ivs = LOOP_IVS (loop);
6407 /* If this is an invariant, forget it, it isn't a giv. */
6408 if (loop_invariant_p (loop, x) == 1)
6412 *ext_val = NULL_RTX;
6413 x = simplify_giv_expr (loop, x, ext_val, pbenefit);
6417 switch (GET_CODE (x))
6421 /* Since this is now an invariant and wasn't before, it must be a giv
6422 with MULT_VAL == 0. It doesn't matter which BIV we associate this
6424 *src_reg = ivs->list->biv->dest_reg;
6425 *mult_val = const0_rtx;
6430 /* This is equivalent to a BIV. */
6432 *mult_val = const1_rtx;
6433 *add_val = const0_rtx;
6437 /* Either (plus (biv) (invar)) or
6438 (plus (mult (biv) (invar_1)) (invar_2)). */
6439 if (GET_CODE (XEXP (x, 0)) == MULT)
6441 *src_reg = XEXP (XEXP (x, 0), 0);
6442 *mult_val = XEXP (XEXP (x, 0), 1);
6446 *src_reg = XEXP (x, 0);
6447 *mult_val = const1_rtx;
6449 *add_val = XEXP (x, 1);
6453 /* ADD_VAL is zero. */
6454 *src_reg = XEXP (x, 0);
6455 *mult_val = XEXP (x, 1);
6456 *add_val = const0_rtx;
6463 /* Remove any enclosing USE from ADD_VAL and MULT_VAL (there will be
6464 unless they are CONST_INT). */
6465 if (GET_CODE (*add_val) == USE)
6466 *add_val = XEXP (*add_val, 0);
6467 if (GET_CODE (*mult_val) == USE)
6468 *mult_val = XEXP (*mult_val, 0);
6471 *pbenefit += address_cost (orig_x, addr_mode) - reg_address_cost;
6473 *pbenefit += rtx_cost (orig_x, SET);
6475 /* Always return true if this is a giv so it will be detected as such,
6476 even if the benefit is zero or negative. This allows elimination
6477 of bivs that might otherwise not be eliminated. */
6481 /* Given an expression, X, try to form it as a linear function of a biv.
6482 We will canonicalize it to be of the form
6483 (plus (mult (BIV) (invar_1))
6485 with possible degeneracies.
6487 The invariant expressions must each be of a form that can be used as a
6488 machine operand. We surround then with a USE rtx (a hack, but localized
6489 and certainly unambiguous!) if not a CONST_INT for simplicity in this
6490 routine; it is the caller's responsibility to strip them.
6492 If no such canonicalization is possible (i.e., two biv's are used or an
6493 expression that is neither invariant nor a biv or giv), this routine
6496 For a nonzero return, the result will have a code of CONST_INT, USE,
6497 REG (for a BIV), PLUS, or MULT. No other codes will occur.
6499 *BENEFIT will be incremented by the benefit of any sub-giv encountered. */
6501 static rtx sge_plus (enum machine_mode, rtx, rtx);
6502 static rtx sge_plus_constant (rtx, rtx);
6505 simplify_giv_expr (const struct loop *loop, rtx x, rtx *ext_val, int *benefit)
6507 struct loop_ivs *ivs = LOOP_IVS (loop);
6508 struct loop_regs *regs = LOOP_REGS (loop);
6509 enum machine_mode mode = GET_MODE (x);
6513 /* If this is not an integer mode, or if we cannot do arithmetic in this
6514 mode, this can't be a giv. */
6515 if (mode != VOIDmode
6516 && (GET_MODE_CLASS (mode) != MODE_INT
6517 || GET_MODE_BITSIZE (mode) > HOST_BITS_PER_WIDE_INT))
6520 switch (GET_CODE (x))
6523 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6524 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6525 if (arg0 == 0 || arg1 == 0)
6528 /* Put constant last, CONST_INT last if both constant. */
6529 if ((GET_CODE (arg0) == USE
6530 || GET_CODE (arg0) == CONST_INT)
6531 && ! ((GET_CODE (arg0) == USE
6532 && GET_CODE (arg1) == USE)
6533 || GET_CODE (arg1) == CONST_INT))
6534 tem = arg0, arg0 = arg1, arg1 = tem;
6536 /* Handle addition of zero, then addition of an invariant. */
6537 if (arg1 == const0_rtx)
6539 else if (GET_CODE (arg1) == CONST_INT || GET_CODE (arg1) == USE)
6540 switch (GET_CODE (arg0))
6544 /* Adding two invariants must result in an invariant, so enclose
6545 addition operation inside a USE and return it. */
6546 if (GET_CODE (arg0) == USE)
6547 arg0 = XEXP (arg0, 0);
6548 if (GET_CODE (arg1) == USE)
6549 arg1 = XEXP (arg1, 0);
6551 if (GET_CODE (arg0) == CONST_INT)
6552 tem = arg0, arg0 = arg1, arg1 = tem;
6553 if (GET_CODE (arg1) == CONST_INT)
6554 tem = sge_plus_constant (arg0, arg1);
6556 tem = sge_plus (mode, arg0, arg1);
6558 if (GET_CODE (tem) != CONST_INT)
6559 tem = gen_rtx_USE (mode, tem);
6564 /* biv + invar or mult + invar. Return sum. */
6565 return gen_rtx_PLUS (mode, arg0, arg1);
6568 /* (a + invar_1) + invar_2. Associate. */
6570 simplify_giv_expr (loop,
6582 /* Each argument must be either REG, PLUS, or MULT. Convert REG to
6583 MULT to reduce cases. */
6584 if (GET_CODE (arg0) == REG)
6585 arg0 = gen_rtx_MULT (mode, arg0, const1_rtx);
6586 if (GET_CODE (arg1) == REG)
6587 arg1 = gen_rtx_MULT (mode, arg1, const1_rtx);
6589 /* Now have PLUS + PLUS, PLUS + MULT, MULT + PLUS, or MULT + MULT.
6590 Put a MULT first, leaving PLUS + PLUS, MULT + PLUS, or MULT + MULT.
6591 Recurse to associate the second PLUS. */
6592 if (GET_CODE (arg1) == MULT)
6593 tem = arg0, arg0 = arg1, arg1 = tem;
6595 if (GET_CODE (arg1) == PLUS)
6597 simplify_giv_expr (loop,
6599 gen_rtx_PLUS (mode, arg0,
6604 /* Now must have MULT + MULT. Distribute if same biv, else not giv. */
6605 if (GET_CODE (arg0) != MULT || GET_CODE (arg1) != MULT)
6608 if (!rtx_equal_p (arg0, arg1))
6611 return simplify_giv_expr (loop,
6620 /* Handle "a - b" as "a + b * (-1)". */
6621 return simplify_giv_expr (loop,
6630 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6631 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6632 if (arg0 == 0 || arg1 == 0)
6635 /* Put constant last, CONST_INT last if both constant. */
6636 if ((GET_CODE (arg0) == USE || GET_CODE (arg0) == CONST_INT)
6637 && GET_CODE (arg1) != CONST_INT)
6638 tem = arg0, arg0 = arg1, arg1 = tem;
6640 /* If second argument is not now constant, not giv. */
6641 if (GET_CODE (arg1) != USE && GET_CODE (arg1) != CONST_INT)
6644 /* Handle multiply by 0 or 1. */
6645 if (arg1 == const0_rtx)
6648 else if (arg1 == const1_rtx)
6651 switch (GET_CODE (arg0))
6654 /* biv * invar. Done. */
6655 return gen_rtx_MULT (mode, arg0, arg1);
6658 /* Product of two constants. */
6659 return GEN_INT (INTVAL (arg0) * INTVAL (arg1));
6662 /* invar * invar is a giv, but attempt to simplify it somehow. */
6663 if (GET_CODE (arg1) != CONST_INT)
6666 arg0 = XEXP (arg0, 0);
6667 if (GET_CODE (arg0) == MULT)
6669 /* (invar_0 * invar_1) * invar_2. Associate. */
6670 return simplify_giv_expr (loop,
6679 /* Propagate the MULT expressions to the innermost nodes. */
6680 else if (GET_CODE (arg0) == PLUS)
6682 /* (invar_0 + invar_1) * invar_2. Distribute. */
6683 return simplify_giv_expr (loop,
6695 return gen_rtx_USE (mode, gen_rtx_MULT (mode, arg0, arg1));
6698 /* (a * invar_1) * invar_2. Associate. */
6699 return simplify_giv_expr (loop,
6708 /* (a + invar_1) * invar_2. Distribute. */
6709 return simplify_giv_expr (loop,
6724 /* Shift by constant is multiply by power of two. */
6725 if (GET_CODE (XEXP (x, 1)) != CONST_INT)
6729 simplify_giv_expr (loop,
6732 GEN_INT ((HOST_WIDE_INT) 1
6733 << INTVAL (XEXP (x, 1)))),
6737 /* "-a" is "a * (-1)" */
6738 return simplify_giv_expr (loop,
6739 gen_rtx_MULT (mode, XEXP (x, 0), constm1_rtx),
6743 /* "~a" is "-a - 1". Silly, but easy. */
6744 return simplify_giv_expr (loop,
6745 gen_rtx_MINUS (mode,
6746 gen_rtx_NEG (mode, XEXP (x, 0)),
6751 /* Already in proper form for invariant. */
6757 /* Conditionally recognize extensions of simple IVs. After we've
6758 computed loop traversal counts and verified the range of the
6759 source IV, we'll reevaluate this as a GIV. */
6760 if (*ext_val == NULL_RTX)
6762 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6763 if (arg0 && *ext_val == NULL_RTX && GET_CODE (arg0) == REG)
6765 *ext_val = gen_rtx_fmt_e (GET_CODE (x), mode, arg0);
6772 /* If this is a new register, we can't deal with it. */
6773 if (REGNO (x) >= max_reg_before_loop)
6776 /* Check for biv or giv. */
6777 switch (REG_IV_TYPE (ivs, REGNO (x)))
6781 case GENERAL_INDUCT:
6783 struct induction *v = REG_IV_INFO (ivs, REGNO (x));
6785 /* Form expression from giv and add benefit. Ensure this giv
6786 can derive another and subtract any needed adjustment if so. */
6788 /* Increasing the benefit here is risky. The only case in which it
6789 is arguably correct is if this is the only use of V. In other
6790 cases, this will artificially inflate the benefit of the current
6791 giv, and lead to suboptimal code. Thus, it is disabled, since
6792 potentially not reducing an only marginally beneficial giv is
6793 less harmful than reducing many givs that are not really
6796 rtx single_use = regs->array[REGNO (x)].single_usage;
6797 if (single_use && single_use != const0_rtx)
6798 *benefit += v->benefit;
6804 tem = gen_rtx_PLUS (mode, gen_rtx_MULT (mode,
6805 v->src_reg, v->mult_val),
6808 if (v->derive_adjustment)
6809 tem = gen_rtx_MINUS (mode, tem, v->derive_adjustment);
6810 arg0 = simplify_giv_expr (loop, tem, ext_val, benefit);
6813 if (!v->ext_dependent)
6818 *ext_val = v->ext_dependent;
6826 /* If it isn't an induction variable, and it is invariant, we
6827 may be able to simplify things further by looking through
6828 the bits we just moved outside the loop. */
6829 if (loop_invariant_p (loop, x) == 1)
6832 struct loop_movables *movables = LOOP_MOVABLES (loop);
6834 for (m = movables->head; m; m = m->next)
6835 if (rtx_equal_p (x, m->set_dest))
6837 /* Ok, we found a match. Substitute and simplify. */
6839 /* If we match another movable, we must use that, as
6840 this one is going away. */
6842 return simplify_giv_expr (loop, m->match->set_dest,
6845 /* If consec is nonzero, this is a member of a group of
6846 instructions that were moved together. We handle this
6847 case only to the point of seeking to the last insn and
6848 looking for a REG_EQUAL. Fail if we don't find one. */
6855 tem = NEXT_INSN (tem);
6859 tem = find_reg_note (tem, REG_EQUAL, NULL_RTX);
6861 tem = XEXP (tem, 0);
6865 tem = single_set (m->insn);
6867 tem = SET_SRC (tem);
6872 /* What we are most interested in is pointer
6873 arithmetic on invariants -- only take
6874 patterns we may be able to do something with. */
6875 if (GET_CODE (tem) == PLUS
6876 || GET_CODE (tem) == MULT
6877 || GET_CODE (tem) == ASHIFT
6878 || GET_CODE (tem) == CONST_INT
6879 || GET_CODE (tem) == SYMBOL_REF)
6881 tem = simplify_giv_expr (loop, tem, ext_val,
6886 else if (GET_CODE (tem) == CONST
6887 && GET_CODE (XEXP (tem, 0)) == PLUS
6888 && GET_CODE (XEXP (XEXP (tem, 0), 0)) == SYMBOL_REF
6889 && GET_CODE (XEXP (XEXP (tem, 0), 1)) == CONST_INT)
6891 tem = simplify_giv_expr (loop, XEXP (tem, 0),
6903 /* Fall through to general case. */
6905 /* If invariant, return as USE (unless CONST_INT).
6906 Otherwise, not giv. */
6907 if (GET_CODE (x) == USE)
6910 if (loop_invariant_p (loop, x) == 1)
6912 if (GET_CODE (x) == CONST_INT)
6914 if (GET_CODE (x) == CONST
6915 && GET_CODE (XEXP (x, 0)) == PLUS
6916 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
6917 && GET_CODE (XEXP (XEXP (x, 0), 1)) == CONST_INT)
6919 return gen_rtx_USE (mode, x);
6926 /* This routine folds invariants such that there is only ever one
6927 CONST_INT in the summation. It is only used by simplify_giv_expr. */
6930 sge_plus_constant (rtx x, rtx c)
6932 if (GET_CODE (x) == CONST_INT)
6933 return GEN_INT (INTVAL (x) + INTVAL (c));
6934 else if (GET_CODE (x) != PLUS)
6935 return gen_rtx_PLUS (GET_MODE (x), x, c);
6936 else if (GET_CODE (XEXP (x, 1)) == CONST_INT)
6938 return gen_rtx_PLUS (GET_MODE (x), XEXP (x, 0),
6939 GEN_INT (INTVAL (XEXP (x, 1)) + INTVAL (c)));
6941 else if (GET_CODE (XEXP (x, 0)) == PLUS
6942 || GET_CODE (XEXP (x, 1)) != PLUS)
6944 return gen_rtx_PLUS (GET_MODE (x),
6945 sge_plus_constant (XEXP (x, 0), c), XEXP (x, 1));
6949 return gen_rtx_PLUS (GET_MODE (x),
6950 sge_plus_constant (XEXP (x, 1), c), XEXP (x, 0));
6955 sge_plus (enum machine_mode mode, rtx x, rtx y)
6957 while (GET_CODE (y) == PLUS)
6959 rtx a = XEXP (y, 0);
6960 if (GET_CODE (a) == CONST_INT)
6961 x = sge_plus_constant (x, a);
6963 x = gen_rtx_PLUS (mode, x, a);
6966 if (GET_CODE (y) == CONST_INT)
6967 x = sge_plus_constant (x, y);
6969 x = gen_rtx_PLUS (mode, x, y);
6973 /* Help detect a giv that is calculated by several consecutive insns;
6977 The caller has already identified the first insn P as having a giv as dest;
6978 we check that all other insns that set the same register follow
6979 immediately after P, that they alter nothing else,
6980 and that the result of the last is still a giv.
6982 The value is 0 if the reg set in P is not really a giv.
6983 Otherwise, the value is the amount gained by eliminating
6984 all the consecutive insns that compute the value.
6986 FIRST_BENEFIT is the amount gained by eliminating the first insn, P.
6987 SRC_REG is the reg of the biv; DEST_REG is the reg of the giv.
6989 The coefficients of the ultimate giv value are stored in
6990 *MULT_VAL and *ADD_VAL. */
6993 consec_sets_giv (const struct loop *loop, int first_benefit, rtx p,
6994 rtx src_reg, rtx dest_reg, rtx *add_val, rtx *mult_val,
6995 rtx *ext_val, rtx *last_consec_insn)
6997 struct loop_ivs *ivs = LOOP_IVS (loop);
6998 struct loop_regs *regs = LOOP_REGS (loop);
7005 /* Indicate that this is a giv so that we can update the value produced in
7006 each insn of the multi-insn sequence.
7008 This induction structure will be used only by the call to
7009 general_induction_var below, so we can allocate it on our stack.
7010 If this is a giv, our caller will replace the induct var entry with
7011 a new induction structure. */
7012 struct induction *v;
7014 if (REG_IV_TYPE (ivs, REGNO (dest_reg)) != UNKNOWN_INDUCT)
7017 v = alloca (sizeof (struct induction));
7018 v->src_reg = src_reg;
7019 v->mult_val = *mult_val;
7020 v->add_val = *add_val;
7021 v->benefit = first_benefit;
7023 v->derive_adjustment = 0;
7024 v->ext_dependent = NULL_RTX;
7026 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
7027 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
7029 count = regs->array[REGNO (dest_reg)].n_times_set - 1;
7034 code = GET_CODE (p);
7036 /* If libcall, skip to end of call sequence. */
7037 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
7041 && (set = single_set (p))
7042 && GET_CODE (SET_DEST (set)) == REG
7043 && SET_DEST (set) == dest_reg
7044 && (general_induction_var (loop, SET_SRC (set), &src_reg,
7045 add_val, mult_val, ext_val, 0,
7047 /* Giv created by equivalent expression. */
7048 || ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX))
7049 && general_induction_var (loop, XEXP (temp, 0), &src_reg,
7050 add_val, mult_val, ext_val, 0,
7051 &benefit, VOIDmode)))
7052 && src_reg == v->src_reg)
7054 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
7055 benefit += libcall_benefit (p);
7058 v->mult_val = *mult_val;
7059 v->add_val = *add_val;
7060 v->benefit += benefit;
7062 else if (code != NOTE)
7064 /* Allow insns that set something other than this giv to a
7065 constant. Such insns are needed on machines which cannot
7066 include long constants and should not disqualify a giv. */
7068 && (set = single_set (p))
7069 && SET_DEST (set) != dest_reg
7070 && CONSTANT_P (SET_SRC (set)))
7073 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7078 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7079 *last_consec_insn = p;
7083 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7084 represented by G1. If no such expression can be found, or it is clear that
7085 it cannot possibly be a valid address, 0 is returned.
7087 To perform the computation, we note that
7090 where `v' is the biv.
7092 So G2 = (y/b) * G1 + (b - a*y/x).
7094 Note that MULT = y/x.
7096 Update: A and B are now allowed to be additive expressions such that
7097 B contains all variables in A. That is, computing B-A will not require
7098 subtracting variables. */
7101 express_from_1 (rtx a, rtx b, rtx mult)
7103 /* If MULT is zero, then A*MULT is zero, and our expression is B. */
7105 if (mult == const0_rtx)
7108 /* If MULT is not 1, we cannot handle A with non-constants, since we
7109 would then be required to subtract multiples of the registers in A.
7110 This is theoretically possible, and may even apply to some Fortran
7111 constructs, but it is a lot of work and we do not attempt it here. */
7113 if (mult != const1_rtx && GET_CODE (a) != CONST_INT)
7116 /* In general these structures are sorted top to bottom (down the PLUS
7117 chain), but not left to right across the PLUS. If B is a higher
7118 order giv than A, we can strip one level and recurse. If A is higher
7119 order, we'll eventually bail out, but won't know that until the end.
7120 If they are the same, we'll strip one level around this loop. */
7122 while (GET_CODE (a) == PLUS && GET_CODE (b) == PLUS)
7124 rtx ra, rb, oa, ob, tmp;
7126 ra = XEXP (a, 0), oa = XEXP (a, 1);
7127 if (GET_CODE (ra) == PLUS)
7128 tmp = ra, ra = oa, oa = tmp;
7130 rb = XEXP (b, 0), ob = XEXP (b, 1);
7131 if (GET_CODE (rb) == PLUS)
7132 tmp = rb, rb = ob, ob = tmp;
7134 if (rtx_equal_p (ra, rb))
7135 /* We matched: remove one reg completely. */
7137 else if (GET_CODE (ob) != PLUS && rtx_equal_p (ra, ob))
7138 /* An alternate match. */
7140 else if (GET_CODE (oa) != PLUS && rtx_equal_p (oa, rb))
7141 /* An alternate match. */
7145 /* Indicates an extra register in B. Strip one level from B and
7146 recurse, hoping B was the higher order expression. */
7147 ob = express_from_1 (a, ob, mult);
7150 return gen_rtx_PLUS (GET_MODE (b), rb, ob);
7154 /* Here we are at the last level of A, go through the cases hoping to
7155 get rid of everything but a constant. */
7157 if (GET_CODE (a) == PLUS)
7161 ra = XEXP (a, 0), oa = XEXP (a, 1);
7162 if (rtx_equal_p (oa, b))
7164 else if (!rtx_equal_p (ra, b))
7167 if (GET_CODE (oa) != CONST_INT)
7170 return GEN_INT (-INTVAL (oa) * INTVAL (mult));
7172 else if (GET_CODE (a) == CONST_INT)
7174 return plus_constant (b, -INTVAL (a) * INTVAL (mult));
7176 else if (CONSTANT_P (a))
7178 enum machine_mode mode_a = GET_MODE (a);
7179 enum machine_mode mode_b = GET_MODE (b);
7180 enum machine_mode mode = mode_b == VOIDmode ? mode_a : mode_b;
7181 return simplify_gen_binary (MINUS, mode, b, a);
7183 else if (GET_CODE (b) == PLUS)
7185 if (rtx_equal_p (a, XEXP (b, 0)))
7187 else if (rtx_equal_p (a, XEXP (b, 1)))
7192 else if (rtx_equal_p (a, b))
7199 express_from (struct induction *g1, struct induction *g2)
7203 /* The value that G1 will be multiplied by must be a constant integer. Also,
7204 the only chance we have of getting a valid address is if b*c/a (see above
7205 for notation) is also an integer. */
7206 if (GET_CODE (g1->mult_val) == CONST_INT
7207 && GET_CODE (g2->mult_val) == CONST_INT)
7209 if (g1->mult_val == const0_rtx
7210 || (g1->mult_val == constm1_rtx
7211 && INTVAL (g2->mult_val)
7212 == (HOST_WIDE_INT) 1 << (HOST_BITS_PER_WIDE_INT - 1))
7213 || INTVAL (g2->mult_val) % INTVAL (g1->mult_val) != 0)
7215 mult = GEN_INT (INTVAL (g2->mult_val) / INTVAL (g1->mult_val));
7217 else if (rtx_equal_p (g1->mult_val, g2->mult_val))
7221 /* ??? Find out if the one is a multiple of the other? */
7225 add = express_from_1 (g1->add_val, g2->add_val, mult);
7226 if (add == NULL_RTX)
7228 /* Failed. If we've got a multiplication factor between G1 and G2,
7229 scale G1's addend and try again. */
7230 if (INTVAL (mult) > 1)
7232 rtx g1_add_val = g1->add_val;
7233 if (GET_CODE (g1_add_val) == MULT
7234 && GET_CODE (XEXP (g1_add_val, 1)) == CONST_INT)
7237 m = INTVAL (mult) * INTVAL (XEXP (g1_add_val, 1));
7238 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val),
7239 XEXP (g1_add_val, 0), GEN_INT (m));
7243 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val), g1_add_val,
7247 add = express_from_1 (g1_add_val, g2->add_val, const1_rtx);
7250 if (add == NULL_RTX)
7253 /* Form simplified final result. */
7254 if (mult == const0_rtx)
7256 else if (mult == const1_rtx)
7257 mult = g1->dest_reg;
7259 mult = gen_rtx_MULT (g2->mode, g1->dest_reg, mult);
7261 if (add == const0_rtx)
7265 if (GET_CODE (add) == PLUS
7266 && CONSTANT_P (XEXP (add, 1)))
7268 rtx tem = XEXP (add, 1);
7269 mult = gen_rtx_PLUS (g2->mode, mult, XEXP (add, 0));
7273 return gen_rtx_PLUS (g2->mode, mult, add);
7277 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7278 represented by G1. This indicates that G2 should be combined with G1 and
7279 that G2 can use (either directly or via an address expression) a register
7280 used to represent G1. */
7283 combine_givs_p (struct induction *g1, struct induction *g2)
7287 /* With the introduction of ext dependent givs, we must care for modes.
7288 G2 must not use a wider mode than G1. */
7289 if (GET_MODE_SIZE (g1->mode) < GET_MODE_SIZE (g2->mode))
7292 ret = comb = express_from (g1, g2);
7293 if (comb == NULL_RTX)
7295 if (g1->mode != g2->mode)
7296 ret = gen_lowpart (g2->mode, comb);
7298 /* If these givs are identical, they can be combined. We use the results
7299 of express_from because the addends are not in a canonical form, so
7300 rtx_equal_p is a weaker test. */
7301 /* But don't combine a DEST_REG giv with a DEST_ADDR giv; we want the
7302 combination to be the other way round. */
7303 if (comb == g1->dest_reg
7304 && (g1->giv_type == DEST_REG || g2->giv_type == DEST_ADDR))
7309 /* If G2 can be expressed as a function of G1 and that function is valid
7310 as an address and no more expensive than using a register for G2,
7311 the expression of G2 in terms of G1 can be used. */
7313 && g2->giv_type == DEST_ADDR
7314 && memory_address_p (GET_MODE (g2->mem), ret))
7320 /* Check each extension dependent giv in this class to see if its
7321 root biv is safe from wrapping in the interior mode, which would
7322 make the giv illegal. */
7325 check_ext_dependent_givs (const struct loop *loop, struct iv_class *bl)
7327 struct loop_info *loop_info = LOOP_INFO (loop);
7328 int ze_ok = 0, se_ok = 0, info_ok = 0;
7329 enum machine_mode biv_mode = GET_MODE (bl->biv->src_reg);
7330 HOST_WIDE_INT start_val;
7331 unsigned HOST_WIDE_INT u_end_val = 0;
7332 unsigned HOST_WIDE_INT u_start_val = 0;
7334 struct induction *v;
7336 /* Make sure the iteration data is available. We must have
7337 constants in order to be certain of no overflow. */
7338 if (loop_info->n_iterations > 0
7339 && bl->initial_value
7340 && GET_CODE (bl->initial_value) == CONST_INT
7341 && (incr = biv_total_increment (bl))
7342 && GET_CODE (incr) == CONST_INT
7343 /* Make sure the host can represent the arithmetic. */
7344 && HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (biv_mode))
7346 unsigned HOST_WIDE_INT abs_incr, total_incr;
7347 HOST_WIDE_INT s_end_val;
7351 start_val = INTVAL (bl->initial_value);
7352 u_start_val = start_val;
7354 neg_incr = 0, abs_incr = INTVAL (incr);
7355 if (INTVAL (incr) < 0)
7356 neg_incr = 1, abs_incr = -abs_incr;
7357 total_incr = abs_incr * loop_info->n_iterations;
7359 /* Check for host arithmetic overflow. */
7360 if (total_incr / loop_info->n_iterations == abs_incr)
7362 unsigned HOST_WIDE_INT u_max;
7363 HOST_WIDE_INT s_max;
7365 u_end_val = start_val + (neg_incr ? -total_incr : total_incr);
7366 s_end_val = u_end_val;
7367 u_max = GET_MODE_MASK (biv_mode);
7370 /* Check zero extension of biv ok. */
7372 /* Check for host arithmetic overflow. */
7374 ? u_end_val < u_start_val
7375 : u_end_val > u_start_val)
7376 /* Check for target arithmetic overflow. */
7378 ? 1 /* taken care of with host overflow */
7379 : u_end_val <= u_max))
7384 /* Check sign extension of biv ok. */
7385 /* ??? While it is true that overflow with signed and pointer
7386 arithmetic is undefined, I fear too many programmers don't
7387 keep this fact in mind -- myself included on occasion.
7388 So leave alone with the signed overflow optimizations. */
7389 if (start_val >= -s_max - 1
7390 /* Check for host arithmetic overflow. */
7392 ? s_end_val < start_val
7393 : s_end_val > start_val)
7394 /* Check for target arithmetic overflow. */
7396 ? s_end_val >= -s_max - 1
7397 : s_end_val <= s_max))
7404 /* If we know the BIV is compared at run-time against an
7405 invariant value, and the increment is +/- 1, we may also
7406 be able to prove that the BIV cannot overflow. */
7407 else if (bl->biv->src_reg == loop_info->iteration_var
7408 && loop_info->comparison_value
7409 && loop_invariant_p (loop, loop_info->comparison_value)
7410 && (incr = biv_total_increment (bl))
7411 && GET_CODE (incr) == CONST_INT)
7413 /* If the increment is +1, and the exit test is a <,
7414 the BIV cannot overflow. (For <=, we have the
7415 problematic case that the comparison value might
7416 be the maximum value of the range.) */
7417 if (INTVAL (incr) == 1)
7419 if (loop_info->comparison_code == LT)
7421 else if (loop_info->comparison_code == LTU)
7425 /* Likewise for increment -1 and exit test >. */
7426 if (INTVAL (incr) == -1)
7428 if (loop_info->comparison_code == GT)
7430 else if (loop_info->comparison_code == GTU)
7435 /* Invalidate givs that fail the tests. */
7436 for (v = bl->giv; v; v = v->next_iv)
7437 if (v->ext_dependent)
7439 enum rtx_code code = GET_CODE (v->ext_dependent);
7452 /* We don't know whether this value is being used as either
7453 signed or unsigned, so to safely truncate we must satisfy
7454 both. The initial check here verifies the BIV itself;
7455 once that is successful we may check its range wrt the
7456 derived GIV. This works only if we were able to determine
7457 constant start and end values above. */
7458 if (se_ok && ze_ok && info_ok)
7460 enum machine_mode outer_mode = GET_MODE (v->ext_dependent);
7461 unsigned HOST_WIDE_INT max = GET_MODE_MASK (outer_mode) >> 1;
7463 /* We know from the above that both endpoints are nonnegative,
7464 and that there is no wrapping. Verify that both endpoints
7465 are within the (signed) range of the outer mode. */
7466 if (u_start_val <= max && u_end_val <= max)
7477 if (loop_dump_stream)
7479 fprintf (loop_dump_stream,
7480 "Verified ext dependent giv at %d of reg %d\n",
7481 INSN_UID (v->insn), bl->regno);
7486 if (loop_dump_stream)
7491 why = "biv iteration values overflowed";
7495 incr = biv_total_increment (bl);
7496 if (incr == const1_rtx)
7497 why = "biv iteration info incomplete; incr by 1";
7499 why = "biv iteration info incomplete";
7502 fprintf (loop_dump_stream,
7503 "Failed ext dependent giv at %d, %s\n",
7504 INSN_UID (v->insn), why);
7507 bl->all_reduced = 0;
7512 /* Generate a version of VALUE in a mode appropriate for initializing V. */
7515 extend_value_for_giv (struct induction *v, rtx value)
7517 rtx ext_dep = v->ext_dependent;
7522 /* Recall that check_ext_dependent_givs verified that the known bounds
7523 of a biv did not overflow or wrap with respect to the extension for
7524 the giv. Therefore, constants need no additional adjustment. */
7525 if (CONSTANT_P (value) && GET_MODE (value) == VOIDmode)
7528 /* Otherwise, we must adjust the value to compensate for the
7529 differing modes of the biv and the giv. */
7530 return gen_rtx_fmt_e (GET_CODE (ext_dep), GET_MODE (ext_dep), value);
7533 struct combine_givs_stats
7540 cmp_combine_givs_stats (const void *xp, const void *yp)
7542 const struct combine_givs_stats * const x =
7543 (const struct combine_givs_stats *) xp;
7544 const struct combine_givs_stats * const y =
7545 (const struct combine_givs_stats *) yp;
7547 d = y->total_benefit - x->total_benefit;
7548 /* Stabilize the sort. */
7550 d = x->giv_number - y->giv_number;
7554 /* Check all pairs of givs for iv_class BL and see if any can be combined with
7555 any other. If so, point SAME to the giv combined with and set NEW_REG to
7556 be an expression (in terms of the other giv's DEST_REG) equivalent to the
7557 giv. Also, update BENEFIT and related fields for cost/benefit analysis. */
7560 combine_givs (struct loop_regs *regs, struct iv_class *bl)
7562 /* Additional benefit to add for being combined multiple times. */
7563 const int extra_benefit = 3;
7565 struct induction *g1, *g2, **giv_array;
7566 int i, j, k, giv_count;
7567 struct combine_givs_stats *stats;
7570 /* Count givs, because bl->giv_count is incorrect here. */
7572 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7576 giv_array = alloca (giv_count * sizeof (struct induction *));
7578 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7580 giv_array[i++] = g1;
7582 stats = xcalloc (giv_count, sizeof (*stats));
7583 can_combine = xcalloc (giv_count, giv_count * sizeof (rtx));
7585 for (i = 0; i < giv_count; i++)
7591 stats[i].giv_number = i;
7593 /* If a DEST_REG GIV is used only once, do not allow it to combine
7594 with anything, for in doing so we will gain nothing that cannot
7595 be had by simply letting the GIV with which we would have combined
7596 to be reduced on its own. The losage shows up in particular with
7597 DEST_ADDR targets on hosts with reg+reg addressing, though it can
7598 be seen elsewhere as well. */
7599 if (g1->giv_type == DEST_REG
7600 && (single_use = regs->array[REGNO (g1->dest_reg)].single_usage)
7601 && single_use != const0_rtx)
7604 this_benefit = g1->benefit;
7605 /* Add an additional weight for zero addends. */
7606 if (g1->no_const_addval)
7609 for (j = 0; j < giv_count; j++)
7615 && (this_combine = combine_givs_p (g1, g2)) != NULL_RTX)
7617 can_combine[i * giv_count + j] = this_combine;
7618 this_benefit += g2->benefit + extra_benefit;
7621 stats[i].total_benefit = this_benefit;
7624 /* Iterate, combining until we can't. */
7626 qsort (stats, giv_count, sizeof (*stats), cmp_combine_givs_stats);
7628 if (loop_dump_stream)
7630 fprintf (loop_dump_stream, "Sorted combine statistics:\n");
7631 for (k = 0; k < giv_count; k++)
7633 g1 = giv_array[stats[k].giv_number];
7634 if (!g1->combined_with && !g1->same)
7635 fprintf (loop_dump_stream, " {%d, %d}",
7636 INSN_UID (giv_array[stats[k].giv_number]->insn),
7637 stats[k].total_benefit);
7639 putc ('\n', loop_dump_stream);
7642 for (k = 0; k < giv_count; k++)
7644 int g1_add_benefit = 0;
7646 i = stats[k].giv_number;
7649 /* If it has already been combined, skip. */
7650 if (g1->combined_with || g1->same)
7653 for (j = 0; j < giv_count; j++)
7656 if (g1 != g2 && can_combine[i * giv_count + j]
7657 /* If it has already been combined, skip. */
7658 && ! g2->same && ! g2->combined_with)
7662 g2->new_reg = can_combine[i * giv_count + j];
7664 /* For destination, we now may replace by mem expression instead
7665 of register. This changes the costs considerably, so add the
7667 if (g2->giv_type == DEST_ADDR)
7668 g2->benefit = (g2->benefit + reg_address_cost
7669 - address_cost (g2->new_reg,
7670 GET_MODE (g2->mem)));
7671 g1->combined_with++;
7672 g1->lifetime += g2->lifetime;
7674 g1_add_benefit += g2->benefit;
7676 /* ??? The new final_[bg]iv_value code does a much better job
7677 of finding replaceable giv's, and hence this code may no
7678 longer be necessary. */
7679 if (! g2->replaceable && REG_USERVAR_P (g2->dest_reg))
7680 g1_add_benefit -= copy_cost;
7682 /* To help optimize the next set of combinations, remove
7683 this giv from the benefits of other potential mates. */
7684 for (l = 0; l < giv_count; ++l)
7686 int m = stats[l].giv_number;
7687 if (can_combine[m * giv_count + j])
7688 stats[l].total_benefit -= g2->benefit + extra_benefit;
7691 if (loop_dump_stream)
7692 fprintf (loop_dump_stream,
7693 "giv at %d combined with giv at %d; new benefit %d + %d, lifetime %d\n",
7694 INSN_UID (g2->insn), INSN_UID (g1->insn),
7695 g1->benefit, g1_add_benefit, g1->lifetime);
7699 /* To help optimize the next set of combinations, remove
7700 this giv from the benefits of other potential mates. */
7701 if (g1->combined_with)
7703 for (j = 0; j < giv_count; ++j)
7705 int m = stats[j].giv_number;
7706 if (can_combine[m * giv_count + i])
7707 stats[j].total_benefit -= g1->benefit + extra_benefit;
7710 g1->benefit += g1_add_benefit;
7712 /* We've finished with this giv, and everything it touched.
7713 Restart the combination so that proper weights for the
7714 rest of the givs are properly taken into account. */
7715 /* ??? Ideally we would compact the arrays at this point, so
7716 as to not cover old ground. But sanely compacting
7717 can_combine is tricky. */
7727 /* Generate sequence for REG = B * M + A. B is the initial value of
7728 the basic induction variable, M a multiplicative constant, A an
7729 additive constant and REG the destination register. */
7732 gen_add_mult (rtx b, rtx m, rtx a, rtx reg)
7738 /* Use unsigned arithmetic. */
7739 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7741 emit_move_insn (reg, result);
7749 /* Update registers created in insn sequence SEQ. */
7752 loop_regs_update (const struct loop *loop ATTRIBUTE_UNUSED, rtx seq)
7756 /* Update register info for alias analysis. */
7759 while (insn != NULL_RTX)
7761 rtx set = single_set (insn);
7763 if (set && GET_CODE (SET_DEST (set)) == REG)
7764 record_base_value (REGNO (SET_DEST (set)), SET_SRC (set), 0);
7766 insn = NEXT_INSN (insn);
7771 /* EMIT code before BEFORE_BB/BEFORE_INSN to set REG = B * M + A. B
7772 is the initial value of the basic induction variable, M a
7773 multiplicative constant, A an additive constant and REG the
7774 destination register. */
7777 loop_iv_add_mult_emit_before (const struct loop *loop, rtx b, rtx m, rtx a,
7778 rtx reg, basic_block before_bb, rtx before_insn)
7784 loop_iv_add_mult_hoist (loop, b, m, a, reg);
7788 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7789 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7791 /* Increase the lifetime of any invariants moved further in code. */
7792 update_reg_last_use (a, before_insn);
7793 update_reg_last_use (b, before_insn);
7794 update_reg_last_use (m, before_insn);
7796 /* It is possible that the expansion created lots of new registers.
7797 Iterate over the sequence we just created and record them all. We
7798 must do this before inserting the sequence. */
7799 loop_regs_update (loop, seq);
7801 loop_insn_emit_before (loop, before_bb, before_insn, seq);
7805 /* Emit insns in loop pre-header to set REG = B * M + A. B is the
7806 initial value of the basic induction variable, M a multiplicative
7807 constant, A an additive constant and REG the destination
7811 loop_iv_add_mult_sink (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7815 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7816 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7818 /* Increase the lifetime of any invariants moved further in code.
7819 ???? Is this really necessary? */
7820 update_reg_last_use (a, loop->sink);
7821 update_reg_last_use (b, loop->sink);
7822 update_reg_last_use (m, loop->sink);
7824 /* It is possible that the expansion created lots of new registers.
7825 Iterate over the sequence we just created and record them all. We
7826 must do this before inserting the sequence. */
7827 loop_regs_update (loop, seq);
7829 loop_insn_sink (loop, seq);
7833 /* Emit insns after loop to set REG = B * M + A. B is the initial
7834 value of the basic induction variable, M a multiplicative constant,
7835 A an additive constant and REG the destination register. */
7838 loop_iv_add_mult_hoist (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7842 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7843 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7845 /* It is possible that the expansion created lots of new registers.
7846 Iterate over the sequence we just created and record them all. We
7847 must do this before inserting the sequence. */
7848 loop_regs_update (loop, seq);
7850 loop_insn_hoist (loop, seq);
7855 /* Similar to gen_add_mult, but compute cost rather than generating
7859 iv_add_mult_cost (rtx b, rtx m, rtx a, rtx reg)
7865 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7867 emit_move_insn (reg, result);
7868 last = get_last_insn ();
7871 rtx t = single_set (last);
7873 cost += rtx_cost (SET_SRC (t), SET);
7874 last = PREV_INSN (last);
7880 /* Test whether A * B can be computed without
7881 an actual multiply insn. Value is 1 if so.
7883 ??? This function stinks because it generates a ton of wasted RTL
7884 ??? and as a result fragments GC memory to no end. There are other
7885 ??? places in the compiler which are invoked a lot and do the same
7886 ??? thing, generate wasted RTL just to see if something is possible. */
7889 product_cheap_p (rtx a, rtx b)
7894 /* If only one is constant, make it B. */
7895 if (GET_CODE (a) == CONST_INT)
7896 tmp = a, a = b, b = tmp;
7898 /* If first constant, both constant, so don't need multiply. */
7899 if (GET_CODE (a) == CONST_INT)
7902 /* If second not constant, neither is constant, so would need multiply. */
7903 if (GET_CODE (b) != CONST_INT)
7906 /* One operand is constant, so might not need multiply insn. Generate the
7907 code for the multiply and see if a call or multiply, or long sequence
7908 of insns is generated. */
7911 expand_mult (GET_MODE (a), a, b, NULL_RTX, 1);
7919 while (tmp != NULL_RTX)
7921 rtx next = NEXT_INSN (tmp);
7924 || GET_CODE (tmp) != INSN
7925 || (GET_CODE (PATTERN (tmp)) == SET
7926 && GET_CODE (SET_SRC (PATTERN (tmp))) == MULT)
7927 || (GET_CODE (PATTERN (tmp)) == PARALLEL
7928 && GET_CODE (XVECEXP (PATTERN (tmp), 0, 0)) == SET
7929 && GET_CODE (SET_SRC (XVECEXP (PATTERN (tmp), 0, 0))) == MULT))
7938 else if (GET_CODE (tmp) == SET
7939 && GET_CODE (SET_SRC (tmp)) == MULT)
7941 else if (GET_CODE (tmp) == PARALLEL
7942 && GET_CODE (XVECEXP (tmp, 0, 0)) == SET
7943 && GET_CODE (SET_SRC (XVECEXP (tmp, 0, 0))) == MULT)
7949 /* Check to see if loop can be terminated by a "decrement and branch until
7950 zero" instruction. If so, add a REG_NONNEG note to the branch insn if so.
7951 Also try reversing an increment loop to a decrement loop
7952 to see if the optimization can be performed.
7953 Value is nonzero if optimization was performed. */
7955 /* This is useful even if the architecture doesn't have such an insn,
7956 because it might change a loops which increments from 0 to n to a loop
7957 which decrements from n to 0. A loop that decrements to zero is usually
7958 faster than one that increments from zero. */
7960 /* ??? This could be rewritten to use some of the loop unrolling procedures,
7961 such as approx_final_value, biv_total_increment, loop_iterations, and
7962 final_[bg]iv_value. */
7965 check_dbra_loop (struct loop *loop, int insn_count)
7967 struct loop_info *loop_info = LOOP_INFO (loop);
7968 struct loop_regs *regs = LOOP_REGS (loop);
7969 struct loop_ivs *ivs = LOOP_IVS (loop);
7970 struct iv_class *bl;
7977 rtx before_comparison;
7981 int compare_and_branch;
7982 rtx loop_start = loop->start;
7983 rtx loop_end = loop->end;
7985 /* If last insn is a conditional branch, and the insn before tests a
7986 register value, try to optimize it. Otherwise, we can't do anything. */
7988 jump = PREV_INSN (loop_end);
7989 comparison = get_condition_for_loop (loop, jump);
7990 if (comparison == 0)
7992 if (!onlyjump_p (jump))
7995 /* Try to compute whether the compare/branch at the loop end is one or
7996 two instructions. */
7997 get_condition (jump, &first_compare, false);
7998 if (first_compare == jump)
7999 compare_and_branch = 1;
8000 else if (first_compare == prev_nonnote_insn (jump))
8001 compare_and_branch = 2;
8006 /* If more than one condition is present to control the loop, then
8007 do not proceed, as this function does not know how to rewrite
8008 loop tests with more than one condition.
8010 Look backwards from the first insn in the last comparison
8011 sequence and see if we've got another comparison sequence. */
8014 if ((jump1 = prev_nonnote_insn (first_compare)) != loop->cont)
8015 if (GET_CODE (jump1) == JUMP_INSN)
8019 /* Check all of the bivs to see if the compare uses one of them.
8020 Skip biv's set more than once because we can't guarantee that
8021 it will be zero on the last iteration. Also skip if the biv is
8022 used between its update and the test insn. */
8024 for (bl = ivs->list; bl; bl = bl->next)
8026 if (bl->biv_count == 1
8027 && ! bl->biv->maybe_multiple
8028 && bl->biv->dest_reg == XEXP (comparison, 0)
8029 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8034 /* Try swapping the comparison to identify a suitable biv. */
8036 for (bl = ivs->list; bl; bl = bl->next)
8037 if (bl->biv_count == 1
8038 && ! bl->biv->maybe_multiple
8039 && bl->biv->dest_reg == XEXP (comparison, 1)
8040 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8043 comparison = gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)),
8045 XEXP (comparison, 1),
8046 XEXP (comparison, 0));
8053 /* Look for the case where the basic induction variable is always
8054 nonnegative, and equals zero on the last iteration.
8055 In this case, add a reg_note REG_NONNEG, which allows the
8056 m68k DBRA instruction to be used. */
8058 if (((GET_CODE (comparison) == GT && XEXP (comparison, 1) == constm1_rtx)
8059 || (GET_CODE (comparison) == NE && XEXP (comparison, 1) == const0_rtx))
8060 && GET_CODE (bl->biv->add_val) == CONST_INT
8061 && INTVAL (bl->biv->add_val) < 0)
8063 /* Initial value must be greater than 0,
8064 init_val % -dec_value == 0 to ensure that it equals zero on
8065 the last iteration */
8067 if (GET_CODE (bl->initial_value) == CONST_INT
8068 && INTVAL (bl->initial_value) > 0
8069 && (INTVAL (bl->initial_value)
8070 % (-INTVAL (bl->biv->add_val))) == 0)
8072 /* register always nonnegative, add REG_NOTE to branch */
8073 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8075 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8082 /* If the decrement is 1 and the value was tested as >= 0 before
8083 the loop, then we can safely optimize. */
8084 for (p = loop_start; p; p = PREV_INSN (p))
8086 if (GET_CODE (p) == CODE_LABEL)
8088 if (GET_CODE (p) != JUMP_INSN)
8091 before_comparison = get_condition_for_loop (loop, p);
8092 if (before_comparison
8093 && XEXP (before_comparison, 0) == bl->biv->dest_reg
8094 && (GET_CODE (before_comparison) == LT
8095 || GET_CODE (before_comparison) == LTU)
8096 && XEXP (before_comparison, 1) == const0_rtx
8097 && ! reg_set_between_p (bl->biv->dest_reg, p, loop_start)
8098 && INTVAL (bl->biv->add_val) == -1)
8100 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8102 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8110 else if (GET_CODE (bl->biv->add_val) == CONST_INT
8111 && INTVAL (bl->biv->add_val) > 0)
8113 /* Try to change inc to dec, so can apply above optimization. */
8115 all registers modified are induction variables or invariant,
8116 all memory references have non-overlapping addresses
8117 (obviously true if only one write)
8118 allow 2 insns for the compare/jump at the end of the loop. */
8119 /* Also, we must avoid any instructions which use both the reversed
8120 biv and another biv. Such instructions will fail if the loop is
8121 reversed. We meet this condition by requiring that either
8122 no_use_except_counting is true, or else that there is only
8124 int num_nonfixed_reads = 0;
8125 /* 1 if the iteration var is used only to count iterations. */
8126 int no_use_except_counting = 0;
8127 /* 1 if the loop has no memory store, or it has a single memory store
8128 which is reversible. */
8129 int reversible_mem_store = 1;
8131 if (bl->giv_count == 0
8132 && !loop->exit_count
8133 && !loop_info->has_multiple_exit_targets)
8135 rtx bivreg = regno_reg_rtx[bl->regno];
8136 struct iv_class *blt;
8138 /* If there are no givs for this biv, and the only exit is the
8139 fall through at the end of the loop, then
8140 see if perhaps there are no uses except to count. */
8141 no_use_except_counting = 1;
8142 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8145 rtx set = single_set (p);
8147 if (set && GET_CODE (SET_DEST (set)) == REG
8148 && REGNO (SET_DEST (set)) == bl->regno)
8149 /* An insn that sets the biv is okay. */
8151 else if (!reg_mentioned_p (bivreg, PATTERN (p)))
8152 /* An insn that doesn't mention the biv is okay. */
8154 else if (p == prev_nonnote_insn (prev_nonnote_insn (loop_end))
8155 || p == prev_nonnote_insn (loop_end))
8157 /* If either of these insns uses the biv and sets a pseudo
8158 that has more than one usage, then the biv has uses
8159 other than counting since it's used to derive a value
8160 that is used more than one time. */
8161 note_stores (PATTERN (p), note_set_pseudo_multiple_uses,
8163 if (regs->multiple_uses)
8165 no_use_except_counting = 0;
8171 no_use_except_counting = 0;
8176 /* A biv has uses besides counting if it is used to set
8178 for (blt = ivs->list; blt; blt = blt->next)
8180 && reg_mentioned_p (bivreg, SET_SRC (blt->init_set)))
8182 no_use_except_counting = 0;
8187 if (no_use_except_counting)
8188 /* No need to worry about MEMs. */
8190 else if (loop_info->num_mem_sets <= 1)
8192 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8194 num_nonfixed_reads += count_nonfixed_reads (loop, PATTERN (p));
8196 /* If the loop has a single store, and the destination address is
8197 invariant, then we can't reverse the loop, because this address
8198 might then have the wrong value at loop exit.
8199 This would work if the source was invariant also, however, in that
8200 case, the insn should have been moved out of the loop. */
8202 if (loop_info->num_mem_sets == 1)
8204 struct induction *v;
8206 /* If we could prove that each of the memory locations
8207 written to was different, then we could reverse the
8208 store -- but we don't presently have any way of
8210 reversible_mem_store = 0;
8212 /* If the store depends on a register that is set after the
8213 store, it depends on the initial value, and is thus not
8215 for (v = bl->giv; reversible_mem_store && v; v = v->next_iv)
8217 if (v->giv_type == DEST_REG
8218 && reg_mentioned_p (v->dest_reg,
8219 PATTERN (loop_info->first_loop_store_insn))
8220 && loop_insn_first_p (loop_info->first_loop_store_insn,
8222 reversible_mem_store = 0;
8229 /* This code only acts for innermost loops. Also it simplifies
8230 the memory address check by only reversing loops with
8231 zero or one memory access.
8232 Two memory accesses could involve parts of the same array,
8233 and that can't be reversed.
8234 If the biv is used only for counting, than we don't need to worry
8235 about all these things. */
8237 if ((num_nonfixed_reads <= 1
8238 && ! loop_info->has_nonconst_call
8239 && ! loop_info->has_prefetch
8240 && ! loop_info->has_volatile
8241 && reversible_mem_store
8242 && (bl->giv_count + bl->biv_count + loop_info->num_mem_sets
8243 + num_unmoved_movables (loop) + compare_and_branch == insn_count)
8244 && (bl == ivs->list && bl->next == 0))
8245 || (no_use_except_counting && ! loop_info->has_prefetch))
8249 /* Loop can be reversed. */
8250 if (loop_dump_stream)
8251 fprintf (loop_dump_stream, "Can reverse loop\n");
8253 /* Now check other conditions:
8255 The increment must be a constant, as must the initial value,
8256 and the comparison code must be LT.
8258 This test can probably be improved since +/- 1 in the constant
8259 can be obtained by changing LT to LE and vice versa; this is
8263 /* for constants, LE gets turned into LT */
8264 && (GET_CODE (comparison) == LT
8265 || (GET_CODE (comparison) == LE
8266 && no_use_except_counting)
8267 || GET_CODE (comparison) == LTU))
8269 HOST_WIDE_INT add_val, add_adjust, comparison_val = 0;
8270 rtx initial_value, comparison_value;
8272 enum rtx_code cmp_code;
8273 int comparison_const_width;
8274 unsigned HOST_WIDE_INT comparison_sign_mask;
8276 add_val = INTVAL (bl->biv->add_val);
8277 comparison_value = XEXP (comparison, 1);
8278 if (GET_MODE (comparison_value) == VOIDmode)
8279 comparison_const_width
8280 = GET_MODE_BITSIZE (GET_MODE (XEXP (comparison, 0)));
8282 comparison_const_width
8283 = GET_MODE_BITSIZE (GET_MODE (comparison_value));
8284 if (comparison_const_width > HOST_BITS_PER_WIDE_INT)
8285 comparison_const_width = HOST_BITS_PER_WIDE_INT;
8286 comparison_sign_mask
8287 = (unsigned HOST_WIDE_INT) 1 << (comparison_const_width - 1);
8289 /* If the comparison value is not a loop invariant, then we
8290 can not reverse this loop.
8292 ??? If the insns which initialize the comparison value as
8293 a whole compute an invariant result, then we could move
8294 them out of the loop and proceed with loop reversal. */
8295 if (! loop_invariant_p (loop, comparison_value))
8298 if (GET_CODE (comparison_value) == CONST_INT)
8299 comparison_val = INTVAL (comparison_value);
8300 initial_value = bl->initial_value;
8302 /* Normalize the initial value if it is an integer and
8303 has no other use except as a counter. This will allow
8304 a few more loops to be reversed. */
8305 if (no_use_except_counting
8306 && GET_CODE (comparison_value) == CONST_INT
8307 && GET_CODE (initial_value) == CONST_INT)
8309 comparison_val = comparison_val - INTVAL (bl->initial_value);
8310 /* The code below requires comparison_val to be a multiple
8311 of add_val in order to do the loop reversal, so
8312 round up comparison_val to a multiple of add_val.
8313 Since comparison_value is constant, we know that the
8314 current comparison code is LT. */
8315 comparison_val = comparison_val + add_val - 1;
8317 -= (unsigned HOST_WIDE_INT) comparison_val % add_val;
8318 /* We postpone overflow checks for COMPARISON_VAL here;
8319 even if there is an overflow, we might still be able to
8320 reverse the loop, if converting the loop exit test to
8322 initial_value = const0_rtx;
8325 /* First check if we can do a vanilla loop reversal. */
8326 if (initial_value == const0_rtx
8327 /* If we have a decrement_and_branch_on_count,
8328 prefer the NE test, since this will allow that
8329 instruction to be generated. Note that we must
8330 use a vanilla loop reversal if the biv is used to
8331 calculate a giv or has a non-counting use. */
8332 #if ! defined (HAVE_decrement_and_branch_until_zero) \
8333 && defined (HAVE_decrement_and_branch_on_count)
8334 && (! (add_val == 1 && loop->vtop
8335 && (bl->biv_count == 0
8336 || no_use_except_counting)))
8338 && GET_CODE (comparison_value) == CONST_INT
8339 /* Now do postponed overflow checks on COMPARISON_VAL. */
8340 && ! (((comparison_val - add_val) ^ INTVAL (comparison_value))
8341 & comparison_sign_mask))
8343 /* Register will always be nonnegative, with value
8344 0 on last iteration */
8345 add_adjust = add_val;
8349 else if (add_val == 1 && loop->vtop
8350 && (bl->biv_count == 0
8351 || no_use_except_counting))
8359 if (GET_CODE (comparison) == LE)
8360 add_adjust -= add_val;
8362 /* If the initial value is not zero, or if the comparison
8363 value is not an exact multiple of the increment, then we
8364 can not reverse this loop. */
8365 if (initial_value == const0_rtx
8366 && GET_CODE (comparison_value) == CONST_INT)
8368 if (((unsigned HOST_WIDE_INT) comparison_val % add_val) != 0)
8373 if (! no_use_except_counting || add_val != 1)
8377 final_value = comparison_value;
8379 /* Reset these in case we normalized the initial value
8380 and comparison value above. */
8381 if (GET_CODE (comparison_value) == CONST_INT
8382 && GET_CODE (initial_value) == CONST_INT)
8384 comparison_value = GEN_INT (comparison_val);
8386 = GEN_INT (comparison_val + INTVAL (bl->initial_value));
8388 bl->initial_value = initial_value;
8390 /* Save some info needed to produce the new insns. */
8391 reg = bl->biv->dest_reg;
8392 jump_label = condjump_label (PREV_INSN (loop_end));
8393 new_add_val = GEN_INT (-INTVAL (bl->biv->add_val));
8395 /* Set start_value; if this is not a CONST_INT, we need
8397 Initialize biv to start_value before loop start.
8398 The old initializing insn will be deleted as a
8399 dead store by flow.c. */
8400 if (initial_value == const0_rtx
8401 && GET_CODE (comparison_value) == CONST_INT)
8403 start_value = GEN_INT (comparison_val - add_adjust);
8404 loop_insn_hoist (loop, gen_move_insn (reg, start_value));
8406 else if (GET_CODE (initial_value) == CONST_INT)
8408 enum machine_mode mode = GET_MODE (reg);
8409 rtx offset = GEN_INT (-INTVAL (initial_value) - add_adjust);
8410 rtx add_insn = gen_add3_insn (reg, comparison_value, offset);
8416 = gen_rtx_PLUS (mode, comparison_value, offset);
8417 loop_insn_hoist (loop, add_insn);
8418 if (GET_CODE (comparison) == LE)
8419 final_value = gen_rtx_PLUS (mode, comparison_value,
8422 else if (! add_adjust)
8424 enum machine_mode mode = GET_MODE (reg);
8425 rtx sub_insn = gen_sub3_insn (reg, comparison_value,
8431 = gen_rtx_MINUS (mode, comparison_value, initial_value);
8432 loop_insn_hoist (loop, sub_insn);
8435 /* We could handle the other cases too, but it'll be
8436 better to have a testcase first. */
8439 /* We may not have a single insn which can increment a reg, so
8440 create a sequence to hold all the insns from expand_inc. */
8442 expand_inc (reg, new_add_val);
8446 p = loop_insn_emit_before (loop, 0, bl->biv->insn, tem);
8447 delete_insn (bl->biv->insn);
8449 /* Update biv info to reflect its new status. */
8451 bl->initial_value = start_value;
8452 bl->biv->add_val = new_add_val;
8454 /* Update loop info. */
8455 loop_info->initial_value = reg;
8456 loop_info->initial_equiv_value = reg;
8457 loop_info->final_value = const0_rtx;
8458 loop_info->final_equiv_value = const0_rtx;
8459 loop_info->comparison_value = const0_rtx;
8460 loop_info->comparison_code = cmp_code;
8461 loop_info->increment = new_add_val;
8463 /* Inc LABEL_NUSES so that delete_insn will
8464 not delete the label. */
8465 LABEL_NUSES (XEXP (jump_label, 0))++;
8467 /* Emit an insn after the end of the loop to set the biv's
8468 proper exit value if it is used anywhere outside the loop. */
8469 if ((REGNO_LAST_UID (bl->regno) != INSN_UID (first_compare))
8471 || REGNO_FIRST_UID (bl->regno) != INSN_UID (bl->init_insn))
8472 loop_insn_sink (loop, gen_load_of_final_value (reg, final_value));
8474 /* Delete compare/branch at end of loop. */
8475 delete_related_insns (PREV_INSN (loop_end));
8476 if (compare_and_branch == 2)
8477 delete_related_insns (first_compare);
8479 /* Add new compare/branch insn at end of loop. */
8481 emit_cmp_and_jump_insns (reg, const0_rtx, cmp_code, NULL_RTX,
8483 XEXP (jump_label, 0));
8486 emit_jump_insn_before (tem, loop_end);
8488 for (tem = PREV_INSN (loop_end);
8489 tem && GET_CODE (tem) != JUMP_INSN;
8490 tem = PREV_INSN (tem))
8494 JUMP_LABEL (tem) = XEXP (jump_label, 0);
8500 /* Increment of LABEL_NUSES done above. */
8501 /* Register is now always nonnegative,
8502 so add REG_NONNEG note to the branch. */
8503 REG_NOTES (tem) = gen_rtx_EXPR_LIST (REG_NONNEG, reg,
8509 /* No insn may reference both the reversed and another biv or it
8510 will fail (see comment near the top of the loop reversal
8512 Earlier on, we have verified that the biv has no use except
8513 counting, or it is the only biv in this function.
8514 However, the code that computes no_use_except_counting does
8515 not verify reg notes. It's possible to have an insn that
8516 references another biv, and has a REG_EQUAL note with an
8517 expression based on the reversed biv. To avoid this case,
8518 remove all REG_EQUAL notes based on the reversed biv
8520 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8524 rtx set = single_set (p);
8525 /* If this is a set of a GIV based on the reversed biv, any
8526 REG_EQUAL notes should still be correct. */
8528 || GET_CODE (SET_DEST (set)) != REG
8529 || (size_t) REGNO (SET_DEST (set)) >= ivs->n_regs
8530 || REG_IV_TYPE (ivs, REGNO (SET_DEST (set))) != GENERAL_INDUCT
8531 || REG_IV_INFO (ivs, REGNO (SET_DEST (set)))->src_reg != bl->biv->src_reg)
8532 for (pnote = ®_NOTES (p); *pnote;)
8534 if (REG_NOTE_KIND (*pnote) == REG_EQUAL
8535 && reg_mentioned_p (regno_reg_rtx[bl->regno],
8537 *pnote = XEXP (*pnote, 1);
8539 pnote = &XEXP (*pnote, 1);
8543 /* Mark that this biv has been reversed. Each giv which depends
8544 on this biv, and which is also live past the end of the loop
8545 will have to be fixed up. */
8549 if (loop_dump_stream)
8551 fprintf (loop_dump_stream, "Reversed loop");
8553 fprintf (loop_dump_stream, " and added reg_nonneg\n");
8555 fprintf (loop_dump_stream, "\n");
8566 /* Verify whether the biv BL appears to be eliminable,
8567 based on the insns in the loop that refer to it.
8569 If ELIMINATE_P is nonzero, actually do the elimination.
8571 THRESHOLD and INSN_COUNT are from loop_optimize and are used to
8572 determine whether invariant insns should be placed inside or at the
8573 start of the loop. */
8576 maybe_eliminate_biv (const struct loop *loop, struct iv_class *bl,
8577 int eliminate_p, int threshold, int insn_count)
8579 struct loop_ivs *ivs = LOOP_IVS (loop);
8580 rtx reg = bl->biv->dest_reg;
8583 /* Scan all insns in the loop, stopping if we find one that uses the
8584 biv in a way that we cannot eliminate. */
8586 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
8588 enum rtx_code code = GET_CODE (p);
8589 basic_block where_bb = 0;
8590 rtx where_insn = threshold >= insn_count ? 0 : p;
8593 /* If this is a libcall that sets a giv, skip ahead to its end. */
8594 if (GET_RTX_CLASS (code) == 'i')
8596 note = find_reg_note (p, REG_LIBCALL, NULL_RTX);
8600 rtx last = XEXP (note, 0);
8601 rtx set = single_set (last);
8603 if (set && GET_CODE (SET_DEST (set)) == REG)
8605 unsigned int regno = REGNO (SET_DEST (set));
8607 if (regno < ivs->n_regs
8608 && REG_IV_TYPE (ivs, regno) == GENERAL_INDUCT
8609 && REG_IV_INFO (ivs, regno)->src_reg == bl->biv->src_reg)
8615 /* Closely examine the insn if the biv is mentioned. */
8616 if ((code == INSN || code == JUMP_INSN || code == CALL_INSN)
8617 && reg_mentioned_p (reg, PATTERN (p))
8618 && ! maybe_eliminate_biv_1 (loop, PATTERN (p), p, bl,
8619 eliminate_p, where_bb, where_insn))
8621 if (loop_dump_stream)
8622 fprintf (loop_dump_stream,
8623 "Cannot eliminate biv %d: biv used in insn %d.\n",
8624 bl->regno, INSN_UID (p));
8628 /* If we are eliminating, kill REG_EQUAL notes mentioning the biv. */
8630 && (note = find_reg_note (p, REG_EQUAL, NULL_RTX)) != NULL_RTX
8631 && reg_mentioned_p (reg, XEXP (note, 0)))
8632 remove_note (p, note);
8637 if (loop_dump_stream)
8638 fprintf (loop_dump_stream, "biv %d %s eliminated.\n",
8639 bl->regno, eliminate_p ? "was" : "can be");
8646 /* INSN and REFERENCE are instructions in the same insn chain.
8647 Return nonzero if INSN is first. */
8650 loop_insn_first_p (rtx insn, rtx reference)
8654 for (p = insn, q = reference;;)
8656 /* Start with test for not first so that INSN == REFERENCE yields not
8658 if (q == insn || ! p)
8660 if (p == reference || ! q)
8663 /* Either of P or Q might be a NOTE. Notes have the same LUID as the
8664 previous insn, hence the <= comparison below does not work if
8666 if (INSN_UID (p) < max_uid_for_loop
8667 && INSN_UID (q) < max_uid_for_loop
8668 && GET_CODE (p) != NOTE)
8669 return INSN_LUID (p) <= INSN_LUID (q);
8671 if (INSN_UID (p) >= max_uid_for_loop
8672 || GET_CODE (p) == NOTE)
8674 if (INSN_UID (q) >= max_uid_for_loop)
8679 /* We are trying to eliminate BIV in INSN using GIV. Return nonzero if
8680 the offset that we have to take into account due to auto-increment /
8681 div derivation is zero. */
8683 biv_elimination_giv_has_0_offset (struct induction *biv,
8684 struct induction *giv, rtx insn)
8686 /* If the giv V had the auto-inc address optimization applied
8687 to it, and INSN occurs between the giv insn and the biv
8688 insn, then we'd have to adjust the value used here.
8689 This is rare, so we don't bother to make this possible. */
8690 if (giv->auto_inc_opt
8691 && ((loop_insn_first_p (giv->insn, insn)
8692 && loop_insn_first_p (insn, biv->insn))
8693 || (loop_insn_first_p (biv->insn, insn)
8694 && loop_insn_first_p (insn, giv->insn))))
8700 /* If BL appears in X (part of the pattern of INSN), see if we can
8701 eliminate its use. If so, return 1. If not, return 0.
8703 If BIV does not appear in X, return 1.
8705 If ELIMINATE_P is nonzero, actually do the elimination.
8706 WHERE_INSN/WHERE_BB indicate where extra insns should be added.
8707 Depending on how many items have been moved out of the loop, it
8708 will either be before INSN (when WHERE_INSN is nonzero) or at the
8709 start of the loop (when WHERE_INSN is zero). */
8712 maybe_eliminate_biv_1 (const struct loop *loop, rtx x, rtx insn,
8713 struct iv_class *bl, int eliminate_p,
8714 basic_block where_bb, rtx where_insn)
8716 enum rtx_code code = GET_CODE (x);
8717 rtx reg = bl->biv->dest_reg;
8718 enum machine_mode mode = GET_MODE (reg);
8719 struct induction *v;
8731 /* If we haven't already been able to do something with this BIV,
8732 we can't eliminate it. */
8738 /* If this sets the BIV, it is not a problem. */
8739 if (SET_DEST (x) == reg)
8742 /* If this is an insn that defines a giv, it is also ok because
8743 it will go away when the giv is reduced. */
8744 for (v = bl->giv; v; v = v->next_iv)
8745 if (v->giv_type == DEST_REG && SET_DEST (x) == v->dest_reg)
8749 if (SET_DEST (x) == cc0_rtx && SET_SRC (x) == reg)
8751 /* Can replace with any giv that was reduced and
8752 that has (MULT_VAL != 0) and (ADD_VAL == 0).
8753 Require a constant for MULT_VAL, so we know it's nonzero.
8754 ??? We disable this optimization to avoid potential
8757 for (v = bl->giv; v; v = v->next_iv)
8758 if (GET_CODE (v->mult_val) == CONST_INT && v->mult_val != const0_rtx
8759 && v->add_val == const0_rtx
8760 && ! v->ignore && ! v->maybe_dead && v->always_computable
8764 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8770 /* If the giv has the opposite direction of change,
8771 then reverse the comparison. */
8772 if (INTVAL (v->mult_val) < 0)
8773 new = gen_rtx_COMPARE (GET_MODE (v->new_reg),
8774 const0_rtx, v->new_reg);
8778 /* We can probably test that giv's reduced reg. */
8779 if (validate_change (insn, &SET_SRC (x), new, 0))
8783 /* Look for a giv with (MULT_VAL != 0) and (ADD_VAL != 0);
8784 replace test insn with a compare insn (cmp REDUCED_GIV ADD_VAL).
8785 Require a constant for MULT_VAL, so we know it's nonzero.
8786 ??? Do this only if ADD_VAL is a pointer to avoid a potential
8787 overflow problem. */
8789 for (v = bl->giv; v; v = v->next_iv)
8790 if (GET_CODE (v->mult_val) == CONST_INT
8791 && v->mult_val != const0_rtx
8792 && ! v->ignore && ! v->maybe_dead && v->always_computable
8794 && (GET_CODE (v->add_val) == SYMBOL_REF
8795 || GET_CODE (v->add_val) == LABEL_REF
8796 || GET_CODE (v->add_val) == CONST
8797 || (GET_CODE (v->add_val) == REG
8798 && REG_POINTER (v->add_val))))
8800 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8806 /* If the giv has the opposite direction of change,
8807 then reverse the comparison. */
8808 if (INTVAL (v->mult_val) < 0)
8809 new = gen_rtx_COMPARE (VOIDmode, copy_rtx (v->add_val),
8812 new = gen_rtx_COMPARE (VOIDmode, v->new_reg,
8813 copy_rtx (v->add_val));
8815 /* Replace biv with the giv's reduced register. */
8816 update_reg_last_use (v->add_val, insn);
8817 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8820 /* Insn doesn't support that constant or invariant. Copy it
8821 into a register (it will be a loop invariant.) */
8822 tem = gen_reg_rtx (GET_MODE (v->new_reg));
8824 loop_insn_emit_before (loop, 0, where_insn,
8826 copy_rtx (v->add_val)));
8828 /* Substitute the new register for its invariant value in
8829 the compare expression. */
8830 XEXP (new, (INTVAL (v->mult_val) < 0) ? 0 : 1) = tem;
8831 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8840 case GT: case GE: case GTU: case GEU:
8841 case LT: case LE: case LTU: case LEU:
8842 /* See if either argument is the biv. */
8843 if (XEXP (x, 0) == reg)
8844 arg = XEXP (x, 1), arg_operand = 1;
8845 else if (XEXP (x, 1) == reg)
8846 arg = XEXP (x, 0), arg_operand = 0;
8850 if (CONSTANT_P (arg))
8852 /* First try to replace with any giv that has constant positive
8853 mult_val and constant add_val. We might be able to support
8854 negative mult_val, but it seems complex to do it in general. */
8856 for (v = bl->giv; v; v = v->next_iv)
8857 if (GET_CODE (v->mult_val) == CONST_INT
8858 && INTVAL (v->mult_val) > 0
8859 && (GET_CODE (v->add_val) == SYMBOL_REF
8860 || GET_CODE (v->add_val) == LABEL_REF
8861 || GET_CODE (v->add_val) == CONST
8862 || (GET_CODE (v->add_val) == REG
8863 && REG_POINTER (v->add_val)))
8864 && ! v->ignore && ! v->maybe_dead && v->always_computable
8867 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8870 /* Don't eliminate if the linear combination that makes up
8871 the giv overflows when it is applied to ARG. */
8872 if (GET_CODE (arg) == CONST_INT)
8876 if (GET_CODE (v->add_val) == CONST_INT)
8877 add_val = v->add_val;
8879 add_val = const0_rtx;
8881 if (const_mult_add_overflow_p (arg, v->mult_val,
8889 /* Replace biv with the giv's reduced reg. */
8890 validate_change (insn, &XEXP (x, 1 - arg_operand), v->new_reg, 1);
8892 /* If all constants are actually constant integers and
8893 the derived constant can be directly placed in the COMPARE,
8895 if (GET_CODE (arg) == CONST_INT
8896 && GET_CODE (v->add_val) == CONST_INT)
8898 tem = expand_mult_add (arg, NULL_RTX, v->mult_val,
8899 v->add_val, mode, 1);
8903 /* Otherwise, load it into a register. */
8904 tem = gen_reg_rtx (mode);
8905 loop_iv_add_mult_emit_before (loop, arg,
8906 v->mult_val, v->add_val,
8907 tem, where_bb, where_insn);
8910 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8912 if (apply_change_group ())
8916 /* Look for giv with positive constant mult_val and nonconst add_val.
8917 Insert insns to calculate new compare value.
8918 ??? Turn this off due to possible overflow. */
8920 for (v = bl->giv; v; v = v->next_iv)
8921 if (GET_CODE (v->mult_val) == CONST_INT
8922 && INTVAL (v->mult_val) > 0
8923 && ! v->ignore && ! v->maybe_dead && v->always_computable
8929 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8935 tem = gen_reg_rtx (mode);
8937 /* Replace biv with giv's reduced register. */
8938 validate_change (insn, &XEXP (x, 1 - arg_operand),
8941 /* Compute value to compare against. */
8942 loop_iv_add_mult_emit_before (loop, arg,
8943 v->mult_val, v->add_val,
8944 tem, where_bb, where_insn);
8945 /* Use it in this insn. */
8946 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8947 if (apply_change_group ())
8951 else if (GET_CODE (arg) == REG || GET_CODE (arg) == MEM)
8953 if (loop_invariant_p (loop, arg) == 1)
8955 /* Look for giv with constant positive mult_val and nonconst
8956 add_val. Insert insns to compute new compare value.
8957 ??? Turn this off due to possible overflow. */
8959 for (v = bl->giv; v; v = v->next_iv)
8960 if (GET_CODE (v->mult_val) == CONST_INT && INTVAL (v->mult_val) > 0
8961 && ! v->ignore && ! v->maybe_dead && v->always_computable
8967 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8973 tem = gen_reg_rtx (mode);
8975 /* Replace biv with giv's reduced register. */
8976 validate_change (insn, &XEXP (x, 1 - arg_operand),
8979 /* Compute value to compare against. */
8980 loop_iv_add_mult_emit_before (loop, arg,
8981 v->mult_val, v->add_val,
8982 tem, where_bb, where_insn);
8983 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8984 if (apply_change_group ())
8989 /* This code has problems. Basically, you can't know when
8990 seeing if we will eliminate BL, whether a particular giv
8991 of ARG will be reduced. If it isn't going to be reduced,
8992 we can't eliminate BL. We can try forcing it to be reduced,
8993 but that can generate poor code.
8995 The problem is that the benefit of reducing TV, below should
8996 be increased if BL can actually be eliminated, but this means
8997 we might have to do a topological sort of the order in which
8998 we try to process biv. It doesn't seem worthwhile to do
8999 this sort of thing now. */
9002 /* Otherwise the reg compared with had better be a biv. */
9003 if (GET_CODE (arg) != REG
9004 || REG_IV_TYPE (ivs, REGNO (arg)) != BASIC_INDUCT)
9007 /* Look for a pair of givs, one for each biv,
9008 with identical coefficients. */
9009 for (v = bl->giv; v; v = v->next_iv)
9011 struct induction *tv;
9013 if (v->ignore || v->maybe_dead || v->mode != mode)
9016 for (tv = REG_IV_CLASS (ivs, REGNO (arg))->giv; tv;
9018 if (! tv->ignore && ! tv->maybe_dead
9019 && rtx_equal_p (tv->mult_val, v->mult_val)
9020 && rtx_equal_p (tv->add_val, v->add_val)
9021 && tv->mode == mode)
9023 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
9029 /* Replace biv with its giv's reduced reg. */
9030 XEXP (x, 1 - arg_operand) = v->new_reg;
9031 /* Replace other operand with the other giv's
9033 XEXP (x, arg_operand) = tv->new_reg;
9040 /* If we get here, the biv can't be eliminated. */
9044 /* If this address is a DEST_ADDR giv, it doesn't matter if the
9045 biv is used in it, since it will be replaced. */
9046 for (v = bl->giv; v; v = v->next_iv)
9047 if (v->giv_type == DEST_ADDR && v->location == &XEXP (x, 0))
9055 /* See if any subexpression fails elimination. */
9056 fmt = GET_RTX_FORMAT (code);
9057 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
9062 if (! maybe_eliminate_biv_1 (loop, XEXP (x, i), insn, bl,
9063 eliminate_p, where_bb, where_insn))
9068 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9069 if (! maybe_eliminate_biv_1 (loop, XVECEXP (x, i, j), insn, bl,
9070 eliminate_p, where_bb, where_insn))
9079 /* Return nonzero if the last use of REG
9080 is in an insn following INSN in the same basic block. */
9083 last_use_this_basic_block (rtx reg, rtx insn)
9087 n && GET_CODE (n) != CODE_LABEL && GET_CODE (n) != JUMP_INSN;
9090 if (REGNO_LAST_UID (REGNO (reg)) == INSN_UID (n))
9096 /* Called via `note_stores' to record the initial value of a biv. Here we
9097 just record the location of the set and process it later. */
9100 record_initial (rtx dest, rtx set, void *data ATTRIBUTE_UNUSED)
9102 struct loop_ivs *ivs = (struct loop_ivs *) data;
9103 struct iv_class *bl;
9105 if (GET_CODE (dest) != REG
9106 || REGNO (dest) >= ivs->n_regs
9107 || REG_IV_TYPE (ivs, REGNO (dest)) != BASIC_INDUCT)
9110 bl = REG_IV_CLASS (ivs, REGNO (dest));
9112 /* If this is the first set found, record it. */
9113 if (bl->init_insn == 0)
9115 bl->init_insn = note_insn;
9120 /* If any of the registers in X are "old" and currently have a last use earlier
9121 than INSN, update them to have a last use of INSN. Their actual last use
9122 will be the previous insn but it will not have a valid uid_luid so we can't
9123 use it. X must be a source expression only. */
9126 update_reg_last_use (rtx x, rtx insn)
9128 /* Check for the case where INSN does not have a valid luid. In this case,
9129 there is no need to modify the regno_last_uid, as this can only happen
9130 when code is inserted after the loop_end to set a pseudo's final value,
9131 and hence this insn will never be the last use of x.
9132 ???? This comment is not correct. See for example loop_givs_reduce.
9133 This may insert an insn before another new insn. */
9134 if (GET_CODE (x) == REG && REGNO (x) < max_reg_before_loop
9135 && INSN_UID (insn) < max_uid_for_loop
9136 && REGNO_LAST_LUID (REGNO (x)) < INSN_LUID (insn))
9138 REGNO_LAST_UID (REGNO (x)) = INSN_UID (insn);
9143 const char *fmt = GET_RTX_FORMAT (GET_CODE (x));
9144 for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
9147 update_reg_last_use (XEXP (x, i), insn);
9148 else if (fmt[i] == 'E')
9149 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9150 update_reg_last_use (XVECEXP (x, i, j), insn);
9155 /* Given an insn INSN and condition COND, return the condition in a
9156 canonical form to simplify testing by callers. Specifically:
9158 (1) The code will always be a comparison operation (EQ, NE, GT, etc.).
9159 (2) Both operands will be machine operands; (cc0) will have been replaced.
9160 (3) If an operand is a constant, it will be the second operand.
9161 (4) (LE x const) will be replaced with (LT x <const+1>) and similarly
9162 for GE, GEU, and LEU.
9164 If the condition cannot be understood, or is an inequality floating-point
9165 comparison which needs to be reversed, 0 will be returned.
9167 If REVERSE is nonzero, then reverse the condition prior to canonizing it.
9169 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9170 insn used in locating the condition was found. If a replacement test
9171 of the condition is desired, it should be placed in front of that
9172 insn and we will be sure that the inputs are still valid.
9174 If WANT_REG is nonzero, we wish the condition to be relative to that
9175 register, if possible. Therefore, do not canonicalize the condition
9176 further. If ALLOW_CC_MODE is nonzero, allow the condition returned
9177 to be a compare to a CC mode register. */
9180 canonicalize_condition (rtx insn, rtx cond, int reverse, rtx *earliest,
9181 rtx want_reg, int allow_cc_mode)
9188 int reverse_code = 0;
9189 enum machine_mode mode;
9191 code = GET_CODE (cond);
9192 mode = GET_MODE (cond);
9193 op0 = XEXP (cond, 0);
9194 op1 = XEXP (cond, 1);
9197 code = reversed_comparison_code (cond, insn);
9198 if (code == UNKNOWN)
9204 /* If we are comparing a register with zero, see if the register is set
9205 in the previous insn to a COMPARE or a comparison operation. Perform
9206 the same tests as a function of STORE_FLAG_VALUE as find_comparison_args
9209 while (GET_RTX_CLASS (code) == '<'
9210 && op1 == CONST0_RTX (GET_MODE (op0))
9213 /* Set nonzero when we find something of interest. */
9217 /* If comparison with cc0, import actual comparison from compare
9221 if ((prev = prev_nonnote_insn (prev)) == 0
9222 || GET_CODE (prev) != INSN
9223 || (set = single_set (prev)) == 0
9224 || SET_DEST (set) != cc0_rtx)
9227 op0 = SET_SRC (set);
9228 op1 = CONST0_RTX (GET_MODE (op0));
9234 /* If this is a COMPARE, pick up the two things being compared. */
9235 if (GET_CODE (op0) == COMPARE)
9237 op1 = XEXP (op0, 1);
9238 op0 = XEXP (op0, 0);
9241 else if (GET_CODE (op0) != REG)
9244 /* Go back to the previous insn. Stop if it is not an INSN. We also
9245 stop if it isn't a single set or if it has a REG_INC note because
9246 we don't want to bother dealing with it. */
9248 if ((prev = prev_nonnote_insn (prev)) == 0
9249 || GET_CODE (prev) != INSN
9250 || FIND_REG_INC_NOTE (prev, NULL_RTX))
9253 set = set_of (op0, prev);
9256 && (GET_CODE (set) != SET
9257 || !rtx_equal_p (SET_DEST (set), op0)))
9260 /* If this is setting OP0, get what it sets it to if it looks
9264 enum machine_mode inner_mode = GET_MODE (SET_DEST (set));
9265 #ifdef FLOAT_STORE_FLAG_VALUE
9266 REAL_VALUE_TYPE fsfv;
9269 /* ??? We may not combine comparisons done in a CCmode with
9270 comparisons not done in a CCmode. This is to aid targets
9271 like Alpha that have an IEEE compliant EQ instruction, and
9272 a non-IEEE compliant BEQ instruction. The use of CCmode is
9273 actually artificial, simply to prevent the combination, but
9274 should not affect other platforms.
9276 However, we must allow VOIDmode comparisons to match either
9277 CCmode or non-CCmode comparison, because some ports have
9278 modeless comparisons inside branch patterns.
9280 ??? This mode check should perhaps look more like the mode check
9281 in simplify_comparison in combine. */
9283 if ((GET_CODE (SET_SRC (set)) == COMPARE
9286 && GET_MODE_CLASS (inner_mode) == MODE_INT
9287 && (GET_MODE_BITSIZE (inner_mode)
9288 <= HOST_BITS_PER_WIDE_INT)
9289 && (STORE_FLAG_VALUE
9290 & ((HOST_WIDE_INT) 1
9291 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9292 #ifdef FLOAT_STORE_FLAG_VALUE
9294 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9295 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9296 REAL_VALUE_NEGATIVE (fsfv)))
9299 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'))
9300 && (((GET_MODE_CLASS (mode) == MODE_CC)
9301 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9302 || mode == VOIDmode || inner_mode == VOIDmode))
9304 else if (((code == EQ
9306 && (GET_MODE_BITSIZE (inner_mode)
9307 <= HOST_BITS_PER_WIDE_INT)
9308 && GET_MODE_CLASS (inner_mode) == MODE_INT
9309 && (STORE_FLAG_VALUE
9310 & ((HOST_WIDE_INT) 1
9311 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9312 #ifdef FLOAT_STORE_FLAG_VALUE
9314 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9315 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9316 REAL_VALUE_NEGATIVE (fsfv)))
9319 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'
9320 && (((GET_MODE_CLASS (mode) == MODE_CC)
9321 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9322 || mode == VOIDmode || inner_mode == VOIDmode))
9332 else if (reg_set_p (op0, prev))
9333 /* If this sets OP0, but not directly, we have to give up. */
9338 if (GET_RTX_CLASS (GET_CODE (x)) == '<')
9339 code = GET_CODE (x);
9342 code = reversed_comparison_code (x, prev);
9343 if (code == UNKNOWN)
9348 op0 = XEXP (x, 0), op1 = XEXP (x, 1);
9354 /* If constant is first, put it last. */
9355 if (CONSTANT_P (op0))
9356 code = swap_condition (code), tem = op0, op0 = op1, op1 = tem;
9358 /* If OP0 is the result of a comparison, we weren't able to find what
9359 was really being compared, so fail. */
9361 && GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
9364 /* Canonicalize any ordered comparison with integers involving equality
9365 if we can do computations in the relevant mode and we do not
9368 if (GET_MODE_CLASS (GET_MODE (op0)) != MODE_CC
9369 && GET_CODE (op1) == CONST_INT
9370 && GET_MODE (op0) != VOIDmode
9371 && GET_MODE_BITSIZE (GET_MODE (op0)) <= HOST_BITS_PER_WIDE_INT)
9373 HOST_WIDE_INT const_val = INTVAL (op1);
9374 unsigned HOST_WIDE_INT uconst_val = const_val;
9375 unsigned HOST_WIDE_INT max_val
9376 = (unsigned HOST_WIDE_INT) GET_MODE_MASK (GET_MODE (op0));
9381 if ((unsigned HOST_WIDE_INT) const_val != max_val >> 1)
9382 code = LT, op1 = gen_int_mode (const_val + 1, GET_MODE (op0));
9385 /* When cross-compiling, const_val might be sign-extended from
9386 BITS_PER_WORD to HOST_BITS_PER_WIDE_INT */
9388 if ((HOST_WIDE_INT) (const_val & max_val)
9389 != (((HOST_WIDE_INT) 1
9390 << (GET_MODE_BITSIZE (GET_MODE (op0)) - 1))))
9391 code = GT, op1 = gen_int_mode (const_val - 1, GET_MODE (op0));
9395 if (uconst_val < max_val)
9396 code = LTU, op1 = gen_int_mode (uconst_val + 1, GET_MODE (op0));
9400 if (uconst_val != 0)
9401 code = GTU, op1 = gen_int_mode (uconst_val - 1, GET_MODE (op0));
9409 /* Never return CC0; return zero instead. */
9413 return gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
9416 /* Given a jump insn JUMP, return the condition that will cause it to branch
9417 to its JUMP_LABEL. If the condition cannot be understood, or is an
9418 inequality floating-point comparison which needs to be reversed, 0 will
9421 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9422 insn used in locating the condition was found. If a replacement test
9423 of the condition is desired, it should be placed in front of that
9424 insn and we will be sure that the inputs are still valid.
9426 If ALLOW_CC_MODE is nonzero, allow the condition returned to be a
9427 compare CC mode register. */
9430 get_condition (rtx jump, rtx *earliest, int allow_cc_mode)
9436 /* If this is not a standard conditional jump, we can't parse it. */
9437 if (GET_CODE (jump) != JUMP_INSN
9438 || ! any_condjump_p (jump))
9440 set = pc_set (jump);
9442 cond = XEXP (SET_SRC (set), 0);
9444 /* If this branches to JUMP_LABEL when the condition is false, reverse
9447 = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
9448 && XEXP (XEXP (SET_SRC (set), 2), 0) == JUMP_LABEL (jump);
9450 return canonicalize_condition (jump, cond, reverse, earliest, NULL_RTX,
9454 /* Similar to above routine, except that we also put an invariant last
9455 unless both operands are invariants. */
9458 get_condition_for_loop (const struct loop *loop, rtx x)
9460 rtx comparison = get_condition (x, (rtx*) 0, false);
9463 || ! loop_invariant_p (loop, XEXP (comparison, 0))
9464 || loop_invariant_p (loop, XEXP (comparison, 1)))
9467 return gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)), VOIDmode,
9468 XEXP (comparison, 1), XEXP (comparison, 0));
9471 /* Scan the function and determine whether it has indirect (computed) jumps.
9473 This is taken mostly from flow.c; similar code exists elsewhere
9474 in the compiler. It may be useful to put this into rtlanal.c. */
9476 indirect_jump_in_function_p (rtx start)
9480 for (insn = start; insn; insn = NEXT_INSN (insn))
9481 if (computed_jump_p (insn))
9487 /* Add MEM to the LOOP_MEMS array, if appropriate. See the
9488 documentation for LOOP_MEMS for the definition of `appropriate'.
9489 This function is called from prescan_loop via for_each_rtx. */
9492 insert_loop_mem (rtx *mem, void *data ATTRIBUTE_UNUSED)
9494 struct loop_info *loop_info = data;
9501 switch (GET_CODE (m))
9507 /* We're not interested in MEMs that are only clobbered. */
9511 /* We're not interested in the MEM associated with a
9512 CONST_DOUBLE, so there's no need to traverse into this. */
9516 /* We're not interested in any MEMs that only appear in notes. */
9520 /* This is not a MEM. */
9524 /* See if we've already seen this MEM. */
9525 for (i = 0; i < loop_info->mems_idx; ++i)
9526 if (rtx_equal_p (m, loop_info->mems[i].mem))
9528 if (GET_MODE (m) != GET_MODE (loop_info->mems[i].mem))
9529 /* The modes of the two memory accesses are different. If
9530 this happens, something tricky is going on, and we just
9531 don't optimize accesses to this MEM. */
9532 loop_info->mems[i].optimize = 0;
9537 /* Resize the array, if necessary. */
9538 if (loop_info->mems_idx == loop_info->mems_allocated)
9540 if (loop_info->mems_allocated != 0)
9541 loop_info->mems_allocated *= 2;
9543 loop_info->mems_allocated = 32;
9545 loop_info->mems = xrealloc (loop_info->mems,
9546 loop_info->mems_allocated * sizeof (loop_mem_info));
9549 /* Actually insert the MEM. */
9550 loop_info->mems[loop_info->mems_idx].mem = m;
9551 /* We can't hoist this MEM out of the loop if it's a BLKmode MEM
9552 because we can't put it in a register. We still store it in the
9553 table, though, so that if we see the same address later, but in a
9554 non-BLK mode, we'll not think we can optimize it at that point. */
9555 loop_info->mems[loop_info->mems_idx].optimize = (GET_MODE (m) != BLKmode);
9556 loop_info->mems[loop_info->mems_idx].reg = NULL_RTX;
9557 ++loop_info->mems_idx;
9563 /* Allocate REGS->ARRAY or reallocate it if it is too small.
9565 Increment REGS->ARRAY[I].SET_IN_LOOP at the index I of each
9566 register that is modified by an insn between FROM and TO. If the
9567 value of an element of REGS->array[I].SET_IN_LOOP becomes 127 or
9568 more, stop incrementing it, to avoid overflow.
9570 Store in REGS->ARRAY[I].SINGLE_USAGE the single insn in which
9571 register I is used, if it is only used once. Otherwise, it is set
9572 to 0 (for no uses) or const0_rtx for more than one use. This
9573 parameter may be zero, in which case this processing is not done.
9575 Set REGS->ARRAY[I].MAY_NOT_OPTIMIZE nonzero if we should not
9576 optimize register I. */
9579 loop_regs_scan (const struct loop *loop, int extra_size)
9581 struct loop_regs *regs = LOOP_REGS (loop);
9583 /* last_set[n] is nonzero iff reg n has been set in the current
9584 basic block. In that case, it is the insn that last set reg n. */
9589 old_nregs = regs->num;
9590 regs->num = max_reg_num ();
9592 /* Grow the regs array if not allocated or too small. */
9593 if (regs->num >= regs->size)
9595 regs->size = regs->num + extra_size;
9597 regs->array = xrealloc (regs->array, regs->size * sizeof (*regs->array));
9599 /* Zero the new elements. */
9600 memset (regs->array + old_nregs, 0,
9601 (regs->size - old_nregs) * sizeof (*regs->array));
9604 /* Clear previously scanned fields but do not clear n_times_set. */
9605 for (i = 0; i < old_nregs; i++)
9607 regs->array[i].set_in_loop = 0;
9608 regs->array[i].may_not_optimize = 0;
9609 regs->array[i].single_usage = NULL_RTX;
9612 last_set = xcalloc (regs->num, sizeof (rtx));
9614 /* Scan the loop, recording register usage. */
9615 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9616 insn = NEXT_INSN (insn))
9620 /* Record registers that have exactly one use. */
9621 find_single_use_in_loop (regs, insn, PATTERN (insn));
9623 /* Include uses in REG_EQUAL notes. */
9624 if (REG_NOTES (insn))
9625 find_single_use_in_loop (regs, insn, REG_NOTES (insn));
9627 if (GET_CODE (PATTERN (insn)) == SET
9628 || GET_CODE (PATTERN (insn)) == CLOBBER)
9629 count_one_set (regs, insn, PATTERN (insn), last_set);
9630 else if (GET_CODE (PATTERN (insn)) == PARALLEL)
9633 for (i = XVECLEN (PATTERN (insn), 0) - 1; i >= 0; i--)
9634 count_one_set (regs, insn, XVECEXP (PATTERN (insn), 0, i),
9639 if (GET_CODE (insn) == CODE_LABEL || GET_CODE (insn) == JUMP_INSN)
9640 memset (last_set, 0, regs->num * sizeof (rtx));
9642 /* Invalidate all registers used for function argument passing.
9643 We check rtx_varies_p for the same reason as below, to allow
9644 optimizing PIC calculations. */
9645 if (GET_CODE (insn) == CALL_INSN)
9648 for (link = CALL_INSN_FUNCTION_USAGE (insn);
9650 link = XEXP (link, 1))
9654 if (GET_CODE (op = XEXP (link, 0)) == USE
9655 && GET_CODE (reg = XEXP (op, 0)) == REG
9656 && rtx_varies_p (reg, 1))
9657 regs->array[REGNO (reg)].may_not_optimize = 1;
9662 /* Invalidate all hard registers clobbered by calls. With one exception:
9663 a call-clobbered PIC register is still function-invariant for our
9664 purposes, since we can hoist any PIC calculations out of the loop.
9665 Thus the call to rtx_varies_p. */
9666 if (LOOP_INFO (loop)->has_call)
9667 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
9668 if (TEST_HARD_REG_BIT (regs_invalidated_by_call, i)
9669 && rtx_varies_p (regno_reg_rtx[i], 1))
9671 regs->array[i].may_not_optimize = 1;
9672 regs->array[i].set_in_loop = 1;
9675 #ifdef AVOID_CCMODE_COPIES
9676 /* Don't try to move insns which set CC registers if we should not
9677 create CCmode register copies. */
9678 for (i = regs->num - 1; i >= FIRST_PSEUDO_REGISTER; i--)
9679 if (GET_MODE_CLASS (GET_MODE (regno_reg_rtx[i])) == MODE_CC)
9680 regs->array[i].may_not_optimize = 1;
9683 /* Set regs->array[I].n_times_set for the new registers. */
9684 for (i = old_nregs; i < regs->num; i++)
9685 regs->array[i].n_times_set = regs->array[i].set_in_loop;
9690 /* Returns the number of real INSNs in the LOOP. */
9693 count_insns_in_loop (const struct loop *loop)
9698 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9699 insn = NEXT_INSN (insn))
9706 /* Move MEMs into registers for the duration of the loop. */
9709 load_mems (const struct loop *loop)
9711 struct loop_info *loop_info = LOOP_INFO (loop);
9712 struct loop_regs *regs = LOOP_REGS (loop);
9713 int maybe_never = 0;
9715 rtx p, prev_ebb_head;
9716 rtx label = NULL_RTX;
9718 /* Nonzero if the next instruction may never be executed. */
9719 int next_maybe_never = 0;
9720 unsigned int last_max_reg = max_reg_num ();
9722 if (loop_info->mems_idx == 0)
9725 /* We cannot use next_label here because it skips over normal insns. */
9726 end_label = next_nonnote_insn (loop->end);
9727 if (end_label && GET_CODE (end_label) != CODE_LABEL)
9728 end_label = NULL_RTX;
9730 /* Check to see if it's possible that some instructions in the loop are
9731 never executed. Also check if there is a goto out of the loop other
9732 than right after the end of the loop. */
9733 for (p = next_insn_in_loop (loop, loop->scan_start);
9735 p = next_insn_in_loop (loop, p))
9737 if (GET_CODE (p) == CODE_LABEL)
9739 else if (GET_CODE (p) == JUMP_INSN
9740 /* If we enter the loop in the middle, and scan
9741 around to the beginning, don't set maybe_never
9742 for that. This must be an unconditional jump,
9743 otherwise the code at the top of the loop might
9744 never be executed. Unconditional jumps are
9745 followed a by barrier then loop end. */
9746 && ! (GET_CODE (p) == JUMP_INSN
9747 && JUMP_LABEL (p) == loop->top
9748 && NEXT_INSN (NEXT_INSN (p)) == loop->end
9749 && any_uncondjump_p (p)))
9751 /* If this is a jump outside of the loop but not right
9752 after the end of the loop, we would have to emit new fixup
9753 sequences for each such label. */
9754 if (/* If we can't tell where control might go when this
9755 JUMP_INSN is executed, we must be conservative. */
9757 || (JUMP_LABEL (p) != end_label
9758 && (INSN_UID (JUMP_LABEL (p)) >= max_uid_for_loop
9759 || INSN_LUID (JUMP_LABEL (p)) < INSN_LUID (loop->start)
9760 || INSN_LUID (JUMP_LABEL (p)) > INSN_LUID (loop->end))))
9763 if (!any_condjump_p (p))
9764 /* Something complicated. */
9767 /* If there are any more instructions in the loop, they
9768 might not be reached. */
9769 next_maybe_never = 1;
9771 else if (next_maybe_never)
9775 /* Find start of the extended basic block that enters the loop. */
9776 for (p = loop->start;
9777 PREV_INSN (p) && GET_CODE (p) != CODE_LABEL;
9784 /* Build table of mems that get set to constant values before the
9786 for (; p != loop->start; p = NEXT_INSN (p))
9787 cselib_process_insn (p);
9789 /* Actually move the MEMs. */
9790 for (i = 0; i < loop_info->mems_idx; ++i)
9792 regset_head load_copies;
9793 regset_head store_copies;
9796 rtx mem = loop_info->mems[i].mem;
9799 if (MEM_VOLATILE_P (mem)
9800 || loop_invariant_p (loop, XEXP (mem, 0)) != 1)
9801 /* There's no telling whether or not MEM is modified. */
9802 loop_info->mems[i].optimize = 0;
9804 /* Go through the MEMs written to in the loop to see if this
9805 one is aliased by one of them. */
9806 mem_list_entry = loop_info->store_mems;
9807 while (mem_list_entry)
9809 if (rtx_equal_p (mem, XEXP (mem_list_entry, 0)))
9811 else if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
9814 /* MEM is indeed aliased by this store. */
9815 loop_info->mems[i].optimize = 0;
9818 mem_list_entry = XEXP (mem_list_entry, 1);
9821 if (flag_float_store && written
9822 && GET_MODE_CLASS (GET_MODE (mem)) == MODE_FLOAT)
9823 loop_info->mems[i].optimize = 0;
9825 /* If this MEM is written to, we must be sure that there
9826 are no reads from another MEM that aliases this one. */
9827 if (loop_info->mems[i].optimize && written)
9831 for (j = 0; j < loop_info->mems_idx; ++j)
9835 else if (true_dependence (mem,
9837 loop_info->mems[j].mem,
9840 /* It's not safe to hoist loop_info->mems[i] out of
9841 the loop because writes to it might not be
9842 seen by reads from loop_info->mems[j]. */
9843 loop_info->mems[i].optimize = 0;
9849 if (maybe_never && may_trap_p (mem))
9850 /* We can't access the MEM outside the loop; it might
9851 cause a trap that wouldn't have happened otherwise. */
9852 loop_info->mems[i].optimize = 0;
9854 if (!loop_info->mems[i].optimize)
9855 /* We thought we were going to lift this MEM out of the
9856 loop, but later discovered that we could not. */
9859 INIT_REG_SET (&load_copies);
9860 INIT_REG_SET (&store_copies);
9862 /* Allocate a pseudo for this MEM. We set REG_USERVAR_P in
9863 order to keep scan_loop from moving stores to this MEM
9864 out of the loop just because this REG is neither a
9865 user-variable nor used in the loop test. */
9866 reg = gen_reg_rtx (GET_MODE (mem));
9867 REG_USERVAR_P (reg) = 1;
9868 loop_info->mems[i].reg = reg;
9870 /* Now, replace all references to the MEM with the
9871 corresponding pseudos. */
9873 for (p = next_insn_in_loop (loop, loop->scan_start);
9875 p = next_insn_in_loop (loop, p))
9881 set = single_set (p);
9883 /* See if this copies the mem into a register that isn't
9884 modified afterwards. We'll try to do copy propagation
9885 a little further on. */
9887 /* @@@ This test is _way_ too conservative. */
9889 && GET_CODE (SET_DEST (set)) == REG
9890 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER
9891 && REGNO (SET_DEST (set)) < last_max_reg
9892 && regs->array[REGNO (SET_DEST (set))].n_times_set == 1
9893 && rtx_equal_p (SET_SRC (set), mem))
9894 SET_REGNO_REG_SET (&load_copies, REGNO (SET_DEST (set)));
9896 /* See if this copies the mem from a register that isn't
9897 modified afterwards. We'll try to remove the
9898 redundant copy later on by doing a little register
9899 renaming and copy propagation. This will help
9900 to untangle things for the BIV detection code. */
9903 && GET_CODE (SET_SRC (set)) == REG
9904 && REGNO (SET_SRC (set)) >= FIRST_PSEUDO_REGISTER
9905 && REGNO (SET_SRC (set)) < last_max_reg
9906 && regs->array[REGNO (SET_SRC (set))].n_times_set == 1
9907 && rtx_equal_p (SET_DEST (set), mem))
9908 SET_REGNO_REG_SET (&store_copies, REGNO (SET_SRC (set)));
9910 /* If this is a call which uses / clobbers this memory
9911 location, we must not change the interface here. */
9912 if (GET_CODE (p) == CALL_INSN
9913 && reg_mentioned_p (loop_info->mems[i].mem,
9914 CALL_INSN_FUNCTION_USAGE (p)))
9917 loop_info->mems[i].optimize = 0;
9921 /* Replace the memory reference with the shadow register. */
9922 replace_loop_mems (p, loop_info->mems[i].mem,
9923 loop_info->mems[i].reg, written);
9926 if (GET_CODE (p) == CODE_LABEL
9927 || GET_CODE (p) == JUMP_INSN)
9931 if (! loop_info->mems[i].optimize)
9932 ; /* We found we couldn't do the replacement, so do nothing. */
9933 else if (! apply_change_group ())
9934 /* We couldn't replace all occurrences of the MEM. */
9935 loop_info->mems[i].optimize = 0;
9938 /* Load the memory immediately before LOOP->START, which is
9939 the NOTE_LOOP_BEG. */
9940 cselib_val *e = cselib_lookup (mem, VOIDmode, 0);
9944 struct elt_loc_list *const_equiv = 0;
9948 struct elt_loc_list *equiv;
9949 struct elt_loc_list *best_equiv = 0;
9950 for (equiv = e->locs; equiv; equiv = equiv->next)
9952 if (CONSTANT_P (equiv->loc))
9953 const_equiv = equiv;
9954 else if (GET_CODE (equiv->loc) == REG
9955 /* Extending hard register lifetimes causes crash
9956 on SRC targets. Doing so on non-SRC is
9957 probably also not good idea, since we most
9958 probably have pseudoregister equivalence as
9960 && REGNO (equiv->loc) >= FIRST_PSEUDO_REGISTER)
9963 /* Use the constant equivalence if that is cheap enough. */
9965 best_equiv = const_equiv;
9966 else if (const_equiv
9967 && (rtx_cost (const_equiv->loc, SET)
9968 <= rtx_cost (best_equiv->loc, SET)))
9970 best_equiv = const_equiv;
9974 /* If best_equiv is nonzero, we know that MEM is set to a
9975 constant or register before the loop. We will use this
9976 knowledge to initialize the shadow register with that
9977 constant or reg rather than by loading from MEM. */
9979 best = copy_rtx (best_equiv->loc);
9982 set = gen_move_insn (reg, best);
9983 set = loop_insn_hoist (loop, set);
9986 for (p = prev_ebb_head; p != loop->start; p = NEXT_INSN (p))
9987 if (REGNO_LAST_UID (REGNO (best)) == INSN_UID (p))
9989 REGNO_LAST_UID (REGNO (best)) = INSN_UID (set);
9995 set_unique_reg_note (set, REG_EQUAL, copy_rtx (const_equiv->loc));
9999 if (label == NULL_RTX)
10001 label = gen_label_rtx ();
10002 emit_label_after (label, loop->end);
10005 /* Store the memory immediately after END, which is
10006 the NOTE_LOOP_END. */
10007 set = gen_move_insn (copy_rtx (mem), reg);
10008 loop_insn_emit_after (loop, 0, label, set);
10011 if (loop_dump_stream)
10013 fprintf (loop_dump_stream, "Hoisted regno %d %s from ",
10014 REGNO (reg), (written ? "r/w" : "r/o"));
10015 print_rtl (loop_dump_stream, mem);
10016 fputc ('\n', loop_dump_stream);
10019 /* Attempt a bit of copy propagation. This helps untangle the
10020 data flow, and enables {basic,general}_induction_var to find
10022 EXECUTE_IF_SET_IN_REG_SET
10023 (&load_copies, FIRST_PSEUDO_REGISTER, j,
10025 try_copy_prop (loop, reg, j);
10027 CLEAR_REG_SET (&load_copies);
10029 EXECUTE_IF_SET_IN_REG_SET
10030 (&store_copies, FIRST_PSEUDO_REGISTER, j,
10032 try_swap_copy_prop (loop, reg, j);
10034 CLEAR_REG_SET (&store_copies);
10038 /* Now, we need to replace all references to the previous exit
10039 label with the new one. */
10040 if (label != NULL_RTX && end_label != NULL_RTX)
10041 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
10042 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == end_label)
10043 redirect_jump (p, label, false);
10048 /* For communication between note_reg_stored and its caller. */
10049 struct note_reg_stored_arg
10055 /* Called via note_stores, record in SET_SEEN whether X, which is written,
10056 is equal to ARG. */
10058 note_reg_stored (rtx x, rtx setter ATTRIBUTE_UNUSED, void *arg)
10060 struct note_reg_stored_arg *t = (struct note_reg_stored_arg *) arg;
10065 /* Try to replace every occurrence of pseudo REGNO with REPLACEMENT.
10066 There must be exactly one insn that sets this pseudo; it will be
10067 deleted if all replacements succeed and we can prove that the register
10068 is not used after the loop. */
10071 try_copy_prop (const struct loop *loop, rtx replacement, unsigned int regno)
10073 /* This is the reg that we are copying from. */
10074 rtx reg_rtx = regno_reg_rtx[regno];
10077 /* These help keep track of whether we replaced all uses of the reg. */
10078 int replaced_last = 0;
10079 int store_is_first = 0;
10081 for (insn = next_insn_in_loop (loop, loop->scan_start);
10083 insn = next_insn_in_loop (loop, insn))
10087 /* Only substitute within one extended basic block from the initializing
10089 if (GET_CODE (insn) == CODE_LABEL && init_insn)
10092 if (! INSN_P (insn))
10095 /* Is this the initializing insn? */
10096 set = single_set (insn);
10098 && GET_CODE (SET_DEST (set)) == REG
10099 && REGNO (SET_DEST (set)) == regno)
10105 if (REGNO_FIRST_UID (regno) == INSN_UID (insn))
10106 store_is_first = 1;
10109 /* Only substitute after seeing the initializing insn. */
10110 if (init_insn && insn != init_insn)
10112 struct note_reg_stored_arg arg;
10114 replace_loop_regs (insn, reg_rtx, replacement);
10115 if (REGNO_LAST_UID (regno) == INSN_UID (insn))
10118 /* Stop replacing when REPLACEMENT is modified. */
10119 arg.reg = replacement;
10121 note_stores (PATTERN (insn), note_reg_stored, &arg);
10124 rtx note = find_reg_note (insn, REG_EQUAL, NULL);
10126 /* It is possible that we've turned previously valid REG_EQUAL to
10127 invalid, as we change the REGNO to REPLACEMENT and unlike REGNO,
10128 REPLACEMENT is modified, we get different meaning. */
10129 if (note && reg_mentioned_p (replacement, XEXP (note, 0)))
10130 remove_note (insn, note);
10137 if (apply_change_group ())
10139 if (loop_dump_stream)
10140 fprintf (loop_dump_stream, " Replaced reg %d", regno);
10141 if (store_is_first && replaced_last)
10146 /* Assume we're just deleting INIT_INSN. */
10148 /* Look for REG_RETVAL note. If we're deleting the end of
10149 the libcall sequence, the whole sequence can go. */
10150 retval_note = find_reg_note (init_insn, REG_RETVAL, NULL_RTX);
10151 /* If we found a REG_RETVAL note, find the first instruction
10152 in the sequence. */
10154 first = XEXP (retval_note, 0);
10156 /* Delete the instructions. */
10157 loop_delete_insns (first, init_insn);
10159 if (loop_dump_stream)
10160 fprintf (loop_dump_stream, ".\n");
10164 /* Replace all the instructions from FIRST up to and including LAST
10165 with NOTE_INSN_DELETED notes. */
10168 loop_delete_insns (rtx first, rtx last)
10172 if (loop_dump_stream)
10173 fprintf (loop_dump_stream, ", deleting init_insn (%d)",
10175 delete_insn (first);
10177 /* If this was the LAST instructions we're supposed to delete,
10182 first = NEXT_INSN (first);
10186 /* Try to replace occurrences of pseudo REGNO with REPLACEMENT within
10187 loop LOOP if the order of the sets of these registers can be
10188 swapped. There must be exactly one insn within the loop that sets
10189 this pseudo followed immediately by a move insn that sets
10190 REPLACEMENT with REGNO. */
10192 try_swap_copy_prop (const struct loop *loop, rtx replacement,
10193 unsigned int regno)
10196 rtx set = NULL_RTX;
10197 unsigned int new_regno;
10199 new_regno = REGNO (replacement);
10201 for (insn = next_insn_in_loop (loop, loop->scan_start);
10203 insn = next_insn_in_loop (loop, insn))
10205 /* Search for the insn that copies REGNO to NEW_REGNO? */
10207 && (set = single_set (insn))
10208 && GET_CODE (SET_DEST (set)) == REG
10209 && REGNO (SET_DEST (set)) == new_regno
10210 && GET_CODE (SET_SRC (set)) == REG
10211 && REGNO (SET_SRC (set)) == regno)
10215 if (insn != NULL_RTX)
10220 /* Some DEF-USE info would come in handy here to make this
10221 function more general. For now, just check the previous insn
10222 which is the most likely candidate for setting REGNO. */
10224 prev_insn = PREV_INSN (insn);
10227 && (prev_set = single_set (prev_insn))
10228 && GET_CODE (SET_DEST (prev_set)) == REG
10229 && REGNO (SET_DEST (prev_set)) == regno)
10232 (set (reg regno) (expr))
10233 (set (reg new_regno) (reg regno))
10235 so try converting this to:
10236 (set (reg new_regno) (expr))
10237 (set (reg regno) (reg new_regno))
10239 The former construct is often generated when a global
10240 variable used for an induction variable is shadowed by a
10241 register (NEW_REGNO). The latter construct improves the
10242 chances of GIV replacement and BIV elimination. */
10244 validate_change (prev_insn, &SET_DEST (prev_set),
10246 validate_change (insn, &SET_DEST (set),
10248 validate_change (insn, &SET_SRC (set),
10251 if (apply_change_group ())
10253 if (loop_dump_stream)
10254 fprintf (loop_dump_stream,
10255 " Swapped set of reg %d at %d with reg %d at %d.\n",
10256 regno, INSN_UID (insn),
10257 new_regno, INSN_UID (prev_insn));
10259 /* Update first use of REGNO. */
10260 if (REGNO_FIRST_UID (regno) == INSN_UID (prev_insn))
10261 REGNO_FIRST_UID (regno) = INSN_UID (insn);
10263 /* Now perform copy propagation to hopefully
10264 remove all uses of REGNO within the loop. */
10265 try_copy_prop (loop, replacement, regno);
10271 /* Worker function for find_mem_in_note, called via for_each_rtx. */
10274 find_mem_in_note_1 (rtx *x, void *data)
10276 if (*x != NULL_RTX && GET_CODE (*x) == MEM)
10278 rtx *res = (rtx *) data;
10285 /* Returns the first MEM found in NOTE by depth-first search. */
10288 find_mem_in_note (rtx note)
10290 if (note && for_each_rtx (¬e, find_mem_in_note_1, ¬e))
10295 /* Replace MEM with its associated pseudo register. This function is
10296 called from load_mems via for_each_rtx. DATA is actually a pointer
10297 to a structure describing the instruction currently being scanned
10298 and the MEM we are currently replacing. */
10301 replace_loop_mem (rtx *mem, void *data)
10303 loop_replace_args *args = (loop_replace_args *) data;
10309 switch (GET_CODE (m))
10315 /* We're not interested in the MEM associated with a
10316 CONST_DOUBLE, so there's no need to traverse into one. */
10320 /* This is not a MEM. */
10324 if (!rtx_equal_p (args->match, m))
10325 /* This is not the MEM we are currently replacing. */
10328 /* Actually replace the MEM. */
10329 validate_change (args->insn, mem, args->replacement, 1);
10335 replace_loop_mems (rtx insn, rtx mem, rtx reg, int written)
10337 loop_replace_args args;
10341 args.replacement = reg;
10343 for_each_rtx (&insn, replace_loop_mem, &args);
10345 /* If we hoist a mem write out of the loop, then REG_EQUAL
10346 notes referring to the mem are no longer valid. */
10352 for (link = ®_NOTES (insn); (note = *link); link = &XEXP (note, 1))
10354 if (REG_NOTE_KIND (note) == REG_EQUAL
10355 && (sub = find_mem_in_note (note))
10356 && true_dependence (mem, VOIDmode, sub, rtx_varies_p))
10358 /* Remove the note. */
10359 validate_change (NULL_RTX, link, XEXP (note, 1), 1);
10366 /* Replace one register with another. Called through for_each_rtx; PX points
10367 to the rtx being scanned. DATA is actually a pointer to
10368 a structure of arguments. */
10371 replace_loop_reg (rtx *px, void *data)
10374 loop_replace_args *args = (loop_replace_args *) data;
10379 if (x == args->match)
10380 validate_change (args->insn, px, args->replacement, 1);
10386 replace_loop_regs (rtx insn, rtx reg, rtx replacement)
10388 loop_replace_args args;
10392 args.replacement = replacement;
10394 for_each_rtx (&insn, replace_loop_reg, &args);
10397 /* Emit insn for PATTERN after WHERE_INSN in basic block WHERE_BB
10398 (ignored in the interim). */
10401 loop_insn_emit_after (const struct loop *loop ATTRIBUTE_UNUSED,
10402 basic_block where_bb ATTRIBUTE_UNUSED, rtx where_insn,
10405 return emit_insn_after (pattern, where_insn);
10409 /* If WHERE_INSN is nonzero emit insn for PATTERN before WHERE_INSN
10410 in basic block WHERE_BB (ignored in the interim) within the loop
10411 otherwise hoist PATTERN into the loop pre-header. */
10414 loop_insn_emit_before (const struct loop *loop,
10415 basic_block where_bb ATTRIBUTE_UNUSED,
10416 rtx where_insn, rtx pattern)
10419 return loop_insn_hoist (loop, pattern);
10420 return emit_insn_before (pattern, where_insn);
10424 /* Emit call insn for PATTERN before WHERE_INSN in basic block
10425 WHERE_BB (ignored in the interim) within the loop. */
10428 loop_call_insn_emit_before (const struct loop *loop ATTRIBUTE_UNUSED,
10429 basic_block where_bb ATTRIBUTE_UNUSED,
10430 rtx where_insn, rtx pattern)
10432 return emit_call_insn_before (pattern, where_insn);
10436 /* Hoist insn for PATTERN into the loop pre-header. */
10439 loop_insn_hoist (const struct loop *loop, rtx pattern)
10441 return loop_insn_emit_before (loop, 0, loop->start, pattern);
10445 /* Hoist call insn for PATTERN into the loop pre-header. */
10448 loop_call_insn_hoist (const struct loop *loop, rtx pattern)
10450 return loop_call_insn_emit_before (loop, 0, loop->start, pattern);
10454 /* Sink insn for PATTERN after the loop end. */
10457 loop_insn_sink (const struct loop *loop, rtx pattern)
10459 return loop_insn_emit_before (loop, 0, loop->sink, pattern);
10462 /* bl->final_value can be either general_operand or PLUS of general_operand
10463 and constant. Emit sequence of instructions to load it into REG. */
10465 gen_load_of_final_value (rtx reg, rtx final_value)
10469 final_value = force_operand (final_value, reg);
10470 if (final_value != reg)
10471 emit_move_insn (reg, final_value);
10472 seq = get_insns ();
10477 /* If the loop has multiple exits, emit insn for PATTERN before the
10478 loop to ensure that it will always be executed no matter how the
10479 loop exits. Otherwise, emit the insn for PATTERN after the loop,
10480 since this is slightly more efficient. */
10483 loop_insn_sink_or_swim (const struct loop *loop, rtx pattern)
10485 if (loop->exit_count)
10486 return loop_insn_hoist (loop, pattern);
10488 return loop_insn_sink (loop, pattern);
10492 loop_ivs_dump (const struct loop *loop, FILE *file, int verbose)
10494 struct iv_class *bl;
10497 if (! loop || ! file)
10500 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10503 fprintf (file, "Loop %d: %d IV classes\n", loop->num, iv_num);
10505 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10507 loop_iv_class_dump (bl, file, verbose);
10508 fputc ('\n', file);
10514 loop_iv_class_dump (const struct iv_class *bl, FILE *file,
10515 int verbose ATTRIBUTE_UNUSED)
10517 struct induction *v;
10521 if (! bl || ! file)
10524 fprintf (file, "IV class for reg %d, benefit %d\n",
10525 bl->regno, bl->total_benefit);
10527 fprintf (file, " Init insn %d", INSN_UID (bl->init_insn));
10528 if (bl->initial_value)
10530 fprintf (file, ", init val: ");
10531 print_simple_rtl (file, bl->initial_value);
10533 if (bl->initial_test)
10535 fprintf (file, ", init test: ");
10536 print_simple_rtl (file, bl->initial_test);
10538 fputc ('\n', file);
10540 if (bl->final_value)
10542 fprintf (file, " Final val: ");
10543 print_simple_rtl (file, bl->final_value);
10544 fputc ('\n', file);
10547 if ((incr = biv_total_increment (bl)))
10549 fprintf (file, " Total increment: ");
10550 print_simple_rtl (file, incr);
10551 fputc ('\n', file);
10554 /* List the increments. */
10555 for (i = 0, v = bl->biv; v; v = v->next_iv, i++)
10557 fprintf (file, " Inc%d: insn %d, incr: ", i, INSN_UID (v->insn));
10558 print_simple_rtl (file, v->add_val);
10559 fputc ('\n', file);
10562 /* List the givs. */
10563 for (i = 0, v = bl->giv; v; v = v->next_iv, i++)
10565 fprintf (file, " Giv%d: insn %d, benefit %d, ",
10566 i, INSN_UID (v->insn), v->benefit);
10567 if (v->giv_type == DEST_ADDR)
10568 print_simple_rtl (file, v->mem);
10570 print_simple_rtl (file, single_set (v->insn));
10571 fputc ('\n', file);
10577 loop_biv_dump (const struct induction *v, FILE *file, int verbose)
10584 REGNO (v->dest_reg), INSN_UID (v->insn));
10585 fprintf (file, " const ");
10586 print_simple_rtl (file, v->add_val);
10588 if (verbose && v->final_value)
10590 fputc ('\n', file);
10591 fprintf (file, " final ");
10592 print_simple_rtl (file, v->final_value);
10595 fputc ('\n', file);
10600 loop_giv_dump (const struct induction *v, FILE *file, int verbose)
10605 if (v->giv_type == DEST_REG)
10606 fprintf (file, "Giv %d: insn %d",
10607 REGNO (v->dest_reg), INSN_UID (v->insn));
10609 fprintf (file, "Dest address: insn %d",
10610 INSN_UID (v->insn));
10612 fprintf (file, " src reg %d benefit %d",
10613 REGNO (v->src_reg), v->benefit);
10614 fprintf (file, " lifetime %d",
10617 if (v->replaceable)
10618 fprintf (file, " replaceable");
10620 if (v->no_const_addval)
10621 fprintf (file, " ncav");
10623 if (v->ext_dependent)
10625 switch (GET_CODE (v->ext_dependent))
10628 fprintf (file, " ext se");
10631 fprintf (file, " ext ze");
10634 fprintf (file, " ext tr");
10641 fputc ('\n', file);
10642 fprintf (file, " mult ");
10643 print_simple_rtl (file, v->mult_val);
10645 fputc ('\n', file);
10646 fprintf (file, " add ");
10647 print_simple_rtl (file, v->add_val);
10649 if (verbose && v->final_value)
10651 fputc ('\n', file);
10652 fprintf (file, " final ");
10653 print_simple_rtl (file, v->final_value);
10656 fputc ('\n', file);
10661 debug_ivs (const struct loop *loop)
10663 loop_ivs_dump (loop, stderr, 1);
10668 debug_iv_class (const struct iv_class *bl)
10670 loop_iv_class_dump (bl, stderr, 1);
10675 debug_biv (const struct induction *v)
10677 loop_biv_dump (v, stderr, 1);
10682 debug_giv (const struct induction *v)
10684 loop_giv_dump (v, stderr, 1);
10688 #define LOOP_BLOCK_NUM_1(INSN) \
10689 ((INSN) ? (BLOCK_FOR_INSN (INSN) ? BLOCK_NUM (INSN) : - 1) : -1)
10691 /* The notes do not have an assigned block, so look at the next insn. */
10692 #define LOOP_BLOCK_NUM(INSN) \
10693 ((INSN) ? (GET_CODE (INSN) == NOTE \
10694 ? LOOP_BLOCK_NUM_1 (next_nonnote_insn (INSN)) \
10695 : LOOP_BLOCK_NUM_1 (INSN)) \
10698 #define LOOP_INSN_UID(INSN) ((INSN) ? INSN_UID (INSN) : -1)
10701 loop_dump_aux (const struct loop *loop, FILE *file,
10702 int verbose ATTRIBUTE_UNUSED)
10706 if (! loop || ! file)
10709 /* Print diagnostics to compare our concept of a loop with
10710 what the loop notes say. */
10711 if (! PREV_INSN (BB_HEAD (loop->first))
10712 || GET_CODE (PREV_INSN (BB_HEAD (loop->first))) != NOTE
10713 || NOTE_LINE_NUMBER (PREV_INSN (BB_HEAD (loop->first)))
10714 != NOTE_INSN_LOOP_BEG)
10715 fprintf (file, ";; No NOTE_INSN_LOOP_BEG at %d\n",
10716 INSN_UID (PREV_INSN (BB_HEAD (loop->first))));
10717 if (! NEXT_INSN (BB_END (loop->last))
10718 || GET_CODE (NEXT_INSN (BB_END (loop->last))) != NOTE
10719 || NOTE_LINE_NUMBER (NEXT_INSN (BB_END (loop->last)))
10720 != NOTE_INSN_LOOP_END)
10721 fprintf (file, ";; No NOTE_INSN_LOOP_END at %d\n",
10722 INSN_UID (NEXT_INSN (BB_END (loop->last))));
10727 ";; start %d (%d), cont dom %d (%d), cont %d (%d), vtop %d (%d), end %d (%d)\n",
10728 LOOP_BLOCK_NUM (loop->start),
10729 LOOP_INSN_UID (loop->start),
10730 LOOP_BLOCK_NUM (loop->cont),
10731 LOOP_INSN_UID (loop->cont),
10732 LOOP_BLOCK_NUM (loop->cont),
10733 LOOP_INSN_UID (loop->cont),
10734 LOOP_BLOCK_NUM (loop->vtop),
10735 LOOP_INSN_UID (loop->vtop),
10736 LOOP_BLOCK_NUM (loop->end),
10737 LOOP_INSN_UID (loop->end));
10738 fprintf (file, ";; top %d (%d), scan start %d (%d)\n",
10739 LOOP_BLOCK_NUM (loop->top),
10740 LOOP_INSN_UID (loop->top),
10741 LOOP_BLOCK_NUM (loop->scan_start),
10742 LOOP_INSN_UID (loop->scan_start));
10743 fprintf (file, ";; exit_count %d", loop->exit_count);
10744 if (loop->exit_count)
10746 fputs (", labels:", file);
10747 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
10749 fprintf (file, " %d ",
10750 LOOP_INSN_UID (XEXP (label, 0)));
10753 fputs ("\n", file);
10755 /* This can happen when a marked loop appears as two nested loops,
10756 say from while (a || b) {}. The inner loop won't match
10757 the loop markers but the outer one will. */
10758 if (LOOP_BLOCK_NUM (loop->cont) != loop->latch->index)
10759 fprintf (file, ";; NOTE_INSN_LOOP_CONT not in loop latch\n");
10763 /* Call this function from the debugger to dump LOOP. */
10766 debug_loop (const struct loop *loop)
10768 flow_loop_dump (loop, stderr, loop_dump_aux, 1);
10771 /* Call this function from the debugger to dump LOOPS. */
10774 debug_loops (const struct loops *loops)
10776 flow_loops_dump (loops, stderr, loop_dump_aux, 1);
OSDN Git Service
