1 /* Perform various loop optimizations, including strength reduction.
2 Copyright (C) 1987, 1988, 1989, 1991, 1992, 1993, 1994, 1995, 1996, 1997,
3 1998, 1999, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
22 /* This is the loop optimization pass of the compiler.
23 It finds invariant computations within loops and moves them
24 to the beginning of the loop. Then it identifies basic and
25 general induction variables.
27 Basic induction variables (BIVs) are a pseudo registers which are set within
28 a loop only by incrementing or decrementing its value. General induction
29 variables (GIVs) are pseudo registers with a value which is a linear function
30 of a basic induction variable. BIVs are recognized by `basic_induction_var';
31 GIVs by `general_induction_var'.
33 Once induction variables are identified, strength reduction is applied to the
34 general induction variables, and induction variable elimination is applied to
35 the basic induction variables.
37 It also finds cases where
38 a register is set within the loop by zero-extending a narrower value
39 and changes these to zero the entire register once before the loop
40 and merely copy the low part within the loop.
42 Most of the complexity is in heuristics to decide when it is worth
43 while to do these things. */
47 #include "coretypes.h"
53 #include "hard-reg-set.h"
54 #include "basic-block.h"
55 #include "insn-config.h"
65 #include "insn-flags.h"
69 /* Not really meaningful values, but at least something. */
70 #ifndef SIMULTANEOUS_PREFETCHES
71 #define SIMULTANEOUS_PREFETCHES 3
73 #ifndef PREFETCH_BLOCK
74 #define PREFETCH_BLOCK 32
77 #define HAVE_prefetch 0
78 #define CODE_FOR_prefetch 0
79 #define gen_prefetch(a,b,c) (abort(), NULL_RTX)
82 /* Give up the prefetch optimizations once we exceed a given threshold.
83 It is unlikely that we would be able to optimize something in a loop
84 with so many detected prefetches. */
85 #define MAX_PREFETCHES 100
86 /* The number of prefetch blocks that are beneficial to fetch at once before
87 a loop with a known (and low) iteration count. */
88 #define PREFETCH_BLOCKS_BEFORE_LOOP_MAX 6
89 /* For very tiny loops it is not worthwhile to prefetch even before the loop,
90 since it is likely that the data are already in the cache. */
91 #define PREFETCH_BLOCKS_BEFORE_LOOP_MIN 2
93 /* Parameterize some prefetch heuristics so they can be turned on and off
94 easily for performance testing on new architectures. These can be
95 defined in target-dependent files. */
97 /* Prefetch is worthwhile only when loads/stores are dense. */
98 #ifndef PREFETCH_ONLY_DENSE_MEM
99 #define PREFETCH_ONLY_DENSE_MEM 1
102 /* Define what we mean by "dense" loads and stores; This value divided by 256
103 is the minimum percentage of memory references that worth prefetching. */
104 #ifndef PREFETCH_DENSE_MEM
105 #define PREFETCH_DENSE_MEM 220
108 /* Do not prefetch for a loop whose iteration count is known to be low. */
109 #ifndef PREFETCH_NO_LOW_LOOPCNT
110 #define PREFETCH_NO_LOW_LOOPCNT 1
113 /* Define what we mean by a "low" iteration count. */
114 #ifndef PREFETCH_LOW_LOOPCNT
115 #define PREFETCH_LOW_LOOPCNT 32
118 /* Do not prefetch for a loop that contains a function call; such a loop is
119 probably not an internal loop. */
120 #ifndef PREFETCH_NO_CALL
121 #define PREFETCH_NO_CALL 1
124 /* Do not prefetch accesses with an extreme stride. */
125 #ifndef PREFETCH_NO_EXTREME_STRIDE
126 #define PREFETCH_NO_EXTREME_STRIDE 1
129 /* Define what we mean by an "extreme" stride. */
130 #ifndef PREFETCH_EXTREME_STRIDE
131 #define PREFETCH_EXTREME_STRIDE 4096
134 /* Define a limit to how far apart indices can be and still be merged
135 into a single prefetch. */
136 #ifndef PREFETCH_EXTREME_DIFFERENCE
137 #define PREFETCH_EXTREME_DIFFERENCE 4096
140 /* Issue prefetch instructions before the loop to fetch data to be used
141 in the first few loop iterations. */
142 #ifndef PREFETCH_BEFORE_LOOP
143 #define PREFETCH_BEFORE_LOOP 1
146 /* Do not handle reversed order prefetches (negative stride). */
147 #ifndef PREFETCH_NO_REVERSE_ORDER
148 #define PREFETCH_NO_REVERSE_ORDER 1
151 /* Prefetch even if the GIV is in conditional code. */
152 #ifndef PREFETCH_CONDITIONAL
153 #define PREFETCH_CONDITIONAL 1
156 #define LOOP_REG_LIFETIME(LOOP, REGNO) \
157 ((REGNO_LAST_LUID (REGNO) - REGNO_FIRST_LUID (REGNO)))
159 #define LOOP_REG_GLOBAL_P(LOOP, REGNO) \
160 ((REGNO_LAST_LUID (REGNO) > INSN_LUID ((LOOP)->end) \
161 || REGNO_FIRST_LUID (REGNO) < INSN_LUID ((LOOP)->start)))
163 #define LOOP_REGNO_NREGS(REGNO, SET_DEST) \
164 ((REGNO) < FIRST_PSEUDO_REGISTER \
165 ? (int) HARD_REGNO_NREGS ((REGNO), GET_MODE (SET_DEST)) : 1)
168 /* Vector mapping INSN_UIDs to luids.
169 The luids are like uids but increase monotonically always.
170 We use them to see whether a jump comes from outside a given loop. */
174 /* Indexed by INSN_UID, contains the ordinal giving the (innermost) loop
175 number the insn is contained in. */
177 struct loop **uid_loop;
179 /* 1 + largest uid of any insn. */
181 int max_uid_for_loop;
183 /* Number of loops detected in current function. Used as index to the
186 static int max_loop_num;
188 /* Bound on pseudo register number before loop optimization.
189 A pseudo has valid regscan info if its number is < max_reg_before_loop. */
190 unsigned int max_reg_before_loop;
192 /* The value to pass to the next call of reg_scan_update. */
193 static int loop_max_reg;
195 /* During the analysis of a loop, a chain of `struct movable's
196 is made to record all the movable insns found.
197 Then the entire chain can be scanned to decide which to move. */
201 rtx insn; /* A movable insn */
202 rtx set_src; /* The expression this reg is set from. */
203 rtx set_dest; /* The destination of this SET. */
204 rtx dependencies; /* When INSN is libcall, this is an EXPR_LIST
205 of any registers used within the LIBCALL. */
206 int consec; /* Number of consecutive following insns
207 that must be moved with this one. */
208 unsigned int regno; /* The register it sets */
209 short lifetime; /* lifetime of that register;
210 may be adjusted when matching movables
211 that load the same value are found. */
212 short savings; /* Number of insns we can move for this reg,
213 including other movables that force this
214 or match this one. */
215 ENUM_BITFIELD(machine_mode) savemode : 8; /* Nonzero means it is a mode for
216 a low part that we should avoid changing when
217 clearing the rest of the reg. */
218 unsigned int cond : 1; /* 1 if only conditionally movable */
219 unsigned int force : 1; /* 1 means MUST move this insn */
220 unsigned int global : 1; /* 1 means reg is live outside this loop */
221 /* If PARTIAL is 1, GLOBAL means something different:
222 that the reg is live outside the range from where it is set
223 to the following label. */
224 unsigned int done : 1; /* 1 inhibits further processing of this */
226 unsigned int partial : 1; /* 1 means this reg is used for zero-extending.
227 In particular, moving it does not make it
229 unsigned int move_insn : 1; /* 1 means that we call emit_move_insn to
230 load SRC, rather than copying INSN. */
231 unsigned int move_insn_first:1;/* Same as above, if this is necessary for the
232 first insn of a consecutive sets group. */
233 unsigned int is_equiv : 1; /* 1 means a REG_EQUIV is present on INSN. */
234 unsigned int insert_temp : 1; /* 1 means we copy to a new pseudo and replace
235 the original insn with a copy from that
236 pseudo, rather than deleting it. */
237 struct movable *match; /* First entry for same value */
238 struct movable *forces; /* An insn that must be moved if this is */
239 struct movable *next;
243 FILE *loop_dump_stream;
245 /* Forward declarations. */
247 static void invalidate_loops_containing_label (rtx);
248 static void find_and_verify_loops (rtx, struct loops *);
249 static void mark_loop_jump (rtx, struct loop *);
250 static void prescan_loop (struct loop *);
251 static int reg_in_basic_block_p (rtx, rtx);
252 static int consec_sets_invariant_p (const struct loop *, rtx, int, rtx);
253 static int labels_in_range_p (rtx, int);
254 static void count_one_set (struct loop_regs *, rtx, rtx, rtx *);
255 static void note_addr_stored (rtx, rtx, void *);
256 static void note_set_pseudo_multiple_uses (rtx, rtx, void *);
257 static int loop_reg_used_before_p (const struct loop *, rtx, rtx);
258 static void scan_loop (struct loop*, int);
260 static void replace_call_address (rtx, rtx, rtx);
262 static rtx skip_consec_insns (rtx, int);
263 static int libcall_benefit (rtx);
264 static void ignore_some_movables (struct loop_movables *);
265 static void force_movables (struct loop_movables *);
266 static void combine_movables (struct loop_movables *, struct loop_regs *);
267 static int num_unmoved_movables (const struct loop *);
268 static int regs_match_p (rtx, rtx, struct loop_movables *);
269 static int rtx_equal_for_loop_p (rtx, rtx, struct loop_movables *,
271 static void add_label_notes (rtx, rtx);
272 static void move_movables (struct loop *loop, struct loop_movables *, int,
274 static void loop_movables_add (struct loop_movables *, struct movable *);
275 static void loop_movables_free (struct loop_movables *);
276 static int count_nonfixed_reads (const struct loop *, rtx);
277 static void loop_bivs_find (struct loop *);
278 static void loop_bivs_init_find (struct loop *);
279 static void loop_bivs_check (struct loop *);
280 static void loop_givs_find (struct loop *);
281 static void loop_givs_check (struct loop *);
282 static int loop_biv_eliminable_p (struct loop *, struct iv_class *, int, int);
283 static int loop_giv_reduce_benefit (struct loop *, struct iv_class *,
284 struct induction *, rtx);
285 static void loop_givs_dead_check (struct loop *, struct iv_class *);
286 static void loop_givs_reduce (struct loop *, struct iv_class *);
287 static void loop_givs_rescan (struct loop *, struct iv_class *, rtx *);
288 static void loop_ivs_free (struct loop *);
289 static void strength_reduce (struct loop *, int);
290 static void find_single_use_in_loop (struct loop_regs *, rtx, rtx);
291 static int valid_initial_value_p (rtx, rtx, int, rtx);
292 static void find_mem_givs (const struct loop *, rtx, rtx, int, int);
293 static void record_biv (struct loop *, struct induction *, rtx, rtx, rtx,
294 rtx, rtx *, int, int);
295 static void check_final_value (const struct loop *, struct induction *);
296 static void loop_ivs_dump (const struct loop *, FILE *, int);
297 static void loop_iv_class_dump (const struct iv_class *, FILE *, int);
298 static void loop_biv_dump (const struct induction *, FILE *, int);
299 static void loop_giv_dump (const struct induction *, FILE *, int);
300 static void record_giv (const struct loop *, struct induction *, rtx, rtx,
301 rtx, rtx, rtx, rtx, int, enum g_types, int, int,
303 static void update_giv_derive (const struct loop *, rtx);
304 static void check_ext_dependent_givs (const struct loop *, struct iv_class *);
305 static int basic_induction_var (const struct loop *, rtx, enum machine_mode,
306 rtx, rtx, rtx *, rtx *, rtx **);
307 static rtx simplify_giv_expr (const struct loop *, rtx, rtx *, int *);
308 static int general_induction_var (const struct loop *loop, rtx, rtx *, rtx *,
309 rtx *, rtx *, int, int *, enum machine_mode);
310 static int consec_sets_giv (const struct loop *, int, rtx, rtx, rtx, rtx *,
311 rtx *, rtx *, rtx *);
312 static int check_dbra_loop (struct loop *, int);
313 static rtx express_from_1 (rtx, rtx, rtx);
314 static rtx combine_givs_p (struct induction *, struct induction *);
315 static int cmp_combine_givs_stats (const void *, const void *);
316 static void combine_givs (struct loop_regs *, struct iv_class *);
317 static int product_cheap_p (rtx, rtx);
318 static int maybe_eliminate_biv (const struct loop *, struct iv_class *, int,
320 static int maybe_eliminate_biv_1 (const struct loop *, rtx, rtx,
321 struct iv_class *, int, basic_block, rtx);
322 static int last_use_this_basic_block (rtx, rtx);
323 static void record_initial (rtx, rtx, void *);
324 static void update_reg_last_use (rtx, rtx);
325 static rtx next_insn_in_loop (const struct loop *, rtx);
326 static void loop_regs_scan (const struct loop *, int);
327 static int count_insns_in_loop (const struct loop *);
328 static int find_mem_in_note_1 (rtx *, void *);
329 static rtx find_mem_in_note (rtx);
330 static void load_mems (const struct loop *);
331 static int insert_loop_mem (rtx *, void *);
332 static int replace_loop_mem (rtx *, void *);
333 static void replace_loop_mems (rtx, rtx, rtx, int);
334 static int replace_loop_reg (rtx *, void *);
335 static void replace_loop_regs (rtx insn, rtx, rtx);
336 static void note_reg_stored (rtx, rtx, void *);
337 static void try_copy_prop (const struct loop *, rtx, unsigned int);
338 static void try_swap_copy_prop (const struct loop *, rtx, unsigned int);
339 static rtx check_insn_for_givs (struct loop *, rtx, int, int);
340 static rtx check_insn_for_bivs (struct loop *, rtx, int, int);
341 static rtx gen_add_mult (rtx, rtx, rtx, rtx);
342 static void loop_regs_update (const struct loop *, rtx);
343 static int iv_add_mult_cost (rtx, rtx, rtx, rtx);
345 static rtx loop_insn_emit_after (const struct loop *, basic_block, rtx, rtx);
346 static rtx loop_call_insn_emit_before (const struct loop *, basic_block,
348 static rtx loop_call_insn_hoist (const struct loop *, rtx);
349 static rtx loop_insn_sink_or_swim (const struct loop *, rtx);
351 static void loop_dump_aux (const struct loop *, FILE *, int);
352 static void loop_delete_insns (rtx, rtx);
353 static HOST_WIDE_INT remove_constant_addition (rtx *);
354 static rtx gen_load_of_final_value (rtx, rtx);
355 void debug_ivs (const struct loop *);
356 void debug_iv_class (const struct iv_class *);
357 void debug_biv (const struct induction *);
358 void debug_giv (const struct induction *);
359 void debug_loop (const struct loop *);
360 void debug_loops (const struct loops *);
362 typedef struct loop_replace_args
369 /* Nonzero iff INSN is between START and END, inclusive. */
370 #define INSN_IN_RANGE_P(INSN, START, END) \
371 (INSN_UID (INSN) < max_uid_for_loop \
372 && INSN_LUID (INSN) >= INSN_LUID (START) \
373 && INSN_LUID (INSN) <= INSN_LUID (END))
375 /* Indirect_jump_in_function is computed once per function. */
376 static int indirect_jump_in_function;
377 static int indirect_jump_in_function_p (rtx);
379 static int compute_luids (rtx, rtx, int);
381 static int biv_elimination_giv_has_0_offset (struct induction *,
382 struct induction *, rtx);
384 /* Benefit penalty, if a giv is not replaceable, i.e. must emit an insn to
385 copy the value of the strength reduced giv to its original register. */
386 static int copy_cost;
388 /* Cost of using a register, to normalize the benefits of a giv. */
389 static int reg_address_cost;
394 rtx reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
396 reg_address_cost = address_cost (reg, SImode);
398 copy_cost = COSTS_N_INSNS (1);
401 /* Compute the mapping from uids to luids.
402 LUIDs are numbers assigned to insns, like uids,
403 except that luids increase monotonically through the code.
404 Start at insn START and stop just before END. Assign LUIDs
405 starting with PREV_LUID + 1. Return the last assigned LUID + 1. */
407 compute_luids (rtx start, rtx end, int prev_luid)
412 for (insn = start, i = prev_luid; insn != end; insn = NEXT_INSN (insn))
414 if (INSN_UID (insn) >= max_uid_for_loop)
416 /* Don't assign luids to line-number NOTEs, so that the distance in
417 luids between two insns is not affected by -g. */
418 if (GET_CODE (insn) != NOTE
419 || NOTE_LINE_NUMBER (insn) <= 0)
420 uid_luid[INSN_UID (insn)] = ++i;
422 /* Give a line number note the same luid as preceding insn. */
423 uid_luid[INSN_UID (insn)] = i;
428 /* Entry point of this file. Perform loop optimization
429 on the current function. F is the first insn of the function
430 and DUMPFILE is a stream for output of a trace of actions taken
431 (or 0 if none should be output). */
434 loop_optimize (rtx f, FILE *dumpfile, int flags)
438 struct loops loops_data;
439 struct loops *loops = &loops_data;
440 struct loop_info *loops_info;
442 loop_dump_stream = dumpfile;
444 init_recog_no_volatile ();
446 max_reg_before_loop = max_reg_num ();
447 loop_max_reg = max_reg_before_loop;
451 /* Count the number of loops. */
454 for (insn = f; insn; insn = NEXT_INSN (insn))
456 if (GET_CODE (insn) == NOTE
457 && NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
461 /* Don't waste time if no loops. */
462 if (max_loop_num == 0)
465 loops->num = max_loop_num;
467 /* Get size to use for tables indexed by uids.
468 Leave some space for labels allocated by find_and_verify_loops. */
469 max_uid_for_loop = get_max_uid () + 1 + max_loop_num * 32;
471 uid_luid = xcalloc (max_uid_for_loop, sizeof (int));
472 uid_loop = xcalloc (max_uid_for_loop, sizeof (struct loop *));
474 /* Allocate storage for array of loops. */
475 loops->array = xcalloc (loops->num, sizeof (struct loop));
477 /* Find and process each loop.
478 First, find them, and record them in order of their beginnings. */
479 find_and_verify_loops (f, loops);
481 /* Allocate and initialize auxiliary loop information. */
482 loops_info = xcalloc (loops->num, sizeof (struct loop_info));
483 for (i = 0; i < (int) loops->num; i++)
484 loops->array[i].aux = loops_info + i;
486 /* Now find all register lifetimes. This must be done after
487 find_and_verify_loops, because it might reorder the insns in the
489 reg_scan (f, max_reg_before_loop, 1);
491 /* This must occur after reg_scan so that registers created by gcse
492 will have entries in the register tables.
494 We could have added a call to reg_scan after gcse_main in toplev.c,
495 but moving this call to init_alias_analysis is more efficient. */
496 init_alias_analysis ();
498 /* See if we went too far. Note that get_max_uid already returns
499 one more that the maximum uid of all insn. */
500 if (get_max_uid () > max_uid_for_loop)
502 /* Now reset it to the actual size we need. See above. */
503 max_uid_for_loop = get_max_uid ();
505 /* find_and_verify_loops has already called compute_luids, but it
506 might have rearranged code afterwards, so we need to recompute
508 compute_luids (f, NULL_RTX, 0);
510 /* Don't leave gaps in uid_luid for insns that have been
511 deleted. It is possible that the first or last insn
512 using some register has been deleted by cross-jumping.
513 Make sure that uid_luid for that former insn's uid
514 points to the general area where that insn used to be. */
515 for (i = 0; i < max_uid_for_loop; i++)
517 uid_luid[0] = uid_luid[i];
518 if (uid_luid[0] != 0)
521 for (i = 0; i < max_uid_for_loop; i++)
522 if (uid_luid[i] == 0)
523 uid_luid[i] = uid_luid[i - 1];
525 /* Determine if the function has indirect jump. On some systems
526 this prevents low overhead loop instructions from being used. */
527 indirect_jump_in_function = indirect_jump_in_function_p (f);
529 /* Now scan the loops, last ones first, since this means inner ones are done
530 before outer ones. */
531 for (i = max_loop_num - 1; i >= 0; i--)
533 struct loop *loop = &loops->array[i];
535 if (! loop->invalid && loop->end)
536 scan_loop (loop, flags);
539 end_alias_analysis ();
548 /* Returns the next insn, in execution order, after INSN. START and
549 END are the NOTE_INSN_LOOP_BEG and NOTE_INSN_LOOP_END for the loop,
550 respectively. LOOP->TOP, if non-NULL, is the top of the loop in the
551 insn-stream; it is used with loops that are entered near the
555 next_insn_in_loop (const struct loop *loop, rtx insn)
557 insn = NEXT_INSN (insn);
559 if (insn == loop->end)
562 /* Go to the top of the loop, and continue there. */
569 if (insn == loop->scan_start)
576 /* Optimize one loop described by LOOP. */
578 /* ??? Could also move memory writes out of loops if the destination address
579 is invariant, the source is invariant, the memory write is not volatile,
580 and if we can prove that no read inside the loop can read this address
581 before the write occurs. If there is a read of this address after the
582 write, then we can also mark the memory read as invariant. */
585 scan_loop (struct loop *loop, int flags)
587 struct loop_info *loop_info = LOOP_INFO (loop);
588 struct loop_regs *regs = LOOP_REGS (loop);
590 rtx loop_start = loop->start;
591 rtx loop_end = loop->end;
593 /* 1 if we are scanning insns that could be executed zero times. */
595 /* 1 if we are scanning insns that might never be executed
596 due to a subroutine call which might exit before they are reached. */
598 /* Number of insns in the loop. */
601 rtx temp, update_start, update_end;
602 /* The SET from an insn, if it is the only SET in the insn. */
604 /* Chain describing insns movable in current loop. */
605 struct loop_movables *movables = LOOP_MOVABLES (loop);
606 /* Ratio of extra register life span we can justify
607 for saving an instruction. More if loop doesn't call subroutines
608 since in that case saving an insn makes more difference
609 and more registers are available. */
611 /* Nonzero if we are scanning instructions in a sub-loop. */
620 /* Determine whether this loop starts with a jump down to a test at
621 the end. This will occur for a small number of loops with a test
622 that is too complex to duplicate in front of the loop.
624 We search for the first insn or label in the loop, skipping NOTEs.
625 However, we must be careful not to skip past a NOTE_INSN_LOOP_BEG
626 (because we might have a loop executed only once that contains a
627 loop which starts with a jump to its exit test) or a NOTE_INSN_LOOP_END
628 (in case we have a degenerate loop).
630 Note that if we mistakenly think that a loop is entered at the top
631 when, in fact, it is entered at the exit test, the only effect will be
632 slightly poorer optimization. Making the opposite error can generate
633 incorrect code. Since very few loops now start with a jump to the
634 exit test, the code here to detect that case is very conservative. */
636 for (p = NEXT_INSN (loop_start);
638 && GET_CODE (p) != CODE_LABEL && ! INSN_P (p)
639 && (GET_CODE (p) != NOTE
640 || (NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_BEG
641 && NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_END));
645 loop->scan_start = p;
647 /* If loop end is the end of the current function, then emit a
648 NOTE_INSN_DELETED after loop_end and set loop->sink to the dummy
649 note insn. This is the position we use when sinking insns out of
651 if (NEXT_INSN (loop->end) != 0)
652 loop->sink = NEXT_INSN (loop->end);
654 loop->sink = emit_note_after (NOTE_INSN_DELETED, loop->end);
656 /* Set up variables describing this loop. */
658 threshold = (loop_info->has_call ? 1 : 2) * (1 + n_non_fixed_regs);
660 /* If loop has a jump before the first label,
661 the true entry is the target of that jump.
662 Start scan from there.
663 But record in LOOP->TOP the place where the end-test jumps
664 back to so we can scan that after the end of the loop. */
665 if (GET_CODE (p) == JUMP_INSN
666 /* Loop entry must be unconditional jump (and not a RETURN) */
667 && any_uncondjump_p (p)
668 && JUMP_LABEL (p) != 0
669 /* Check to see whether the jump actually
670 jumps out of the loop (meaning it's no loop).
671 This case can happen for things like
672 do {..} while (0). If this label was generated previously
673 by loop, we can't tell anything about it and have to reject
675 && INSN_IN_RANGE_P (JUMP_LABEL (p), loop_start, loop_end))
677 loop->top = next_label (loop->scan_start);
678 loop->scan_start = JUMP_LABEL (p);
681 /* If LOOP->SCAN_START was an insn created by loop, we don't know its luid
682 as required by loop_reg_used_before_p. So skip such loops. (This
683 test may never be true, but it's best to play it safe.)
685 Also, skip loops where we do not start scanning at a label. This
686 test also rejects loops starting with a JUMP_INSN that failed the
689 if (INSN_UID (loop->scan_start) >= max_uid_for_loop
690 || GET_CODE (loop->scan_start) != CODE_LABEL)
692 if (loop_dump_stream)
693 fprintf (loop_dump_stream, "\nLoop from %d to %d is phony.\n\n",
694 INSN_UID (loop_start), INSN_UID (loop_end));
698 /* Allocate extra space for REGs that might be created by load_mems.
699 We allocate a little extra slop as well, in the hopes that we
700 won't have to reallocate the regs array. */
701 loop_regs_scan (loop, loop_info->mems_idx + 16);
702 insn_count = count_insns_in_loop (loop);
704 if (loop_dump_stream)
706 fprintf (loop_dump_stream, "\nLoop from %d to %d: %d real insns.\n",
707 INSN_UID (loop_start), INSN_UID (loop_end), insn_count);
709 fprintf (loop_dump_stream, "Continue at insn %d.\n",
710 INSN_UID (loop->cont));
713 /* Scan through the loop finding insns that are safe to move.
714 Set REGS->ARRAY[I].SET_IN_LOOP negative for the reg I being set, so that
715 this reg will be considered invariant for subsequent insns.
716 We consider whether subsequent insns use the reg
717 in deciding whether it is worth actually moving.
719 MAYBE_NEVER is nonzero if we have passed a conditional jump insn
720 and therefore it is possible that the insns we are scanning
721 would never be executed. At such times, we must make sure
722 that it is safe to execute the insn once instead of zero times.
723 When MAYBE_NEVER is 0, all insns will be executed at least once
724 so that is not a problem. */
726 for (in_libcall = 0, p = next_insn_in_loop (loop, loop->scan_start);
728 p = next_insn_in_loop (loop, p))
730 if (in_libcall && INSN_P (p) && find_reg_note (p, REG_RETVAL, NULL_RTX))
732 if (GET_CODE (p) == INSN)
734 temp = find_reg_note (p, REG_LIBCALL, NULL_RTX);
738 && (set = single_set (p))
739 && GET_CODE (SET_DEST (set)) == REG
740 #ifdef PIC_OFFSET_TABLE_REG_CALL_CLOBBERED
741 && SET_DEST (set) != pic_offset_table_rtx
743 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
749 rtx src = SET_SRC (set);
750 rtx dependencies = 0;
752 /* Figure out what to use as a source of this insn. If a
753 REG_EQUIV note is given or if a REG_EQUAL note with a
754 constant operand is specified, use it as the source and
755 mark that we should move this insn by calling
756 emit_move_insn rather that duplicating the insn.
758 Otherwise, only use the REG_EQUAL contents if a REG_RETVAL
760 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
762 src = XEXP (temp, 0), move_insn = 1;
765 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
766 if (temp && CONSTANT_P (XEXP (temp, 0)))
767 src = XEXP (temp, 0), move_insn = 1;
768 if (temp && find_reg_note (p, REG_RETVAL, NULL_RTX))
770 src = XEXP (temp, 0);
771 /* A libcall block can use regs that don't appear in
772 the equivalent expression. To move the libcall,
773 we must move those regs too. */
774 dependencies = libcall_other_reg (p, src);
778 /* For parallels, add any possible uses to the dependencies, as
779 we can't move the insn without resolving them first. */
780 if (GET_CODE (PATTERN (p)) == PARALLEL)
782 for (i = 0; i < XVECLEN (PATTERN (p), 0); i++)
784 rtx x = XVECEXP (PATTERN (p), 0, i);
785 if (GET_CODE (x) == USE)
787 = gen_rtx_EXPR_LIST (VOIDmode, XEXP (x, 0),
792 if (/* The register is used in basic blocks other
793 than the one where it is set (meaning that
794 something after this point in the loop might
795 depend on its value before the set). */
796 ! reg_in_basic_block_p (p, SET_DEST (set))
797 /* And the set is not guaranteed to be executed once
798 the loop starts, or the value before the set is
799 needed before the set occurs...
801 ??? Note we have quadratic behavior here, mitigated
802 by the fact that the previous test will often fail for
803 large loops. Rather than re-scanning the entire loop
804 each time for register usage, we should build tables
805 of the register usage and use them here instead. */
807 || loop_reg_used_before_p (loop, set, p)))
808 /* It is unsafe to move the set. However, it may be OK to
809 move the source into a new pseudo, and substitute a
810 reg-to-reg copy for the original insn.
812 This code used to consider it OK to move a set of a variable
813 which was not created by the user and not used in an exit
815 That behavior is incorrect and was removed. */
818 /* Don't try to optimize a MODE_CC set with a constant
819 source. It probably will be combined with a conditional
821 if (GET_MODE_CLASS (GET_MODE (SET_DEST (set))) == MODE_CC
824 /* Don't try to optimize a register that was made
825 by loop-optimization for an inner loop.
826 We don't know its life-span, so we can't compute
828 else if (REGNO (SET_DEST (set)) >= max_reg_before_loop)
830 /* Don't move the source and add a reg-to-reg copy:
831 - with -Os (this certainly increases size),
832 - if the mode doesn't support copy operations (obviously),
833 - if the source is already a reg (the motion will gain nothing),
834 - if the source is a legitimate constant (likewise). */
837 || ! can_copy_p (GET_MODE (SET_SRC (set)))
838 || GET_CODE (SET_SRC (set)) == REG
839 || (CONSTANT_P (SET_SRC (set))
840 && LEGITIMATE_CONSTANT_P (SET_SRC (set)))))
842 else if ((tem = loop_invariant_p (loop, src))
843 && (dependencies == 0
845 = loop_invariant_p (loop, dependencies)) != 0)
846 && (regs->array[REGNO (SET_DEST (set))].set_in_loop == 1
848 = consec_sets_invariant_p
849 (loop, SET_DEST (set),
850 regs->array[REGNO (SET_DEST (set))].set_in_loop,
852 /* If the insn can cause a trap (such as divide by zero),
853 can't move it unless it's guaranteed to be executed
854 once loop is entered. Even a function call might
855 prevent the trap insn from being reached
856 (since it might exit!) */
857 && ! ((maybe_never || call_passed)
858 && may_trap_p (src)))
861 int regno = REGNO (SET_DEST (set));
863 /* A potential lossage is where we have a case where two insns
864 can be combined as long as they are both in the loop, but
865 we move one of them outside the loop. For large loops,
866 this can lose. The most common case of this is the address
867 of a function being called.
869 Therefore, if this register is marked as being used
870 exactly once if we are in a loop with calls
871 (a "large loop"), see if we can replace the usage of
872 this register with the source of this SET. If we can,
875 Don't do this if P has a REG_RETVAL note or if we have
876 SMALL_REGISTER_CLASSES and SET_SRC is a hard register. */
878 if (loop_info->has_call
879 && regs->array[regno].single_usage != 0
880 && regs->array[regno].single_usage != const0_rtx
881 && REGNO_FIRST_UID (regno) == INSN_UID (p)
882 && (REGNO_LAST_UID (regno)
883 == INSN_UID (regs->array[regno].single_usage))
884 && regs->array[regno].set_in_loop == 1
885 && GET_CODE (SET_SRC (set)) != ASM_OPERANDS
886 && ! side_effects_p (SET_SRC (set))
887 && ! find_reg_note (p, REG_RETVAL, NULL_RTX)
888 && (! SMALL_REGISTER_CLASSES
889 || (! (GET_CODE (SET_SRC (set)) == REG
890 && (REGNO (SET_SRC (set))
891 < FIRST_PSEUDO_REGISTER))))
892 /* This test is not redundant; SET_SRC (set) might be
893 a call-clobbered register and the life of REGNO
894 might span a call. */
895 && ! modified_between_p (SET_SRC (set), p,
896 regs->array[regno].single_usage)
897 && no_labels_between_p (p,
898 regs->array[regno].single_usage)
899 && validate_replace_rtx (SET_DEST (set), SET_SRC (set),
900 regs->array[regno].single_usage))
902 /* Replace any usage in a REG_EQUAL note. Must copy
903 the new source, so that we don't get rtx sharing
904 between the SET_SOURCE and REG_NOTES of insn p. */
905 REG_NOTES (regs->array[regno].single_usage)
907 (REG_NOTES (regs->array[regno].single_usage),
908 SET_DEST (set), copy_rtx (SET_SRC (set))));
911 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
913 regs->array[regno+i].set_in_loop = 0;
917 m = xmalloc (sizeof (struct movable));
921 m->dependencies = dependencies;
922 m->set_dest = SET_DEST (set);
925 = regs->array[REGNO (SET_DEST (set))].set_in_loop - 1;
929 m->move_insn = move_insn;
930 m->move_insn_first = 0;
931 m->insert_temp = insert_temp;
932 m->is_equiv = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
933 m->savemode = VOIDmode;
935 /* Set M->cond if either loop_invariant_p
936 or consec_sets_invariant_p returned 2
937 (only conditionally invariant). */
938 m->cond = ((tem | tem1 | tem2) > 1);
939 m->global = LOOP_REG_GLOBAL_P (loop, regno);
941 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
942 m->savings = regs->array[regno].n_times_set;
943 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
944 m->savings += libcall_benefit (p);
945 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set)); i++)
946 regs->array[regno+i].set_in_loop = move_insn ? -2 : -1;
947 /* Add M to the end of the chain MOVABLES. */
948 loop_movables_add (movables, m);
952 /* It is possible for the first instruction to have a
953 REG_EQUAL note but a non-invariant SET_SRC, so we must
954 remember the status of the first instruction in case
955 the last instruction doesn't have a REG_EQUAL note. */
956 m->move_insn_first = m->move_insn;
958 /* Skip this insn, not checking REG_LIBCALL notes. */
959 p = next_nonnote_insn (p);
960 /* Skip the consecutive insns, if there are any. */
961 p = skip_consec_insns (p, m->consec);
962 /* Back up to the last insn of the consecutive group. */
963 p = prev_nonnote_insn (p);
965 /* We must now reset m->move_insn, m->is_equiv, and
966 possibly m->set_src to correspond to the effects of
968 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
970 m->set_src = XEXP (temp, 0), m->move_insn = 1;
973 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
974 if (temp && CONSTANT_P (XEXP (temp, 0)))
975 m->set_src = XEXP (temp, 0), m->move_insn = 1;
981 = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
984 /* If this register is always set within a STRICT_LOW_PART
985 or set to zero, then its high bytes are constant.
986 So clear them outside the loop and within the loop
987 just load the low bytes.
988 We must check that the machine has an instruction to do so.
989 Also, if the value loaded into the register
990 depends on the same register, this cannot be done. */
991 else if (SET_SRC (set) == const0_rtx
992 && GET_CODE (NEXT_INSN (p)) == INSN
993 && (set1 = single_set (NEXT_INSN (p)))
994 && GET_CODE (set1) == SET
995 && (GET_CODE (SET_DEST (set1)) == STRICT_LOW_PART)
996 && (GET_CODE (XEXP (SET_DEST (set1), 0)) == SUBREG)
997 && (SUBREG_REG (XEXP (SET_DEST (set1), 0))
999 && !reg_mentioned_p (SET_DEST (set), SET_SRC (set1)))
1001 int regno = REGNO (SET_DEST (set));
1002 if (regs->array[regno].set_in_loop == 2)
1005 m = xmalloc (sizeof (struct movable));
1008 m->set_dest = SET_DEST (set);
1009 m->dependencies = 0;
1015 m->move_insn_first = 0;
1016 m->insert_temp = insert_temp;
1018 /* If the insn may not be executed on some cycles,
1019 we can't clear the whole reg; clear just high part.
1020 Not even if the reg is used only within this loop.
1027 Clearing x before the inner loop could clobber a value
1028 being saved from the last time around the outer loop.
1029 However, if the reg is not used outside this loop
1030 and all uses of the register are in the same
1031 basic block as the store, there is no problem.
1033 If this insn was made by loop, we don't know its
1034 INSN_LUID and hence must make a conservative
1036 m->global = (INSN_UID (p) >= max_uid_for_loop
1037 || LOOP_REG_GLOBAL_P (loop, regno)
1038 || (labels_in_range_p
1039 (p, REGNO_FIRST_LUID (regno))));
1040 if (maybe_never && m->global)
1041 m->savemode = GET_MODE (SET_SRC (set1));
1043 m->savemode = VOIDmode;
1047 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
1050 i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
1052 regs->array[regno+i].set_in_loop = -1;
1053 /* Add M to the end of the chain MOVABLES. */
1054 loop_movables_add (movables, m);
1059 /* Past a call insn, we get to insns which might not be executed
1060 because the call might exit. This matters for insns that trap.
1061 Constant and pure call insns always return, so they don't count. */
1062 else if (GET_CODE (p) == CALL_INSN && ! CONST_OR_PURE_CALL_P (p))
1064 /* Past a label or a jump, we get to insns for which we
1065 can't count on whether or how many times they will be
1066 executed during each iteration. Therefore, we can
1067 only move out sets of trivial variables
1068 (those not used after the loop). */
1069 /* Similar code appears twice in strength_reduce. */
1070 else if ((GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN)
1071 /* If we enter the loop in the middle, and scan around to the
1072 beginning, don't set maybe_never for that. This must be an
1073 unconditional jump, otherwise the code at the top of the
1074 loop might never be executed. Unconditional jumps are
1075 followed by a barrier then the loop_end. */
1076 && ! (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == loop->top
1077 && NEXT_INSN (NEXT_INSN (p)) == loop_end
1078 && any_uncondjump_p (p)))
1080 else if (GET_CODE (p) == NOTE)
1082 /* At the virtual top of a converted loop, insns are again known to
1083 be executed: logically, the loop begins here even though the exit
1084 code has been duplicated. */
1085 if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP && loop_depth == 0)
1086 maybe_never = call_passed = 0;
1087 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
1089 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
1094 /* If one movable subsumes another, ignore that other. */
1096 ignore_some_movables (movables);
1098 /* For each movable insn, see if the reg that it loads
1099 leads when it dies right into another conditionally movable insn.
1100 If so, record that the second insn "forces" the first one,
1101 since the second can be moved only if the first is. */
1103 force_movables (movables);
1105 /* See if there are multiple movable insns that load the same value.
1106 If there are, make all but the first point at the first one
1107 through the `match' field, and add the priorities of them
1108 all together as the priority of the first. */
1110 combine_movables (movables, regs);
1112 /* Now consider each movable insn to decide whether it is worth moving.
1113 Store 0 in regs->array[I].set_in_loop for each reg I that is moved.
1115 For machines with few registers this increases code size, so do not
1116 move moveables when optimizing for code size on such machines.
1117 (The 18 below is the value for i386.) */
1120 || (reg_class_size[GENERAL_REGS] > 18 && !loop_info->has_call))
1122 move_movables (loop, movables, threshold, insn_count);
1124 /* Recalculate regs->array if move_movables has created new
1126 if (max_reg_num () > regs->num)
1128 loop_regs_scan (loop, 0);
1129 for (update_start = loop_start;
1130 PREV_INSN (update_start)
1131 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1132 update_start = PREV_INSN (update_start))
1134 update_end = NEXT_INSN (loop_end);
1136 reg_scan_update (update_start, update_end, loop_max_reg);
1137 loop_max_reg = max_reg_num ();
1141 /* Now candidates that still are negative are those not moved.
1142 Change regs->array[I].set_in_loop to indicate that those are not actually
1144 for (i = 0; i < regs->num; i++)
1145 if (regs->array[i].set_in_loop < 0)
1146 regs->array[i].set_in_loop = regs->array[i].n_times_set;
1148 /* Now that we've moved some things out of the loop, we might be able to
1149 hoist even more memory references. */
1152 /* Recalculate regs->array if load_mems has created new registers. */
1153 if (max_reg_num () > regs->num)
1154 loop_regs_scan (loop, 0);
1156 for (update_start = loop_start;
1157 PREV_INSN (update_start)
1158 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1159 update_start = PREV_INSN (update_start))
1161 update_end = NEXT_INSN (loop_end);
1163 reg_scan_update (update_start, update_end, loop_max_reg);
1164 loop_max_reg = max_reg_num ();
1166 if (flag_strength_reduce)
1168 if (update_end && GET_CODE (update_end) == CODE_LABEL)
1169 /* Ensure our label doesn't go away. */
1170 LABEL_NUSES (update_end)++;
1172 strength_reduce (loop, flags);
1174 reg_scan_update (update_start, update_end, loop_max_reg);
1175 loop_max_reg = max_reg_num ();
1177 if (update_end && GET_CODE (update_end) == CODE_LABEL
1178 && --LABEL_NUSES (update_end) == 0)
1179 delete_related_insns (update_end);
1183 /* The movable information is required for strength reduction. */
1184 loop_movables_free (movables);
1191 /* Add elements to *OUTPUT to record all the pseudo-regs
1192 mentioned in IN_THIS but not mentioned in NOT_IN_THIS. */
1195 record_excess_regs (rtx in_this, rtx not_in_this, rtx *output)
1201 code = GET_CODE (in_this);
1215 if (REGNO (in_this) >= FIRST_PSEUDO_REGISTER
1216 && ! reg_mentioned_p (in_this, not_in_this))
1217 *output = gen_rtx_EXPR_LIST (VOIDmode, in_this, *output);
1224 fmt = GET_RTX_FORMAT (code);
1225 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1232 for (j = 0; j < XVECLEN (in_this, i); j++)
1233 record_excess_regs (XVECEXP (in_this, i, j), not_in_this, output);
1237 record_excess_regs (XEXP (in_this, i), not_in_this, output);
1243 /* Check what regs are referred to in the libcall block ending with INSN,
1244 aside from those mentioned in the equivalent value.
1245 If there are none, return 0.
1246 If there are one or more, return an EXPR_LIST containing all of them. */
1249 libcall_other_reg (rtx insn, rtx equiv)
1251 rtx note = find_reg_note (insn, REG_RETVAL, NULL_RTX);
1252 rtx p = XEXP (note, 0);
1255 /* First, find all the regs used in the libcall block
1256 that are not mentioned as inputs to the result. */
1260 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
1261 || GET_CODE (p) == CALL_INSN)
1262 record_excess_regs (PATTERN (p), equiv, &output);
1269 /* Return 1 if all uses of REG
1270 are between INSN and the end of the basic block. */
1273 reg_in_basic_block_p (rtx insn, rtx reg)
1275 int regno = REGNO (reg);
1278 if (REGNO_FIRST_UID (regno) != INSN_UID (insn))
1281 /* Search this basic block for the already recorded last use of the reg. */
1282 for (p = insn; p; p = NEXT_INSN (p))
1284 switch (GET_CODE (p))
1291 /* Ordinary insn: if this is the last use, we win. */
1292 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1297 /* Jump insn: if this is the last use, we win. */
1298 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1300 /* Otherwise, it's the end of the basic block, so we lose. */
1305 /* It's the end of the basic block, so we lose. */
1313 /* The "last use" that was recorded can't be found after the first
1314 use. This can happen when the last use was deleted while
1315 processing an inner loop, this inner loop was then completely
1316 unrolled, and the outer loop is always exited after the inner loop,
1317 so that everything after the first use becomes a single basic block. */
1321 /* Compute the benefit of eliminating the insns in the block whose
1322 last insn is LAST. This may be a group of insns used to compute a
1323 value directly or can contain a library call. */
1326 libcall_benefit (rtx last)
1331 for (insn = XEXP (find_reg_note (last, REG_RETVAL, NULL_RTX), 0);
1332 insn != last; insn = NEXT_INSN (insn))
1334 if (GET_CODE (insn) == CALL_INSN)
1335 benefit += 10; /* Assume at least this many insns in a library
1337 else if (GET_CODE (insn) == INSN
1338 && GET_CODE (PATTERN (insn)) != USE
1339 && GET_CODE (PATTERN (insn)) != CLOBBER)
1346 /* Skip COUNT insns from INSN, counting library calls as 1 insn. */
1349 skip_consec_insns (rtx insn, int count)
1351 for (; count > 0; count--)
1355 /* If first insn of libcall sequence, skip to end. */
1356 /* Do this at start of loop, since INSN is guaranteed to
1358 if (GET_CODE (insn) != NOTE
1359 && (temp = find_reg_note (insn, REG_LIBCALL, NULL_RTX)))
1360 insn = XEXP (temp, 0);
1363 insn = NEXT_INSN (insn);
1364 while (GET_CODE (insn) == NOTE);
1370 /* Ignore any movable whose insn falls within a libcall
1371 which is part of another movable.
1372 We make use of the fact that the movable for the libcall value
1373 was made later and so appears later on the chain. */
1376 ignore_some_movables (struct loop_movables *movables)
1378 struct movable *m, *m1;
1380 for (m = movables->head; m; m = m->next)
1382 /* Is this a movable for the value of a libcall? */
1383 rtx note = find_reg_note (m->insn, REG_RETVAL, NULL_RTX);
1387 /* Check for earlier movables inside that range,
1388 and mark them invalid. We cannot use LUIDs here because
1389 insns created by loop.c for prior loops don't have LUIDs.
1390 Rather than reject all such insns from movables, we just
1391 explicitly check each insn in the libcall (since invariant
1392 libcalls aren't that common). */
1393 for (insn = XEXP (note, 0); insn != m->insn; insn = NEXT_INSN (insn))
1394 for (m1 = movables->head; m1 != m; m1 = m1->next)
1395 if (m1->insn == insn)
1401 /* For each movable insn, see if the reg that it loads
1402 leads when it dies right into another conditionally movable insn.
1403 If so, record that the second insn "forces" the first one,
1404 since the second can be moved only if the first is. */
1407 force_movables (struct loop_movables *movables)
1409 struct movable *m, *m1;
1411 for (m1 = movables->head; m1; m1 = m1->next)
1412 /* Omit this if moving just the (SET (REG) 0) of a zero-extend. */
1413 if (!m1->partial && !m1->done)
1415 int regno = m1->regno;
1416 for (m = m1->next; m; m = m->next)
1417 /* ??? Could this be a bug? What if CSE caused the
1418 register of M1 to be used after this insn?
1419 Since CSE does not update regno_last_uid,
1420 this insn M->insn might not be where it dies.
1421 But very likely this doesn't matter; what matters is
1422 that M's reg is computed from M1's reg. */
1423 if (INSN_UID (m->insn) == REGNO_LAST_UID (regno)
1426 if (m != 0 && m->set_src == m1->set_dest
1427 /* If m->consec, m->set_src isn't valid. */
1431 /* Increase the priority of the moving the first insn
1432 since it permits the second to be moved as well. */
1436 m1->lifetime += m->lifetime;
1437 m1->savings += m->savings;
1442 /* Find invariant expressions that are equal and can be combined into
1446 combine_movables (struct loop_movables *movables, struct loop_regs *regs)
1449 char *matched_regs = xmalloc (regs->num);
1450 enum machine_mode mode;
1452 /* Regs that are set more than once are not allowed to match
1453 or be matched. I'm no longer sure why not. */
1454 /* Only pseudo registers are allowed to match or be matched,
1455 since move_movables does not validate the change. */
1456 /* Perhaps testing m->consec_sets would be more appropriate here? */
1458 for (m = movables->head; m; m = m->next)
1459 if (m->match == 0 && regs->array[m->regno].n_times_set == 1
1460 && m->regno >= FIRST_PSEUDO_REGISTER
1465 int regno = m->regno;
1467 memset (matched_regs, 0, regs->num);
1468 matched_regs[regno] = 1;
1470 /* We want later insns to match the first one. Don't make the first
1471 one match any later ones. So start this loop at m->next. */
1472 for (m1 = m->next; m1; m1 = m1->next)
1473 if (m != m1 && m1->match == 0
1475 && regs->array[m1->regno].n_times_set == 1
1476 && m1->regno >= FIRST_PSEUDO_REGISTER
1477 /* A reg used outside the loop mustn't be eliminated. */
1479 /* A reg used for zero-extending mustn't be eliminated. */
1481 && (matched_regs[m1->regno]
1484 /* Can combine regs with different modes loaded from the
1485 same constant only if the modes are the same or
1486 if both are integer modes with M wider or the same
1487 width as M1. The check for integer is redundant, but
1488 safe, since the only case of differing destination
1489 modes with equal sources is when both sources are
1490 VOIDmode, i.e., CONST_INT. */
1491 (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest)
1492 || (GET_MODE_CLASS (GET_MODE (m->set_dest)) == MODE_INT
1493 && GET_MODE_CLASS (GET_MODE (m1->set_dest)) == MODE_INT
1494 && (GET_MODE_BITSIZE (GET_MODE (m->set_dest))
1495 >= GET_MODE_BITSIZE (GET_MODE (m1->set_dest)))))
1496 /* See if the source of M1 says it matches M. */
1497 && ((GET_CODE (m1->set_src) == REG
1498 && matched_regs[REGNO (m1->set_src)])
1499 || rtx_equal_for_loop_p (m->set_src, m1->set_src,
1501 && ((m->dependencies == m1->dependencies)
1502 || rtx_equal_p (m->dependencies, m1->dependencies)))
1504 m->lifetime += m1->lifetime;
1505 m->savings += m1->savings;
1508 matched_regs[m1->regno] = 1;
1512 /* Now combine the regs used for zero-extension.
1513 This can be done for those not marked `global'
1514 provided their lives don't overlap. */
1516 for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
1517 mode = GET_MODE_WIDER_MODE (mode))
1519 struct movable *m0 = 0;
1521 /* Combine all the registers for extension from mode MODE.
1522 Don't combine any that are used outside this loop. */
1523 for (m = movables->head; m; m = m->next)
1524 if (m->partial && ! m->global
1525 && mode == GET_MODE (SET_SRC (PATTERN (NEXT_INSN (m->insn)))))
1529 int first = REGNO_FIRST_LUID (m->regno);
1530 int last = REGNO_LAST_LUID (m->regno);
1534 /* First one: don't check for overlap, just record it. */
1539 /* Make sure they extend to the same mode.
1540 (Almost always true.) */
1541 if (GET_MODE (m->set_dest) != GET_MODE (m0->set_dest))
1544 /* We already have one: check for overlap with those
1545 already combined together. */
1546 for (m1 = movables->head; m1 != m; m1 = m1->next)
1547 if (m1 == m0 || (m1->partial && m1->match == m0))
1548 if (! (REGNO_FIRST_LUID (m1->regno) > last
1549 || REGNO_LAST_LUID (m1->regno) < first))
1552 /* No overlap: we can combine this with the others. */
1553 m0->lifetime += m->lifetime;
1554 m0->savings += m->savings;
1564 free (matched_regs);
1567 /* Returns the number of movable instructions in LOOP that were not
1568 moved outside the loop. */
1571 num_unmoved_movables (const struct loop *loop)
1576 for (m = LOOP_MOVABLES (loop)->head; m; m = m->next)
1584 /* Return 1 if regs X and Y will become the same if moved. */
1587 regs_match_p (rtx x, rtx y, struct loop_movables *movables)
1589 unsigned int xn = REGNO (x);
1590 unsigned int yn = REGNO (y);
1591 struct movable *mx, *my;
1593 for (mx = movables->head; mx; mx = mx->next)
1594 if (mx->regno == xn)
1597 for (my = movables->head; my; my = my->next)
1598 if (my->regno == yn)
1602 && ((mx->match == my->match && mx->match != 0)
1604 || mx == my->match));
1607 /* Return 1 if X and Y are identical-looking rtx's.
1608 This is the Lisp function EQUAL for rtx arguments.
1610 If two registers are matching movables or a movable register and an
1611 equivalent constant, consider them equal. */
1614 rtx_equal_for_loop_p (rtx x, rtx y, struct loop_movables *movables,
1615 struct loop_regs *regs)
1625 if (x == 0 || y == 0)
1628 code = GET_CODE (x);
1630 /* If we have a register and a constant, they may sometimes be
1632 if (GET_CODE (x) == REG && regs->array[REGNO (x)].set_in_loop == -2
1635 for (m = movables->head; m; m = m->next)
1636 if (m->move_insn && m->regno == REGNO (x)
1637 && rtx_equal_p (m->set_src, y))
1640 else if (GET_CODE (y) == REG && regs->array[REGNO (y)].set_in_loop == -2
1643 for (m = movables->head; m; m = m->next)
1644 if (m->move_insn && m->regno == REGNO (y)
1645 && rtx_equal_p (m->set_src, x))
1649 /* Otherwise, rtx's of different codes cannot be equal. */
1650 if (code != GET_CODE (y))
1653 /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent.
1654 (REG:SI x) and (REG:HI x) are NOT equivalent. */
1656 if (GET_MODE (x) != GET_MODE (y))
1659 /* These three types of rtx's can be compared nonrecursively. */
1661 return (REGNO (x) == REGNO (y) || regs_match_p (x, y, movables));
1663 if (code == LABEL_REF)
1664 return XEXP (x, 0) == XEXP (y, 0);
1665 if (code == SYMBOL_REF)
1666 return XSTR (x, 0) == XSTR (y, 0);
1668 /* Compare the elements. If any pair of corresponding elements
1669 fail to match, return 0 for the whole things. */
1671 fmt = GET_RTX_FORMAT (code);
1672 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1677 if (XWINT (x, i) != XWINT (y, i))
1682 if (XINT (x, i) != XINT (y, i))
1687 /* Two vectors must have the same length. */
1688 if (XVECLEN (x, i) != XVECLEN (y, i))
1691 /* And the corresponding elements must match. */
1692 for (j = 0; j < XVECLEN (x, i); j++)
1693 if (rtx_equal_for_loop_p (XVECEXP (x, i, j), XVECEXP (y, i, j),
1694 movables, regs) == 0)
1699 if (rtx_equal_for_loop_p (XEXP (x, i), XEXP (y, i), movables, regs)
1705 if (strcmp (XSTR (x, i), XSTR (y, i)))
1710 /* These are just backpointers, so they don't matter. */
1716 /* It is believed that rtx's at this level will never
1717 contain anything but integers and other rtx's,
1718 except for within LABEL_REFs and SYMBOL_REFs. */
1726 /* If X contains any LABEL_REF's, add REG_LABEL notes for them to all
1727 insns in INSNS which use the reference. LABEL_NUSES for CODE_LABEL
1728 references is incremented once for each added note. */
1731 add_label_notes (rtx x, rtx insns)
1733 enum rtx_code code = GET_CODE (x);
1738 if (code == LABEL_REF && !LABEL_REF_NONLOCAL_P (x))
1740 /* This code used to ignore labels that referred to dispatch tables to
1741 avoid flow generating (slightly) worse code.
1743 We no longer ignore such label references (see LABEL_REF handling in
1744 mark_jump_label for additional information). */
1745 for (insn = insns; insn; insn = NEXT_INSN (insn))
1746 if (reg_mentioned_p (XEXP (x, 0), insn))
1748 REG_NOTES (insn) = gen_rtx_INSN_LIST (REG_LABEL, XEXP (x, 0),
1750 if (LABEL_P (XEXP (x, 0)))
1751 LABEL_NUSES (XEXP (x, 0))++;
1755 fmt = GET_RTX_FORMAT (code);
1756 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1759 add_label_notes (XEXP (x, i), insns);
1760 else if (fmt[i] == 'E')
1761 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1762 add_label_notes (XVECEXP (x, i, j), insns);
1766 /* Scan MOVABLES, and move the insns that deserve to be moved.
1767 If two matching movables are combined, replace one reg with the
1768 other throughout. */
1771 move_movables (struct loop *loop, struct loop_movables *movables,
1772 int threshold, int insn_count)
1774 struct loop_regs *regs = LOOP_REGS (loop);
1775 int nregs = regs->num;
1779 rtx loop_start = loop->start;
1780 rtx loop_end = loop->end;
1781 /* Map of pseudo-register replacements to handle combining
1782 when we move several insns that load the same value
1783 into different pseudo-registers. */
1784 rtx *reg_map = xcalloc (nregs, sizeof (rtx));
1785 char *already_moved = xcalloc (nregs, sizeof (char));
1787 for (m = movables->head; m; m = m->next)
1789 /* Describe this movable insn. */
1791 if (loop_dump_stream)
1793 fprintf (loop_dump_stream, "Insn %d: regno %d (life %d), ",
1794 INSN_UID (m->insn), m->regno, m->lifetime);
1796 fprintf (loop_dump_stream, "consec %d, ", m->consec);
1798 fprintf (loop_dump_stream, "cond ");
1800 fprintf (loop_dump_stream, "force ");
1802 fprintf (loop_dump_stream, "global ");
1804 fprintf (loop_dump_stream, "done ");
1806 fprintf (loop_dump_stream, "move-insn ");
1808 fprintf (loop_dump_stream, "matches %d ",
1809 INSN_UID (m->match->insn));
1811 fprintf (loop_dump_stream, "forces %d ",
1812 INSN_UID (m->forces->insn));
1815 /* Ignore the insn if it's already done (it matched something else).
1816 Otherwise, see if it is now safe to move. */
1820 || (1 == loop_invariant_p (loop, m->set_src)
1821 && (m->dependencies == 0
1822 || 1 == loop_invariant_p (loop, m->dependencies))
1824 || 1 == consec_sets_invariant_p (loop, m->set_dest,
1827 && (! m->forces || m->forces->done))
1831 int savings = m->savings;
1833 /* We have an insn that is safe to move.
1834 Compute its desirability. */
1839 if (loop_dump_stream)
1840 fprintf (loop_dump_stream, "savings %d ", savings);
1842 if (regs->array[regno].moved_once && loop_dump_stream)
1843 fprintf (loop_dump_stream, "halved since already moved ");
1845 /* An insn MUST be moved if we already moved something else
1846 which is safe only if this one is moved too: that is,
1847 if already_moved[REGNO] is nonzero. */
1849 /* An insn is desirable to move if the new lifetime of the
1850 register is no more than THRESHOLD times the old lifetime.
1851 If it's not desirable, it means the loop is so big
1852 that moving won't speed things up much,
1853 and it is liable to make register usage worse. */
1855 /* It is also desirable to move if it can be moved at no
1856 extra cost because something else was already moved. */
1858 if (already_moved[regno]
1859 || flag_move_all_movables
1860 || (threshold * savings * m->lifetime) >=
1861 (regs->array[regno].moved_once ? insn_count * 2 : insn_count)
1862 || (m->forces && m->forces->done
1863 && regs->array[m->forces->regno].n_times_set == 1))
1867 rtx first = NULL_RTX;
1868 rtx newreg = NULL_RTX;
1871 newreg = gen_reg_rtx (GET_MODE (m->set_dest));
1873 /* Now move the insns that set the reg. */
1875 if (m->partial && m->match)
1879 /* Find the end of this chain of matching regs.
1880 Thus, we load each reg in the chain from that one reg.
1881 And that reg is loaded with 0 directly,
1882 since it has ->match == 0. */
1883 for (m1 = m; m1->match; m1 = m1->match);
1884 newpat = gen_move_insn (SET_DEST (PATTERN (m->insn)),
1885 SET_DEST (PATTERN (m1->insn)));
1886 i1 = loop_insn_hoist (loop, newpat);
1888 /* Mark the moved, invariant reg as being allowed to
1889 share a hard reg with the other matching invariant. */
1890 REG_NOTES (i1) = REG_NOTES (m->insn);
1891 r1 = SET_DEST (PATTERN (m->insn));
1892 r2 = SET_DEST (PATTERN (m1->insn));
1894 = gen_rtx_EXPR_LIST (VOIDmode, r1,
1895 gen_rtx_EXPR_LIST (VOIDmode, r2,
1897 delete_insn (m->insn);
1902 if (loop_dump_stream)
1903 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
1905 /* If we are to re-generate the item being moved with a
1906 new move insn, first delete what we have and then emit
1907 the move insn before the loop. */
1908 else if (m->move_insn)
1912 for (count = m->consec; count >= 0; count--)
1914 /* If this is the first insn of a library call sequence,
1915 something is very wrong. */
1916 if (GET_CODE (p) != NOTE
1917 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
1920 /* If this is the last insn of a libcall sequence, then
1921 delete every insn in the sequence except the last.
1922 The last insn is handled in the normal manner. */
1923 if (GET_CODE (p) != NOTE
1924 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
1926 temp = XEXP (temp, 0);
1928 temp = delete_insn (temp);
1932 p = delete_insn (p);
1934 /* simplify_giv_expr expects that it can walk the insns
1935 at m->insn forwards and see this old sequence we are
1936 tossing here. delete_insn does preserve the next
1937 pointers, but when we skip over a NOTE we must fix
1938 it up. Otherwise that code walks into the non-deleted
1940 while (p && GET_CODE (p) == NOTE)
1941 p = NEXT_INSN (temp) = NEXT_INSN (p);
1945 /* Replace the original insn with a move from
1946 our newly created temp. */
1948 emit_move_insn (m->set_dest, newreg);
1951 emit_insn_before (seq, p);
1956 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
1961 add_label_notes (m->set_src, seq);
1963 i1 = loop_insn_hoist (loop, seq);
1964 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
1965 set_unique_reg_note (i1,
1966 m->is_equiv ? REG_EQUIV : REG_EQUAL,
1969 if (loop_dump_stream)
1970 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
1972 /* The more regs we move, the less we like moving them. */
1977 for (count = m->consec; count >= 0; count--)
1981 /* If first insn of libcall sequence, skip to end. */
1982 /* Do this at start of loop, since p is guaranteed to
1984 if (GET_CODE (p) != NOTE
1985 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
1988 /* If last insn of libcall sequence, move all
1989 insns except the last before the loop. The last
1990 insn is handled in the normal manner. */
1991 if (GET_CODE (p) != NOTE
1992 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
1996 rtx fn_address_insn = 0;
1999 for (temp = XEXP (temp, 0); temp != p;
2000 temp = NEXT_INSN (temp))
2006 if (GET_CODE (temp) == NOTE)
2009 body = PATTERN (temp);
2011 /* Find the next insn after TEMP,
2012 not counting USE or NOTE insns. */
2013 for (next = NEXT_INSN (temp); next != p;
2014 next = NEXT_INSN (next))
2015 if (! (GET_CODE (next) == INSN
2016 && GET_CODE (PATTERN (next)) == USE)
2017 && GET_CODE (next) != NOTE)
2020 /* If that is the call, this may be the insn
2021 that loads the function address.
2023 Extract the function address from the insn
2024 that loads it into a register.
2025 If this insn was cse'd, we get incorrect code.
2027 So emit a new move insn that copies the
2028 function address into the register that the
2029 call insn will use. flow.c will delete any
2030 redundant stores that we have created. */
2031 if (GET_CODE (next) == CALL_INSN
2032 && GET_CODE (body) == SET
2033 && GET_CODE (SET_DEST (body)) == REG
2034 && (n = find_reg_note (temp, REG_EQUAL,
2037 fn_reg = SET_SRC (body);
2038 if (GET_CODE (fn_reg) != REG)
2039 fn_reg = SET_DEST (body);
2040 fn_address = XEXP (n, 0);
2041 fn_address_insn = temp;
2043 /* We have the call insn.
2044 If it uses the register we suspect it might,
2045 load it with the correct address directly. */
2046 if (GET_CODE (temp) == CALL_INSN
2048 && reg_referenced_p (fn_reg, body))
2049 loop_insn_emit_after (loop, 0, fn_address_insn,
2051 (fn_reg, fn_address));
2053 if (GET_CODE (temp) == CALL_INSN)
2055 i1 = loop_call_insn_hoist (loop, body);
2056 /* Because the USAGE information potentially
2057 contains objects other than hard registers
2058 we need to copy it. */
2059 if (CALL_INSN_FUNCTION_USAGE (temp))
2060 CALL_INSN_FUNCTION_USAGE (i1)
2061 = copy_rtx (CALL_INSN_FUNCTION_USAGE (temp));
2064 i1 = loop_insn_hoist (loop, body);
2067 if (temp == fn_address_insn)
2068 fn_address_insn = i1;
2069 REG_NOTES (i1) = REG_NOTES (temp);
2070 REG_NOTES (temp) = NULL;
2076 if (m->savemode != VOIDmode)
2078 /* P sets REG to zero; but we should clear only
2079 the bits that are not covered by the mode
2081 rtx reg = m->set_dest;
2086 tem = expand_simple_binop
2087 (GET_MODE (reg), AND, reg,
2088 GEN_INT ((((HOST_WIDE_INT) 1
2089 << GET_MODE_BITSIZE (m->savemode)))
2091 reg, 1, OPTAB_LIB_WIDEN);
2095 emit_move_insn (reg, tem);
2096 sequence = get_insns ();
2098 i1 = loop_insn_hoist (loop, sequence);
2100 else if (GET_CODE (p) == CALL_INSN)
2102 i1 = loop_call_insn_hoist (loop, PATTERN (p));
2103 /* Because the USAGE information potentially
2104 contains objects other than hard registers
2105 we need to copy it. */
2106 if (CALL_INSN_FUNCTION_USAGE (p))
2107 CALL_INSN_FUNCTION_USAGE (i1)
2108 = copy_rtx (CALL_INSN_FUNCTION_USAGE (p));
2110 else if (count == m->consec && m->move_insn_first)
2113 /* The SET_SRC might not be invariant, so we must
2114 use the REG_EQUAL note. */
2116 emit_move_insn (m->set_dest, m->set_src);
2120 add_label_notes (m->set_src, seq);
2122 i1 = loop_insn_hoist (loop, seq);
2123 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2124 set_unique_reg_note (i1, m->is_equiv ? REG_EQUIV
2125 : REG_EQUAL, m->set_src);
2127 else if (m->insert_temp)
2129 rtx *reg_map2 = xcalloc (REGNO (newreg),
2131 reg_map2 [m->regno] = newreg;
2133 i1 = loop_insn_hoist (loop, copy_rtx (PATTERN (p)));
2134 replace_regs (i1, reg_map2, REGNO (newreg), 1);
2138 i1 = loop_insn_hoist (loop, PATTERN (p));
2140 if (REG_NOTES (i1) == 0)
2142 REG_NOTES (i1) = REG_NOTES (p);
2143 REG_NOTES (p) = NULL;
2145 /* If there is a REG_EQUAL note present whose value
2146 is not loop invariant, then delete it, since it
2147 may cause problems with later optimization passes.
2148 It is possible for cse to create such notes
2149 like this as a result of record_jump_cond. */
2151 if ((temp = find_reg_note (i1, REG_EQUAL, NULL_RTX))
2152 && ! loop_invariant_p (loop, XEXP (temp, 0)))
2153 remove_note (i1, temp);
2159 if (loop_dump_stream)
2160 fprintf (loop_dump_stream, " moved to %d",
2163 /* If library call, now fix the REG_NOTES that contain
2164 insn pointers, namely REG_LIBCALL on FIRST
2165 and REG_RETVAL on I1. */
2166 if ((temp = find_reg_note (i1, REG_RETVAL, NULL_RTX)))
2168 XEXP (temp, 0) = first;
2169 temp = find_reg_note (first, REG_LIBCALL, NULL_RTX);
2170 XEXP (temp, 0) = i1;
2177 /* simplify_giv_expr expects that it can walk the insns
2178 at m->insn forwards and see this old sequence we are
2179 tossing here. delete_insn does preserve the next
2180 pointers, but when we skip over a NOTE we must fix
2181 it up. Otherwise that code walks into the non-deleted
2183 while (p && GET_CODE (p) == NOTE)
2184 p = NEXT_INSN (temp) = NEXT_INSN (p);
2189 /* Replace the original insn with a move from
2190 our newly created temp. */
2192 emit_move_insn (m->set_dest, newreg);
2195 emit_insn_before (seq, p);
2199 /* The more regs we move, the less we like moving them. */
2205 if (!m->insert_temp)
2207 /* Any other movable that loads the same register
2209 already_moved[regno] = 1;
2211 /* This reg has been moved out of one loop. */
2212 regs->array[regno].moved_once = 1;
2214 /* The reg set here is now invariant. */
2218 for (i = 0; i < LOOP_REGNO_NREGS (regno, m->set_dest); i++)
2219 regs->array[regno+i].set_in_loop = 0;
2222 /* Change the length-of-life info for the register
2223 to say it lives at least the full length of this loop.
2224 This will help guide optimizations in outer loops. */
2226 if (REGNO_FIRST_LUID (regno) > INSN_LUID (loop_start))
2227 /* This is the old insn before all the moved insns.
2228 We can't use the moved insn because it is out of range
2229 in uid_luid. Only the old insns have luids. */
2230 REGNO_FIRST_UID (regno) = INSN_UID (loop_start);
2231 if (REGNO_LAST_LUID (regno) < INSN_LUID (loop_end))
2232 REGNO_LAST_UID (regno) = INSN_UID (loop_end);
2235 /* Combine with this moved insn any other matching movables. */
2238 for (m1 = movables->head; m1; m1 = m1->next)
2243 /* Schedule the reg loaded by M1
2244 for replacement so that shares the reg of M.
2245 If the modes differ (only possible in restricted
2246 circumstances, make a SUBREG.
2248 Note this assumes that the target dependent files
2249 treat REG and SUBREG equally, including within
2250 GO_IF_LEGITIMATE_ADDRESS and in all the
2251 predicates since we never verify that replacing the
2252 original register with a SUBREG results in a
2253 recognizable insn. */
2254 if (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest))
2255 reg_map[m1->regno] = m->set_dest;
2258 = gen_lowpart_common (GET_MODE (m1->set_dest),
2261 /* Get rid of the matching insn
2262 and prevent further processing of it. */
2265 /* If library call, delete all insns. */
2266 if ((temp = find_reg_note (m1->insn, REG_RETVAL,
2268 delete_insn_chain (XEXP (temp, 0), m1->insn);
2270 delete_insn (m1->insn);
2272 /* Any other movable that loads the same register
2274 already_moved[m1->regno] = 1;
2276 /* The reg merged here is now invariant,
2277 if the reg it matches is invariant. */
2282 i < LOOP_REGNO_NREGS (regno, m1->set_dest);
2284 regs->array[m1->regno+i].set_in_loop = 0;
2288 else if (loop_dump_stream)
2289 fprintf (loop_dump_stream, "not desirable");
2291 else if (loop_dump_stream && !m->match)
2292 fprintf (loop_dump_stream, "not safe");
2294 if (loop_dump_stream)
2295 fprintf (loop_dump_stream, "\n");
2299 new_start = loop_start;
2301 /* Go through all the instructions in the loop, making
2302 all the register substitutions scheduled in REG_MAP. */
2303 for (p = new_start; p != loop_end; p = NEXT_INSN (p))
2304 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
2305 || GET_CODE (p) == CALL_INSN)
2307 replace_regs (PATTERN (p), reg_map, nregs, 0);
2308 replace_regs (REG_NOTES (p), reg_map, nregs, 0);
2314 free (already_moved);
2319 loop_movables_add (struct loop_movables *movables, struct movable *m)
2321 if (movables->head == 0)
2324 movables->last->next = m;
2330 loop_movables_free (struct loop_movables *movables)
2333 struct movable *m_next;
2335 for (m = movables->head; m; m = m_next)
2343 /* Scan X and replace the address of any MEM in it with ADDR.
2344 REG is the address that MEM should have before the replacement. */
2347 replace_call_address (rtx x, rtx reg, rtx addr)
2355 code = GET_CODE (x);
2369 /* Short cut for very common case. */
2370 replace_call_address (XEXP (x, 1), reg, addr);
2374 /* Short cut for very common case. */
2375 replace_call_address (XEXP (x, 0), reg, addr);
2379 /* If this MEM uses a reg other than the one we expected,
2380 something is wrong. */
2381 if (XEXP (x, 0) != reg)
2390 fmt = GET_RTX_FORMAT (code);
2391 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2394 replace_call_address (XEXP (x, i), reg, addr);
2395 else if (fmt[i] == 'E')
2398 for (j = 0; j < XVECLEN (x, i); j++)
2399 replace_call_address (XVECEXP (x, i, j), reg, addr);
2405 /* Return the number of memory refs to addresses that vary
2409 count_nonfixed_reads (const struct loop *loop, rtx x)
2419 code = GET_CODE (x);
2433 return ((loop_invariant_p (loop, XEXP (x, 0)) != 1)
2434 + count_nonfixed_reads (loop, XEXP (x, 0)));
2441 fmt = GET_RTX_FORMAT (code);
2442 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2445 value += count_nonfixed_reads (loop, XEXP (x, i));
2449 for (j = 0; j < XVECLEN (x, i); j++)
2450 value += count_nonfixed_reads (loop, XVECEXP (x, i, j));
2456 /* Scan a loop setting the elements `cont', `vtop', `loops_enclosed',
2457 `has_call', `has_nonconst_call', `has_volatile', `has_tablejump',
2458 `unknown_address_altered', `unknown_constant_address_altered', and
2459 `num_mem_sets' in LOOP. Also, fill in the array `mems' and the
2460 list `store_mems' in LOOP. */
2463 prescan_loop (struct loop *loop)
2467 struct loop_info *loop_info = LOOP_INFO (loop);
2468 rtx start = loop->start;
2469 rtx end = loop->end;
2470 /* The label after END. Jumping here is just like falling off the
2471 end of the loop. We use next_nonnote_insn instead of next_label
2472 as a hedge against the (pathological) case where some actual insn
2473 might end up between the two. */
2474 rtx exit_target = next_nonnote_insn (end);
2476 loop_info->has_indirect_jump = indirect_jump_in_function;
2477 loop_info->pre_header_has_call = 0;
2478 loop_info->has_call = 0;
2479 loop_info->has_nonconst_call = 0;
2480 loop_info->has_prefetch = 0;
2481 loop_info->has_volatile = 0;
2482 loop_info->has_tablejump = 0;
2483 loop_info->has_multiple_exit_targets = 0;
2486 loop_info->unknown_address_altered = 0;
2487 loop_info->unknown_constant_address_altered = 0;
2488 loop_info->store_mems = NULL_RTX;
2489 loop_info->first_loop_store_insn = NULL_RTX;
2490 loop_info->mems_idx = 0;
2491 loop_info->num_mem_sets = 0;
2492 /* If loop opts run twice, this was set on 1st pass for 2nd. */
2493 loop_info->preconditioned = NOTE_PRECONDITIONED (end);
2495 for (insn = start; insn && GET_CODE (insn) != CODE_LABEL;
2496 insn = PREV_INSN (insn))
2498 if (GET_CODE (insn) == CALL_INSN)
2500 loop_info->pre_header_has_call = 1;
2505 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2506 insn = NEXT_INSN (insn))
2508 switch (GET_CODE (insn))
2511 if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
2514 /* Count number of loops contained in this one. */
2517 else if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_END)
2522 if (! CONST_OR_PURE_CALL_P (insn))
2524 loop_info->unknown_address_altered = 1;
2525 loop_info->has_nonconst_call = 1;
2527 else if (pure_call_p (insn))
2528 loop_info->has_nonconst_call = 1;
2529 loop_info->has_call = 1;
2530 if (can_throw_internal (insn))
2531 loop_info->has_multiple_exit_targets = 1;
2533 /* Calls initializing constant objects have CLOBBER of MEM /u in the
2534 attached FUNCTION_USAGE expression list, not accounted for by the
2535 code above. We should note these to avoid missing dependencies in
2536 later references. */
2540 for (fusage_entry = CALL_INSN_FUNCTION_USAGE (insn);
2541 fusage_entry; fusage_entry = XEXP (fusage_entry, 1))
2543 rtx fusage = XEXP (fusage_entry, 0);
2545 if (GET_CODE (fusage) == CLOBBER
2546 && GET_CODE (XEXP (fusage, 0)) == MEM
2547 && RTX_UNCHANGING_P (XEXP (fusage, 0)))
2549 note_stores (fusage, note_addr_stored, loop_info);
2550 if (! loop_info->first_loop_store_insn
2551 && loop_info->store_mems)
2552 loop_info->first_loop_store_insn = insn;
2559 if (! loop_info->has_multiple_exit_targets)
2561 rtx set = pc_set (insn);
2565 rtx src = SET_SRC (set);
2568 if (GET_CODE (src) == IF_THEN_ELSE)
2570 label1 = XEXP (src, 1);
2571 label2 = XEXP (src, 2);
2581 if (label1 && label1 != pc_rtx)
2583 if (GET_CODE (label1) != LABEL_REF)
2585 /* Something tricky. */
2586 loop_info->has_multiple_exit_targets = 1;
2589 else if (XEXP (label1, 0) != exit_target
2590 && LABEL_OUTSIDE_LOOP_P (label1))
2592 /* A jump outside the current loop. */
2593 loop_info->has_multiple_exit_targets = 1;
2605 /* A return, or something tricky. */
2606 loop_info->has_multiple_exit_targets = 1;
2612 if (volatile_refs_p (PATTERN (insn)))
2613 loop_info->has_volatile = 1;
2615 if (GET_CODE (insn) == JUMP_INSN
2616 && (GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC
2617 || GET_CODE (PATTERN (insn)) == ADDR_VEC))
2618 loop_info->has_tablejump = 1;
2620 note_stores (PATTERN (insn), note_addr_stored, loop_info);
2621 if (! loop_info->first_loop_store_insn && loop_info->store_mems)
2622 loop_info->first_loop_store_insn = insn;
2624 if (flag_non_call_exceptions && can_throw_internal (insn))
2625 loop_info->has_multiple_exit_targets = 1;
2633 /* Now, rescan the loop, setting up the LOOP_MEMS array. */
2634 if (/* An exception thrown by a called function might land us
2636 ! loop_info->has_nonconst_call
2637 /* We don't want loads for MEMs moved to a location before the
2638 one at which their stack memory becomes allocated. (Note
2639 that this is not a problem for malloc, etc., since those
2640 require actual function calls. */
2641 && ! current_function_calls_alloca
2642 /* There are ways to leave the loop other than falling off the
2644 && ! loop_info->has_multiple_exit_targets)
2645 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2646 insn = NEXT_INSN (insn))
2647 for_each_rtx (&insn, insert_loop_mem, loop_info);
2649 /* BLKmode MEMs are added to LOOP_STORE_MEM as necessary so
2650 that loop_invariant_p and load_mems can use true_dependence
2651 to determine what is really clobbered. */
2652 if (loop_info->unknown_address_altered)
2654 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2656 loop_info->store_mems
2657 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2659 if (loop_info->unknown_constant_address_altered)
2661 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2663 RTX_UNCHANGING_P (mem) = 1;
2664 loop_info->store_mems
2665 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2669 /* Invalidate all loops containing LABEL. */
2672 invalidate_loops_containing_label (rtx label)
2675 for (loop = uid_loop[INSN_UID (label)]; loop; loop = loop->outer)
2679 /* Scan the function looking for loops. Record the start and end of each loop.
2680 Also mark as invalid loops any loops that contain a setjmp or are branched
2681 to from outside the loop. */
2684 find_and_verify_loops (rtx f, struct loops *loops)
2689 struct loop *current_loop;
2690 struct loop *next_loop;
2693 num_loops = loops->num;
2695 compute_luids (f, NULL_RTX, 0);
2697 /* If there are jumps to undefined labels,
2698 treat them as jumps out of any/all loops.
2699 This also avoids writing past end of tables when there are no loops. */
2702 /* Find boundaries of loops, mark which loops are contained within
2703 loops, and invalidate loops that have setjmp. */
2706 current_loop = NULL;
2707 for (insn = f; insn; insn = NEXT_INSN (insn))
2709 if (GET_CODE (insn) == NOTE)
2710 switch (NOTE_LINE_NUMBER (insn))
2712 case NOTE_INSN_LOOP_BEG:
2713 next_loop = loops->array + num_loops;
2714 next_loop->num = num_loops;
2716 next_loop->start = insn;
2717 next_loop->outer = current_loop;
2718 current_loop = next_loop;
2721 case NOTE_INSN_LOOP_CONT:
2722 current_loop->cont = insn;
2725 case NOTE_INSN_LOOP_VTOP:
2726 current_loop->vtop = insn;
2729 case NOTE_INSN_LOOP_END:
2733 current_loop->end = insn;
2734 current_loop = current_loop->outer;
2741 if (GET_CODE (insn) == CALL_INSN
2742 && find_reg_note (insn, REG_SETJMP, NULL))
2744 /* In this case, we must invalidate our current loop and any
2746 for (loop = current_loop; loop; loop = loop->outer)
2749 if (loop_dump_stream)
2750 fprintf (loop_dump_stream,
2751 "\nLoop at %d ignored due to setjmp.\n",
2752 INSN_UID (loop->start));
2756 /* Note that this will mark the NOTE_INSN_LOOP_END note as being in the
2757 enclosing loop, but this doesn't matter. */
2758 uid_loop[INSN_UID (insn)] = current_loop;
2761 /* Any loop containing a label used in an initializer must be invalidated,
2762 because it can be jumped into from anywhere. */
2763 for (label = forced_labels; label; label = XEXP (label, 1))
2764 invalidate_loops_containing_label (XEXP (label, 0));
2766 /* Any loop containing a label used for an exception handler must be
2767 invalidated, because it can be jumped into from anywhere. */
2768 for_each_eh_label (invalidate_loops_containing_label);
2770 /* Now scan all insn's in the function. If any JUMP_INSN branches into a
2771 loop that it is not contained within, that loop is marked invalid.
2772 If any INSN or CALL_INSN uses a label's address, then the loop containing
2773 that label is marked invalid, because it could be jumped into from
2776 Also look for blocks of code ending in an unconditional branch that
2777 exits the loop. If such a block is surrounded by a conditional
2778 branch around the block, move the block elsewhere (see below) and
2779 invert the jump to point to the code block. This may eliminate a
2780 label in our loop and will simplify processing by both us and a
2781 possible second cse pass. */
2783 for (insn = f; insn; insn = NEXT_INSN (insn))
2786 struct loop *this_loop = uid_loop[INSN_UID (insn)];
2788 if (GET_CODE (insn) == INSN || GET_CODE (insn) == CALL_INSN)
2790 rtx note = find_reg_note (insn, REG_LABEL, NULL_RTX);
2792 invalidate_loops_containing_label (XEXP (note, 0));
2795 if (GET_CODE (insn) != JUMP_INSN)
2798 mark_loop_jump (PATTERN (insn), this_loop);
2800 /* See if this is an unconditional branch outside the loop. */
2802 && (GET_CODE (PATTERN (insn)) == RETURN
2803 || (any_uncondjump_p (insn)
2804 && onlyjump_p (insn)
2805 && (uid_loop[INSN_UID (JUMP_LABEL (insn))]
2807 && get_max_uid () < max_uid_for_loop)
2810 rtx our_next = next_real_insn (insn);
2811 rtx last_insn_to_move = NEXT_INSN (insn);
2812 struct loop *dest_loop;
2813 struct loop *outer_loop = NULL;
2815 /* Go backwards until we reach the start of the loop, a label,
2817 for (p = PREV_INSN (insn);
2818 GET_CODE (p) != CODE_LABEL
2819 && ! (GET_CODE (p) == NOTE
2820 && NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
2821 && GET_CODE (p) != JUMP_INSN;
2825 /* Check for the case where we have a jump to an inner nested
2826 loop, and do not perform the optimization in that case. */
2828 if (JUMP_LABEL (insn))
2830 dest_loop = uid_loop[INSN_UID (JUMP_LABEL (insn))];
2833 for (outer_loop = dest_loop; outer_loop;
2834 outer_loop = outer_loop->outer)
2835 if (outer_loop == this_loop)
2840 /* Make sure that the target of P is within the current loop. */
2842 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
2843 && uid_loop[INSN_UID (JUMP_LABEL (p))] != this_loop)
2844 outer_loop = this_loop;
2846 /* If we stopped on a JUMP_INSN to the next insn after INSN,
2847 we have a block of code to try to move.
2849 We look backward and then forward from the target of INSN
2850 to find a BARRIER at the same loop depth as the target.
2851 If we find such a BARRIER, we make a new label for the start
2852 of the block, invert the jump in P and point it to that label,
2853 and move the block of code to the spot we found. */
2856 && GET_CODE (p) == JUMP_INSN
2857 && JUMP_LABEL (p) != 0
2858 /* Just ignore jumps to labels that were never emitted.
2859 These always indicate compilation errors. */
2860 && INSN_UID (JUMP_LABEL (p)) != 0
2861 && any_condjump_p (p) && onlyjump_p (p)
2862 && next_real_insn (JUMP_LABEL (p)) == our_next
2863 /* If it's not safe to move the sequence, then we
2865 && insns_safe_to_move_p (p, NEXT_INSN (insn),
2866 &last_insn_to_move))
2869 = JUMP_LABEL (insn) ? JUMP_LABEL (insn) : get_last_insn ();
2870 struct loop *target_loop = uid_loop[INSN_UID (target)];
2874 /* Search for possible garbage past the conditional jumps
2875 and look for the last barrier. */
2876 for (tmp = last_insn_to_move;
2877 tmp && GET_CODE (tmp) != CODE_LABEL; tmp = NEXT_INSN (tmp))
2878 if (GET_CODE (tmp) == BARRIER)
2879 last_insn_to_move = tmp;
2881 for (loc = target; loc; loc = PREV_INSN (loc))
2882 if (GET_CODE (loc) == BARRIER
2883 /* Don't move things inside a tablejump. */
2884 && ((loc2 = next_nonnote_insn (loc)) == 0
2885 || GET_CODE (loc2) != CODE_LABEL
2886 || (loc2 = next_nonnote_insn (loc2)) == 0
2887 || GET_CODE (loc2) != JUMP_INSN
2888 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2889 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2890 && uid_loop[INSN_UID (loc)] == target_loop)
2894 for (loc = target; loc; loc = NEXT_INSN (loc))
2895 if (GET_CODE (loc) == BARRIER
2896 /* Don't move things inside a tablejump. */
2897 && ((loc2 = next_nonnote_insn (loc)) == 0
2898 || GET_CODE (loc2) != CODE_LABEL
2899 || (loc2 = next_nonnote_insn (loc2)) == 0
2900 || GET_CODE (loc2) != JUMP_INSN
2901 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2902 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2903 && uid_loop[INSN_UID (loc)] == target_loop)
2908 rtx cond_label = JUMP_LABEL (p);
2909 rtx new_label = get_label_after (p);
2911 /* Ensure our label doesn't go away. */
2912 LABEL_NUSES (cond_label)++;
2914 /* Verify that uid_loop is large enough and that
2916 if (invert_jump (p, new_label, 1))
2920 /* If no suitable BARRIER was found, create a suitable
2921 one before TARGET. Since TARGET is a fall through
2922 path, we'll need to insert a jump around our block
2923 and add a BARRIER before TARGET.
2925 This creates an extra unconditional jump outside
2926 the loop. However, the benefits of removing rarely
2927 executed instructions from inside the loop usually
2928 outweighs the cost of the extra unconditional jump
2929 outside the loop. */
2934 temp = gen_jump (JUMP_LABEL (insn));
2935 temp = emit_jump_insn_before (temp, target);
2936 JUMP_LABEL (temp) = JUMP_LABEL (insn);
2937 LABEL_NUSES (JUMP_LABEL (insn))++;
2938 loc = emit_barrier_before (target);
2941 /* Include the BARRIER after INSN and copy the
2943 if (squeeze_notes (&new_label, &last_insn_to_move))
2945 reorder_insns (new_label, last_insn_to_move, loc);
2947 /* All those insns are now in TARGET_LOOP. */
2949 q != NEXT_INSN (last_insn_to_move);
2951 uid_loop[INSN_UID (q)] = target_loop;
2953 /* The label jumped to by INSN is no longer a loop
2954 exit. Unless INSN does not have a label (e.g.,
2955 it is a RETURN insn), search loop->exit_labels
2956 to find its label_ref, and remove it. Also turn
2957 off LABEL_OUTSIDE_LOOP_P bit. */
2958 if (JUMP_LABEL (insn))
2960 for (q = 0, r = this_loop->exit_labels;
2962 q = r, r = LABEL_NEXTREF (r))
2963 if (XEXP (r, 0) == JUMP_LABEL (insn))
2965 LABEL_OUTSIDE_LOOP_P (r) = 0;
2967 LABEL_NEXTREF (q) = LABEL_NEXTREF (r);
2969 this_loop->exit_labels = LABEL_NEXTREF (r);
2973 for (loop = this_loop; loop && loop != target_loop;
2977 /* If we didn't find it, then something is
2983 /* P is now a jump outside the loop, so it must be put
2984 in loop->exit_labels, and marked as such.
2985 The easiest way to do this is to just call
2986 mark_loop_jump again for P. */
2987 mark_loop_jump (PATTERN (p), this_loop);
2989 /* If INSN now jumps to the insn after it,
2991 if (JUMP_LABEL (insn) != 0
2992 && (next_real_insn (JUMP_LABEL (insn))
2993 == next_real_insn (insn)))
2994 delete_related_insns (insn);
2997 /* Continue the loop after where the conditional
2998 branch used to jump, since the only branch insn
2999 in the block (if it still remains) is an inter-loop
3000 branch and hence needs no processing. */
3001 insn = NEXT_INSN (cond_label);
3003 if (--LABEL_NUSES (cond_label) == 0)
3004 delete_related_insns (cond_label);
3006 /* This loop will be continued with NEXT_INSN (insn). */
3007 insn = PREV_INSN (insn);
3014 /* If any label in X jumps to a loop different from LOOP_NUM and any of the
3015 loops it is contained in, mark the target loop invalid.
3017 For speed, we assume that X is part of a pattern of a JUMP_INSN. */
3020 mark_loop_jump (rtx x, struct loop *loop)
3022 struct loop *dest_loop;
3023 struct loop *outer_loop;
3026 switch (GET_CODE (x))
3039 /* There could be a label reference in here. */
3040 mark_loop_jump (XEXP (x, 0), loop);
3046 mark_loop_jump (XEXP (x, 0), loop);
3047 mark_loop_jump (XEXP (x, 1), loop);
3051 /* This may refer to a LABEL_REF or SYMBOL_REF. */
3052 mark_loop_jump (XEXP (x, 1), loop);
3057 mark_loop_jump (XEXP (x, 0), loop);
3061 dest_loop = uid_loop[INSN_UID (XEXP (x, 0))];
3063 /* Link together all labels that branch outside the loop. This
3064 is used by final_[bg]iv_value and the loop unrolling code. Also
3065 mark this LABEL_REF so we know that this branch should predict
3068 /* A check to make sure the label is not in an inner nested loop,
3069 since this does not count as a loop exit. */
3072 for (outer_loop = dest_loop; outer_loop;
3073 outer_loop = outer_loop->outer)
3074 if (outer_loop == loop)
3080 if (loop && ! outer_loop)
3082 LABEL_OUTSIDE_LOOP_P (x) = 1;
3083 LABEL_NEXTREF (x) = loop->exit_labels;
3084 loop->exit_labels = x;
3086 for (outer_loop = loop;
3087 outer_loop && outer_loop != dest_loop;
3088 outer_loop = outer_loop->outer)
3089 outer_loop->exit_count++;
3092 /* If this is inside a loop, but not in the current loop or one enclosed
3093 by it, it invalidates at least one loop. */
3098 /* We must invalidate every nested loop containing the target of this
3099 label, except those that also contain the jump insn. */
3101 for (; dest_loop; dest_loop = dest_loop->outer)
3103 /* Stop when we reach a loop that also contains the jump insn. */
3104 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3105 if (dest_loop == outer_loop)
3108 /* If we get here, we know we need to invalidate a loop. */
3109 if (loop_dump_stream && ! dest_loop->invalid)
3110 fprintf (loop_dump_stream,
3111 "\nLoop at %d ignored due to multiple entry points.\n",
3112 INSN_UID (dest_loop->start));
3114 dest_loop->invalid = 1;
3119 /* If this is not setting pc, ignore. */
3120 if (SET_DEST (x) == pc_rtx)
3121 mark_loop_jump (SET_SRC (x), loop);
3125 mark_loop_jump (XEXP (x, 1), loop);
3126 mark_loop_jump (XEXP (x, 2), loop);
3131 for (i = 0; i < XVECLEN (x, 0); i++)
3132 mark_loop_jump (XVECEXP (x, 0, i), loop);
3136 for (i = 0; i < XVECLEN (x, 1); i++)
3137 mark_loop_jump (XVECEXP (x, 1, i), loop);
3141 /* Strictly speaking this is not a jump into the loop, only a possible
3142 jump out of the loop. However, we have no way to link the destination
3143 of this jump onto the list of exit labels. To be safe we mark this
3144 loop and any containing loops as invalid. */
3147 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3149 if (loop_dump_stream && ! outer_loop->invalid)
3150 fprintf (loop_dump_stream,
3151 "\nLoop at %d ignored due to unknown exit jump.\n",
3152 INSN_UID (outer_loop->start));
3153 outer_loop->invalid = 1;
3160 /* Return nonzero if there is a label in the range from
3161 insn INSN to and including the insn whose luid is END
3162 INSN must have an assigned luid (i.e., it must not have
3163 been previously created by loop.c). */
3166 labels_in_range_p (rtx insn, int end)
3168 while (insn && INSN_LUID (insn) <= end)
3170 if (GET_CODE (insn) == CODE_LABEL)
3172 insn = NEXT_INSN (insn);
3178 /* Record that a memory reference X is being set. */
3181 note_addr_stored (rtx x, rtx y ATTRIBUTE_UNUSED,
3182 void *data ATTRIBUTE_UNUSED)
3184 struct loop_info *loop_info = data;
3186 if (x == 0 || GET_CODE (x) != MEM)
3189 /* Count number of memory writes.
3190 This affects heuristics in strength_reduce. */
3191 loop_info->num_mem_sets++;
3193 /* BLKmode MEM means all memory is clobbered. */
3194 if (GET_MODE (x) == BLKmode)
3196 if (RTX_UNCHANGING_P (x))
3197 loop_info->unknown_constant_address_altered = 1;
3199 loop_info->unknown_address_altered = 1;
3204 loop_info->store_mems = gen_rtx_EXPR_LIST (VOIDmode, x,
3205 loop_info->store_mems);
3208 /* X is a value modified by an INSN that references a biv inside a loop
3209 exit test (ie, X is somehow related to the value of the biv). If X
3210 is a pseudo that is used more than once, then the biv is (effectively)
3211 used more than once. DATA is a pointer to a loop_regs structure. */
3214 note_set_pseudo_multiple_uses (rtx x, rtx y ATTRIBUTE_UNUSED, void *data)
3216 struct loop_regs *regs = (struct loop_regs *) data;
3221 while (GET_CODE (x) == STRICT_LOW_PART
3222 || GET_CODE (x) == SIGN_EXTRACT
3223 || GET_CODE (x) == ZERO_EXTRACT
3224 || GET_CODE (x) == SUBREG)
3227 if (GET_CODE (x) != REG || REGNO (x) < FIRST_PSEUDO_REGISTER)
3230 /* If we do not have usage information, or if we know the register
3231 is used more than once, note that fact for check_dbra_loop. */
3232 if (REGNO (x) >= max_reg_before_loop
3233 || ! regs->array[REGNO (x)].single_usage
3234 || regs->array[REGNO (x)].single_usage == const0_rtx)
3235 regs->multiple_uses = 1;
3238 /* Return nonzero if the rtx X is invariant over the current loop.
3240 The value is 2 if we refer to something only conditionally invariant.
3242 A memory ref is invariant if it is not volatile and does not conflict
3243 with anything stored in `loop_info->store_mems'. */
3246 loop_invariant_p (const struct loop *loop, rtx x)
3248 struct loop_info *loop_info = LOOP_INFO (loop);
3249 struct loop_regs *regs = LOOP_REGS (loop);
3253 int conditional = 0;
3258 code = GET_CODE (x);
3268 /* A LABEL_REF is normally invariant, however, if we are unrolling
3269 loops, and this label is inside the loop, then it isn't invariant.
3270 This is because each unrolled copy of the loop body will have
3271 a copy of this label. If this was invariant, then an insn loading
3272 the address of this label into a register might get moved outside
3273 the loop, and then each loop body would end up using the same label.
3275 We don't know the loop bounds here though, so just fail for all
3277 if (flag_old_unroll_loops)
3284 case UNSPEC_VOLATILE:
3288 /* We used to check RTX_UNCHANGING_P (x) here, but that is invalid
3289 since the reg might be set by initialization within the loop. */
3291 if ((x == frame_pointer_rtx || x == hard_frame_pointer_rtx
3292 || x == arg_pointer_rtx || x == pic_offset_table_rtx)
3293 && ! current_function_has_nonlocal_goto)
3296 if (LOOP_INFO (loop)->has_call
3297 && REGNO (x) < FIRST_PSEUDO_REGISTER && call_used_regs[REGNO (x)])
3300 /* Out-of-range regs can occur when we are called from unrolling.
3301 These registers created by the unroller are set in the loop,
3302 hence are never invariant.
3303 Other out-of-range regs can be generated by load_mems; those that
3304 are written to in the loop are not invariant, while those that are
3305 not written to are invariant. It would be easy for load_mems
3306 to set n_times_set correctly for these registers, however, there
3307 is no easy way to distinguish them from registers created by the
3310 if (REGNO (x) >= (unsigned) regs->num)
3313 if (regs->array[REGNO (x)].set_in_loop < 0)
3316 return regs->array[REGNO (x)].set_in_loop == 0;
3319 /* Volatile memory references must be rejected. Do this before
3320 checking for read-only items, so that volatile read-only items
3321 will be rejected also. */
3322 if (MEM_VOLATILE_P (x))
3325 /* See if there is any dependence between a store and this load. */
3326 mem_list_entry = loop_info->store_mems;
3327 while (mem_list_entry)
3329 if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
3333 mem_list_entry = XEXP (mem_list_entry, 1);
3336 /* It's not invalidated by a store in memory
3337 but we must still verify the address is invariant. */
3341 /* Don't mess with insns declared volatile. */
3342 if (MEM_VOLATILE_P (x))
3350 fmt = GET_RTX_FORMAT (code);
3351 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3355 int tem = loop_invariant_p (loop, XEXP (x, i));
3361 else if (fmt[i] == 'E')
3364 for (j = 0; j < XVECLEN (x, i); j++)
3366 int tem = loop_invariant_p (loop, XVECEXP (x, i, j));
3376 return 1 + conditional;
3379 /* Return nonzero if all the insns in the loop that set REG
3380 are INSN and the immediately following insns,
3381 and if each of those insns sets REG in an invariant way
3382 (not counting uses of REG in them).
3384 The value is 2 if some of these insns are only conditionally invariant.
3386 We assume that INSN itself is the first set of REG
3387 and that its source is invariant. */
3390 consec_sets_invariant_p (const struct loop *loop, rtx reg, int n_sets,
3393 struct loop_regs *regs = LOOP_REGS (loop);
3395 unsigned int regno = REGNO (reg);
3397 /* Number of sets we have to insist on finding after INSN. */
3398 int count = n_sets - 1;
3399 int old = regs->array[regno].set_in_loop;
3403 /* If N_SETS hit the limit, we can't rely on its value. */
3407 regs->array[regno].set_in_loop = 0;
3415 code = GET_CODE (p);
3417 /* If library call, skip to end of it. */
3418 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
3423 && (set = single_set (p))
3424 && GET_CODE (SET_DEST (set)) == REG
3425 && REGNO (SET_DEST (set)) == regno)
3427 this = loop_invariant_p (loop, SET_SRC (set));
3430 else if ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX)))
3432 /* If this is a libcall, then any invariant REG_EQUAL note is OK.
3433 If this is an ordinary insn, then only CONSTANT_P REG_EQUAL
3435 this = (CONSTANT_P (XEXP (temp, 0))
3436 || (find_reg_note (p, REG_RETVAL, NULL_RTX)
3437 && loop_invariant_p (loop, XEXP (temp, 0))));
3444 else if (code != NOTE)
3446 regs->array[regno].set_in_loop = old;
3451 regs->array[regno].set_in_loop = old;
3452 /* If loop_invariant_p ever returned 2, we return 2. */
3453 return 1 + (value & 2);
3457 /* I don't think this condition is sufficient to allow INSN
3458 to be moved, so we no longer test it. */
3460 /* Return 1 if all insns in the basic block of INSN and following INSN
3461 that set REG are invariant according to TABLE. */
3464 all_sets_invariant_p (rtx reg, rtx insn, short *table)
3467 int regno = REGNO (reg);
3473 code = GET_CODE (p);
3474 if (code == CODE_LABEL || code == JUMP_INSN)
3476 if (code == INSN && GET_CODE (PATTERN (p)) == SET
3477 && GET_CODE (SET_DEST (PATTERN (p))) == REG
3478 && REGNO (SET_DEST (PATTERN (p))) == regno)
3480 if (! loop_invariant_p (loop, SET_SRC (PATTERN (p)), table))
3487 /* Look at all uses (not sets) of registers in X. For each, if it is
3488 the single use, set USAGE[REGNO] to INSN; if there was a previous use in
3489 a different insn, set USAGE[REGNO] to const0_rtx. */
3492 find_single_use_in_loop (struct loop_regs *regs, rtx insn, rtx x)
3494 enum rtx_code code = GET_CODE (x);
3495 const char *fmt = GET_RTX_FORMAT (code);
3499 regs->array[REGNO (x)].single_usage
3500 = (regs->array[REGNO (x)].single_usage != 0
3501 && regs->array[REGNO (x)].single_usage != insn)
3502 ? const0_rtx : insn;
3504 else if (code == SET)
3506 /* Don't count SET_DEST if it is a REG; otherwise count things
3507 in SET_DEST because if a register is partially modified, it won't
3508 show up as a potential movable so we don't care how USAGE is set
3510 if (GET_CODE (SET_DEST (x)) != REG)
3511 find_single_use_in_loop (regs, insn, SET_DEST (x));
3512 find_single_use_in_loop (regs, insn, SET_SRC (x));
3515 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3517 if (fmt[i] == 'e' && XEXP (x, i) != 0)
3518 find_single_use_in_loop (regs, insn, XEXP (x, i));
3519 else if (fmt[i] == 'E')
3520 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
3521 find_single_use_in_loop (regs, insn, XVECEXP (x, i, j));
3525 /* Count and record any set in X which is contained in INSN. Update
3526 REGS->array[I].MAY_NOT_OPTIMIZE and LAST_SET for any register I set
3530 count_one_set (struct loop_regs *regs, rtx insn, rtx x, rtx *last_set)
3532 if (GET_CODE (x) == CLOBBER && GET_CODE (XEXP (x, 0)) == REG)
3533 /* Don't move a reg that has an explicit clobber.
3534 It's not worth the pain to try to do it correctly. */
3535 regs->array[REGNO (XEXP (x, 0))].may_not_optimize = 1;
3537 if (GET_CODE (x) == SET || GET_CODE (x) == CLOBBER)
3539 rtx dest = SET_DEST (x);
3540 while (GET_CODE (dest) == SUBREG
3541 || GET_CODE (dest) == ZERO_EXTRACT
3542 || GET_CODE (dest) == SIGN_EXTRACT
3543 || GET_CODE (dest) == STRICT_LOW_PART)
3544 dest = XEXP (dest, 0);
3545 if (GET_CODE (dest) == REG)
3548 int regno = REGNO (dest);
3549 for (i = 0; i < LOOP_REGNO_NREGS (regno, dest); i++)
3551 /* If this is the first setting of this reg
3552 in current basic block, and it was set before,
3553 it must be set in two basic blocks, so it cannot
3554 be moved out of the loop. */
3555 if (regs->array[regno].set_in_loop > 0
3556 && last_set[regno] == 0)
3557 regs->array[regno+i].may_not_optimize = 1;
3558 /* If this is not first setting in current basic block,
3559 see if reg was used in between previous one and this.
3560 If so, neither one can be moved. */
3561 if (last_set[regno] != 0
3562 && reg_used_between_p (dest, last_set[regno], insn))
3563 regs->array[regno+i].may_not_optimize = 1;
3564 if (regs->array[regno+i].set_in_loop < 127)
3565 ++regs->array[regno+i].set_in_loop;
3566 last_set[regno+i] = insn;
3572 /* Given a loop that is bounded by LOOP->START and LOOP->END and that
3573 is entered at LOOP->SCAN_START, return 1 if the register set in SET
3574 contained in insn INSN is used by any insn that precedes INSN in
3575 cyclic order starting from the loop entry point.
3577 We don't want to use INSN_LUID here because if we restrict INSN to those
3578 that have a valid INSN_LUID, it means we cannot move an invariant out
3579 from an inner loop past two loops. */
3582 loop_reg_used_before_p (const struct loop *loop, rtx set, rtx insn)
3584 rtx reg = SET_DEST (set);
3587 /* Scan forward checking for register usage. If we hit INSN, we
3588 are done. Otherwise, if we hit LOOP->END, wrap around to LOOP->START. */
3589 for (p = loop->scan_start; p != insn; p = NEXT_INSN (p))
3591 if (INSN_P (p) && reg_overlap_mentioned_p (reg, PATTERN (p)))
3602 /* Information we collect about arrays that we might want to prefetch. */
3603 struct prefetch_info
3605 struct iv_class *class; /* Class this prefetch is based on. */
3606 struct induction *giv; /* GIV this prefetch is based on. */
3607 rtx base_address; /* Start prefetching from this address plus
3609 HOST_WIDE_INT index;
3610 HOST_WIDE_INT stride; /* Prefetch stride in bytes in each
3612 unsigned int bytes_accessed; /* Sum of sizes of all accesses to this
3613 prefetch area in one iteration. */
3614 unsigned int total_bytes; /* Total bytes loop will access in this block.
3615 This is set only for loops with known
3616 iteration counts and is 0xffffffff
3618 int prefetch_in_loop; /* Number of prefetch insns in loop. */
3619 int prefetch_before_loop; /* Number of prefetch insns before loop. */
3620 unsigned int write : 1; /* 1 for read/write prefetches. */
3623 /* Data used by check_store function. */
3624 struct check_store_data
3630 static void check_store (rtx, rtx, void *);
3631 static void emit_prefetch_instructions (struct loop *);
3632 static int rtx_equal_for_prefetch_p (rtx, rtx);
3634 /* Set mem_write when mem_address is found. Used as callback to
3637 check_store (rtx x, rtx pat ATTRIBUTE_UNUSED, void *data)
3639 struct check_store_data *d = (struct check_store_data *) data;
3641 if ((GET_CODE (x) == MEM) && rtx_equal_p (d->mem_address, XEXP (x, 0)))
3645 /* Like rtx_equal_p, but attempts to swap commutative operands. This is
3646 important to get some addresses combined. Later more sophisticated
3647 transformations can be added when necessary.
3649 ??? Same trick with swapping operand is done at several other places.
3650 It can be nice to develop some common way to handle this. */
3653 rtx_equal_for_prefetch_p (rtx x, rtx y)
3657 enum rtx_code code = GET_CODE (x);
3662 if (code != GET_CODE (y))
3665 code = GET_CODE (x);
3667 if (GET_RTX_CLASS (code) == 'c')
3669 return ((rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 0))
3670 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 1)))
3671 || (rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 1))
3672 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 0))));
3674 /* Compare the elements. If any pair of corresponding elements fails to
3675 match, return 0 for the whole thing. */
3677 fmt = GET_RTX_FORMAT (code);
3678 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3683 if (XWINT (x, i) != XWINT (y, i))
3688 if (XINT (x, i) != XINT (y, i))
3693 /* Two vectors must have the same length. */
3694 if (XVECLEN (x, i) != XVECLEN (y, i))
3697 /* And the corresponding elements must match. */
3698 for (j = 0; j < XVECLEN (x, i); j++)
3699 if (rtx_equal_for_prefetch_p (XVECEXP (x, i, j),
3700 XVECEXP (y, i, j)) == 0)
3705 if (rtx_equal_for_prefetch_p (XEXP (x, i), XEXP (y, i)) == 0)
3710 if (strcmp (XSTR (x, i), XSTR (y, i)))
3715 /* These are just backpointers, so they don't matter. */
3721 /* It is believed that rtx's at this level will never
3722 contain anything but integers and other rtx's,
3723 except for within LABEL_REFs and SYMBOL_REFs. */
3731 /* Remove constant addition value from the expression X (when present)
3734 static HOST_WIDE_INT
3735 remove_constant_addition (rtx *x)
3737 HOST_WIDE_INT addval = 0;
3740 /* Avoid clobbering a shared CONST expression. */
3741 if (GET_CODE (exp) == CONST)
3743 if (GET_CODE (XEXP (exp, 0)) == PLUS
3744 && GET_CODE (XEXP (XEXP (exp, 0), 0)) == SYMBOL_REF
3745 && GET_CODE (XEXP (XEXP (exp, 0), 1)) == CONST_INT)
3747 *x = XEXP (XEXP (exp, 0), 0);
3748 return INTVAL (XEXP (XEXP (exp, 0), 1));
3753 if (GET_CODE (exp) == CONST_INT)
3755 addval = INTVAL (exp);
3759 /* For plus expression recurse on ourself. */
3760 else if (GET_CODE (exp) == PLUS)
3762 addval += remove_constant_addition (&XEXP (exp, 0));
3763 addval += remove_constant_addition (&XEXP (exp, 1));
3765 /* In case our parameter was constant, remove extra zero from the
3767 if (XEXP (exp, 0) == const0_rtx)
3769 else if (XEXP (exp, 1) == const0_rtx)
3776 /* Attempt to identify accesses to arrays that are most likely to cause cache
3777 misses, and emit prefetch instructions a few prefetch blocks forward.
3779 To detect the arrays we use the GIV information that was collected by the
3780 strength reduction pass.
3782 The prefetch instructions are generated after the GIV information is done
3783 and before the strength reduction process. The new GIVs are injected into
3784 the strength reduction tables, so the prefetch addresses are optimized as
3787 GIVs are split into base address, stride, and constant addition values.
3788 GIVs with the same address, stride and close addition values are combined
3789 into a single prefetch. Also writes to GIVs are detected, so that prefetch
3790 for write instructions can be used for the block we write to, on machines
3791 that support write prefetches.
3793 Several heuristics are used to determine when to prefetch. They are
3794 controlled by defined symbols that can be overridden for each target. */
3797 emit_prefetch_instructions (struct loop *loop)
3799 int num_prefetches = 0;
3800 int num_real_prefetches = 0;
3801 int num_real_write_prefetches = 0;
3802 int num_prefetches_before = 0;
3803 int num_write_prefetches_before = 0;
3806 struct iv_class *bl;
3807 struct induction *iv;
3808 struct prefetch_info info[MAX_PREFETCHES];
3809 struct loop_ivs *ivs = LOOP_IVS (loop);
3814 /* Consider only loops w/o calls. When a call is done, the loop is probably
3815 slow enough to read the memory. */
3816 if (PREFETCH_NO_CALL && LOOP_INFO (loop)->has_call)
3818 if (loop_dump_stream)
3819 fprintf (loop_dump_stream, "Prefetch: ignoring loop: has call.\n");
3824 /* Don't prefetch in loops known to have few iterations. */
3825 if (PREFETCH_NO_LOW_LOOPCNT
3826 && LOOP_INFO (loop)->n_iterations
3827 && LOOP_INFO (loop)->n_iterations <= PREFETCH_LOW_LOOPCNT)
3829 if (loop_dump_stream)
3830 fprintf (loop_dump_stream,
3831 "Prefetch: ignoring loop: not enough iterations.\n");
3835 /* Search all induction variables and pick those interesting for the prefetch
3837 for (bl = ivs->list; bl; bl = bl->next)
3839 struct induction *biv = bl->biv, *biv1;
3844 /* Expect all BIVs to be executed in each iteration. This makes our
3845 analysis more conservative. */
3848 /* Discard non-constant additions that we can't handle well yet, and
3849 BIVs that are executed multiple times; such BIVs ought to be
3850 handled in the nested loop. We accept not_every_iteration BIVs,
3851 since these only result in larger strides and make our
3852 heuristics more conservative. */
3853 if (GET_CODE (biv->add_val) != CONST_INT)
3855 if (loop_dump_stream)
3857 fprintf (loop_dump_stream,
3858 "Prefetch: ignoring biv %d: non-constant addition at insn %d:",
3859 REGNO (biv->src_reg), INSN_UID (biv->insn));
3860 print_rtl (loop_dump_stream, biv->add_val);
3861 fprintf (loop_dump_stream, "\n");
3866 if (biv->maybe_multiple)
3868 if (loop_dump_stream)
3870 fprintf (loop_dump_stream,
3871 "Prefetch: ignoring biv %d: maybe_multiple at insn %i:",
3872 REGNO (biv->src_reg), INSN_UID (biv->insn));
3873 print_rtl (loop_dump_stream, biv->add_val);
3874 fprintf (loop_dump_stream, "\n");
3879 basestride += INTVAL (biv1->add_val);
3880 biv1 = biv1->next_iv;
3883 if (biv1 || !basestride)
3886 for (iv = bl->giv; iv; iv = iv->next_iv)
3890 HOST_WIDE_INT index = 0;
3892 HOST_WIDE_INT stride = 0;
3893 int stride_sign = 1;
3894 struct check_store_data d;
3895 const char *ignore_reason = NULL;
3896 int size = GET_MODE_SIZE (GET_MODE (iv));
3898 /* See whether an induction variable is interesting to us and if
3899 not, report the reason. */
3900 if (iv->giv_type != DEST_ADDR)
3901 ignore_reason = "giv is not a destination address";
3903 /* We are interested only in constant stride memory references
3904 in order to be able to compute density easily. */
3905 else if (GET_CODE (iv->mult_val) != CONST_INT)
3906 ignore_reason = "stride is not constant";
3910 stride = INTVAL (iv->mult_val) * basestride;
3917 /* On some targets, reversed order prefetches are not
3919 if (PREFETCH_NO_REVERSE_ORDER && stride_sign < 0)
3920 ignore_reason = "reversed order stride";
3922 /* Prefetch of accesses with an extreme stride might not be
3923 worthwhile, either. */
3924 else if (PREFETCH_NO_EXTREME_STRIDE
3925 && stride > PREFETCH_EXTREME_STRIDE)
3926 ignore_reason = "extreme stride";
3928 /* Ignore GIVs with varying add values; we can't predict the
3929 value for the next iteration. */
3930 else if (!loop_invariant_p (loop, iv->add_val))
3931 ignore_reason = "giv has varying add value";
3933 /* Ignore GIVs in the nested loops; they ought to have been
3935 else if (iv->maybe_multiple)
3936 ignore_reason = "giv is in nested loop";
3939 if (ignore_reason != NULL)
3941 if (loop_dump_stream)
3942 fprintf (loop_dump_stream,
3943 "Prefetch: ignoring giv at %d: %s.\n",
3944 INSN_UID (iv->insn), ignore_reason);
3948 /* Determine the pointer to the basic array we are examining. It is
3949 the sum of the BIV's initial value and the GIV's add_val. */
3950 address = copy_rtx (iv->add_val);
3951 temp = copy_rtx (bl->initial_value);
3953 address = simplify_gen_binary (PLUS, Pmode, temp, address);
3954 index = remove_constant_addition (&address);
3957 d.mem_address = *iv->location;
3959 /* When the GIV is not always executed, we might be better off by
3960 not dirtying the cache pages. */
3961 if (PREFETCH_CONDITIONAL || iv->always_executed)
3962 note_stores (PATTERN (iv->insn), check_store, &d);
3965 if (loop_dump_stream)
3966 fprintf (loop_dump_stream, "Prefetch: Ignoring giv at %d: %s\n",
3967 INSN_UID (iv->insn), "in conditional code.");
3971 /* Attempt to find another prefetch to the same array and see if we
3972 can merge this one. */
3973 for (i = 0; i < num_prefetches; i++)
3974 if (rtx_equal_for_prefetch_p (address, info[i].base_address)
3975 && stride == info[i].stride)
3977 /* In case both access same array (same location
3978 just with small difference in constant indexes), merge
3979 the prefetches. Just do the later and the earlier will
3980 get prefetched from previous iteration.
3981 The artificial threshold should not be too small,
3982 but also not bigger than small portion of memory usually
3983 traversed by single loop. */
3984 if (index >= info[i].index
3985 && index - info[i].index < PREFETCH_EXTREME_DIFFERENCE)
3987 info[i].write |= d.mem_write;
3988 info[i].bytes_accessed += size;
3989 info[i].index = index;
3992 info[num_prefetches].base_address = address;
3997 if (index < info[i].index
3998 && info[i].index - index < PREFETCH_EXTREME_DIFFERENCE)
4000 info[i].write |= d.mem_write;
4001 info[i].bytes_accessed += size;
4007 /* Merging failed. */
4010 info[num_prefetches].giv = iv;
4011 info[num_prefetches].class = bl;
4012 info[num_prefetches].index = index;
4013 info[num_prefetches].stride = stride;
4014 info[num_prefetches].base_address = address;
4015 info[num_prefetches].write = d.mem_write;
4016 info[num_prefetches].bytes_accessed = size;
4018 if (num_prefetches >= MAX_PREFETCHES)
4020 if (loop_dump_stream)
4021 fprintf (loop_dump_stream,
4022 "Maximal number of prefetches exceeded.\n");
4029 for (i = 0; i < num_prefetches; i++)
4033 /* Attempt to calculate the total number of bytes fetched by all
4034 iterations of the loop. Avoid overflow. */
4035 if (LOOP_INFO (loop)->n_iterations
4036 && ((unsigned HOST_WIDE_INT) (0xffffffff / info[i].stride)
4037 >= LOOP_INFO (loop)->n_iterations))
4038 info[i].total_bytes = info[i].stride * LOOP_INFO (loop)->n_iterations;
4040 info[i].total_bytes = 0xffffffff;
4042 density = info[i].bytes_accessed * 100 / info[i].stride;
4044 /* Prefetch might be worthwhile only when the loads/stores are dense. */
4045 if (PREFETCH_ONLY_DENSE_MEM)
4046 if (density * 256 > PREFETCH_DENSE_MEM * 100
4047 && (info[i].total_bytes / PREFETCH_BLOCK
4048 >= PREFETCH_BLOCKS_BEFORE_LOOP_MIN))
4050 info[i].prefetch_before_loop = 1;
4051 info[i].prefetch_in_loop
4052 = (info[i].total_bytes / PREFETCH_BLOCK
4053 > PREFETCH_BLOCKS_BEFORE_LOOP_MAX);
4057 info[i].prefetch_in_loop = 0, info[i].prefetch_before_loop = 0;
4058 if (loop_dump_stream)
4059 fprintf (loop_dump_stream,
4060 "Prefetch: ignoring giv at %d: %d%% density is too low.\n",
4061 INSN_UID (info[i].giv->insn), density);
4064 info[i].prefetch_in_loop = 1, info[i].prefetch_before_loop = 1;
4066 /* Find how many prefetch instructions we'll use within the loop. */
4067 if (info[i].prefetch_in_loop != 0)
4069 info[i].prefetch_in_loop = ((info[i].stride + PREFETCH_BLOCK - 1)
4071 num_real_prefetches += info[i].prefetch_in_loop;
4073 num_real_write_prefetches += info[i].prefetch_in_loop;
4077 /* Determine how many iterations ahead to prefetch within the loop, based
4078 on how many prefetches we currently expect to do within the loop. */
4079 if (num_real_prefetches != 0)
4081 if ((ahead = SIMULTANEOUS_PREFETCHES / num_real_prefetches) == 0)
4083 if (loop_dump_stream)
4084 fprintf (loop_dump_stream,
4085 "Prefetch: ignoring prefetches within loop: ahead is zero; %d < %d\n",
4086 SIMULTANEOUS_PREFETCHES, num_real_prefetches);
4087 num_real_prefetches = 0, num_real_write_prefetches = 0;
4090 /* We'll also use AHEAD to determine how many prefetch instructions to
4091 emit before a loop, so don't leave it zero. */
4093 ahead = PREFETCH_BLOCKS_BEFORE_LOOP_MAX;
4095 for (i = 0; i < num_prefetches; i++)
4097 /* Update if we've decided not to prefetch anything within the loop. */
4098 if (num_real_prefetches == 0)
4099 info[i].prefetch_in_loop = 0;
4101 /* Find how many prefetch instructions we'll use before the loop. */
4102 if (info[i].prefetch_before_loop != 0)
4104 int n = info[i].total_bytes / PREFETCH_BLOCK;
4107 info[i].prefetch_before_loop = n;
4108 num_prefetches_before += n;
4110 num_write_prefetches_before += n;
4113 if (loop_dump_stream)
4115 if (info[i].prefetch_in_loop == 0
4116 && info[i].prefetch_before_loop == 0)
4118 fprintf (loop_dump_stream, "Prefetch insn: %d",
4119 INSN_UID (info[i].giv->insn));
4120 fprintf (loop_dump_stream,
4121 "; in loop: %d; before: %d; %s\n",
4122 info[i].prefetch_in_loop,
4123 info[i].prefetch_before_loop,
4124 info[i].write ? "read/write" : "read only");
4125 fprintf (loop_dump_stream,
4126 " density: %d%%; bytes_accessed: %u; total_bytes: %u\n",
4127 (int) (info[i].bytes_accessed * 100 / info[i].stride),
4128 info[i].bytes_accessed, info[i].total_bytes);
4129 fprintf (loop_dump_stream, " index: " HOST_WIDE_INT_PRINT_DEC
4130 "; stride: " HOST_WIDE_INT_PRINT_DEC "; address: ",
4131 info[i].index, info[i].stride);
4132 print_rtl (loop_dump_stream, info[i].base_address);
4133 fprintf (loop_dump_stream, "\n");
4137 if (num_real_prefetches + num_prefetches_before > 0)
4139 /* Record that this loop uses prefetch instructions. */
4140 LOOP_INFO (loop)->has_prefetch = 1;
4142 if (loop_dump_stream)
4144 fprintf (loop_dump_stream, "Real prefetches needed within loop: %d (write: %d)\n",
4145 num_real_prefetches, num_real_write_prefetches);
4146 fprintf (loop_dump_stream, "Real prefetches needed before loop: %d (write: %d)\n",
4147 num_prefetches_before, num_write_prefetches_before);
4151 for (i = 0; i < num_prefetches; i++)
4155 for (y = 0; y < info[i].prefetch_in_loop; y++)
4157 rtx loc = copy_rtx (*info[i].giv->location);
4159 int bytes_ahead = PREFETCH_BLOCK * (ahead + y);
4160 rtx before_insn = info[i].giv->insn;
4161 rtx prev_insn = PREV_INSN (info[i].giv->insn);
4164 /* We can save some effort by offsetting the address on
4165 architectures with offsettable memory references. */
4166 if (offsettable_address_p (0, VOIDmode, loc))
4167 loc = plus_constant (loc, bytes_ahead);
4170 rtx reg = gen_reg_rtx (Pmode);
4171 loop_iv_add_mult_emit_before (loop, loc, const1_rtx,
4172 GEN_INT (bytes_ahead), reg,
4178 /* Make sure the address operand is valid for prefetch. */
4179 if (! (*insn_data[(int)CODE_FOR_prefetch].operand[0].predicate)
4180 (loc, insn_data[(int)CODE_FOR_prefetch].operand[0].mode))
4181 loc = force_reg (Pmode, loc);
4182 emit_insn (gen_prefetch (loc, GEN_INT (info[i].write),
4186 emit_insn_before (seq, before_insn);
4188 /* Check all insns emitted and record the new GIV
4190 insn = NEXT_INSN (prev_insn);
4191 while (insn != before_insn)
4193 insn = check_insn_for_givs (loop, insn,
4194 info[i].giv->always_executed,
4195 info[i].giv->maybe_multiple);
4196 insn = NEXT_INSN (insn);
4200 if (PREFETCH_BEFORE_LOOP)
4202 /* Emit insns before the loop to fetch the first cache lines or,
4203 if we're not prefetching within the loop, everything we expect
4205 for (y = 0; y < info[i].prefetch_before_loop; y++)
4207 rtx reg = gen_reg_rtx (Pmode);
4208 rtx loop_start = loop->start;
4209 rtx init_val = info[i].class->initial_value;
4210 rtx add_val = simplify_gen_binary (PLUS, Pmode,
4211 info[i].giv->add_val,
4212 GEN_INT (y * PREFETCH_BLOCK));
4214 /* Functions called by LOOP_IV_ADD_EMIT_BEFORE expect a
4215 non-constant INIT_VAL to have the same mode as REG, which
4216 in this case we know to be Pmode. */
4217 if (GET_MODE (init_val) != Pmode && !CONSTANT_P (init_val))
4222 init_val = convert_to_mode (Pmode, init_val, 0);
4225 loop_insn_emit_before (loop, 0, loop_start, seq);
4227 loop_iv_add_mult_emit_before (loop, init_val,
4228 info[i].giv->mult_val,
4229 add_val, reg, 0, loop_start);
4230 emit_insn_before (gen_prefetch (reg, GEN_INT (info[i].write),
4240 /* Communication with routines called via `note_stores'. */
4242 static rtx note_insn;
4244 /* Dummy register to have nonzero DEST_REG for DEST_ADDR type givs. */
4246 static rtx addr_placeholder;
4248 /* ??? Unfinished optimizations, and possible future optimizations,
4249 for the strength reduction code. */
4251 /* ??? The interaction of biv elimination, and recognition of 'constant'
4252 bivs, may cause problems. */
4254 /* ??? Add heuristics so that DEST_ADDR strength reduction does not cause
4255 performance problems.
4257 Perhaps don't eliminate things that can be combined with an addressing
4258 mode. Find all givs that have the same biv, mult_val, and add_val;
4259 then for each giv, check to see if its only use dies in a following
4260 memory address. If so, generate a new memory address and check to see
4261 if it is valid. If it is valid, then store the modified memory address,
4262 otherwise, mark the giv as not done so that it will get its own iv. */
4264 /* ??? Could try to optimize branches when it is known that a biv is always
4267 /* ??? When replace a biv in a compare insn, we should replace with closest
4268 giv so that an optimized branch can still be recognized by the combiner,
4269 e.g. the VAX acb insn. */
4271 /* ??? Many of the checks involving uid_luid could be simplified if regscan
4272 was rerun in loop_optimize whenever a register was added or moved.
4273 Also, some of the optimizations could be a little less conservative. */
4275 /* Scan the loop body and call FNCALL for each insn. In the addition to the
4276 LOOP and INSN parameters pass MAYBE_MULTIPLE and NOT_EVERY_ITERATION to the
4279 NOT_EVERY_ITERATION is 1 if current insn is not known to be executed at
4280 least once for every loop iteration except for the last one.
4282 MAYBE_MULTIPLE is 1 if current insn may be executed more than once for every
4286 for_each_insn_in_loop (struct loop *loop, loop_insn_callback fncall)
4288 int not_every_iteration = 0;
4289 int maybe_multiple = 0;
4290 int past_loop_latch = 0;
4294 /* If loop_scan_start points to the loop exit test, we have to be wary of
4295 subversive use of gotos inside expression statements. */
4296 if (prev_nonnote_insn (loop->scan_start) != prev_nonnote_insn (loop->start))
4297 maybe_multiple = back_branch_in_range_p (loop, loop->scan_start);
4299 /* Scan through loop and update NOT_EVERY_ITERATION and MAYBE_MULTIPLE. */
4300 for (p = next_insn_in_loop (loop, loop->scan_start);
4302 p = next_insn_in_loop (loop, p))
4304 p = fncall (loop, p, not_every_iteration, maybe_multiple);
4306 /* Past CODE_LABEL, we get to insns that may be executed multiple
4307 times. The only way we can be sure that they can't is if every
4308 jump insn between here and the end of the loop either
4309 returns, exits the loop, is a jump to a location that is still
4310 behind the label, or is a jump to the loop start. */
4312 if (GET_CODE (p) == CODE_LABEL)
4320 insn = NEXT_INSN (insn);
4321 if (insn == loop->scan_start)
4323 if (insn == loop->end)
4329 if (insn == loop->scan_start)
4333 if (GET_CODE (insn) == JUMP_INSN
4334 && GET_CODE (PATTERN (insn)) != RETURN
4335 && (!any_condjump_p (insn)
4336 || (JUMP_LABEL (insn) != 0
4337 && JUMP_LABEL (insn) != loop->scan_start
4338 && !loop_insn_first_p (p, JUMP_LABEL (insn)))))
4346 /* Past a jump, we get to insns for which we can't count
4347 on whether they will be executed during each iteration. */
4348 /* This code appears twice in strength_reduce. There is also similar
4349 code in scan_loop. */
4350 if (GET_CODE (p) == JUMP_INSN
4351 /* If we enter the loop in the middle, and scan around to the
4352 beginning, don't set not_every_iteration for that.
4353 This can be any kind of jump, since we want to know if insns
4354 will be executed if the loop is executed. */
4355 && !(JUMP_LABEL (p) == loop->top
4356 && ((NEXT_INSN (NEXT_INSN (p)) == loop->end
4357 && any_uncondjump_p (p))
4358 || (NEXT_INSN (p) == loop->end && any_condjump_p (p)))))
4362 /* If this is a jump outside the loop, then it also doesn't
4363 matter. Check to see if the target of this branch is on the
4364 loop->exits_labels list. */
4366 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
4367 if (XEXP (label, 0) == JUMP_LABEL (p))
4371 not_every_iteration = 1;
4374 else if (GET_CODE (p) == NOTE)
4376 /* At the virtual top of a converted loop, insns are again known to
4377 be executed each iteration: logically, the loop begins here
4378 even though the exit code has been duplicated.
4380 Insns are also again known to be executed each iteration at
4381 the LOOP_CONT note. */
4382 if ((NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP
4383 || NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_CONT)
4385 not_every_iteration = 0;
4386 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
4388 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
4392 /* Note if we pass a loop latch. If we do, then we can not clear
4393 NOT_EVERY_ITERATION below when we pass the last CODE_LABEL in
4394 a loop since a jump before the last CODE_LABEL may have started
4395 a new loop iteration.
4397 Note that LOOP_TOP is only set for rotated loops and we need
4398 this check for all loops, so compare against the CODE_LABEL
4399 which immediately follows LOOP_START. */
4400 if (GET_CODE (p) == JUMP_INSN
4401 && JUMP_LABEL (p) == NEXT_INSN (loop->start))
4402 past_loop_latch = 1;
4404 /* Unlike in the code motion pass where MAYBE_NEVER indicates that
4405 an insn may never be executed, NOT_EVERY_ITERATION indicates whether
4406 or not an insn is known to be executed each iteration of the
4407 loop, whether or not any iterations are known to occur.
4409 Therefore, if we have just passed a label and have no more labels
4410 between here and the test insn of the loop, and we have not passed
4411 a jump to the top of the loop, then we know these insns will be
4412 executed each iteration. */
4414 if (not_every_iteration
4416 && GET_CODE (p) == CODE_LABEL
4417 && no_labels_between_p (p, loop->end)
4418 && loop_insn_first_p (p, loop->cont))
4419 not_every_iteration = 0;
4424 loop_bivs_find (struct loop *loop)
4426 struct loop_regs *regs = LOOP_REGS (loop);
4427 struct loop_ivs *ivs = LOOP_IVS (loop);
4428 /* Temporary list pointers for traversing ivs->list. */
4429 struct iv_class *bl, **backbl;
4433 for_each_insn_in_loop (loop, check_insn_for_bivs);
4435 /* Scan ivs->list to remove all regs that proved not to be bivs.
4436 Make a sanity check against regs->n_times_set. */
4437 for (backbl = &ivs->list, bl = *backbl; bl; bl = bl->next)
4439 if (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4440 /* Above happens if register modified by subreg, etc. */
4441 /* Make sure it is not recognized as a basic induction var: */
4442 || regs->array[bl->regno].n_times_set != bl->biv_count
4443 /* If never incremented, it is invariant that we decided not to
4444 move. So leave it alone. */
4445 || ! bl->incremented)
4447 if (loop_dump_stream)
4448 fprintf (loop_dump_stream, "Biv %d: discarded, %s\n",
4450 (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4451 ? "not induction variable"
4452 : (! bl->incremented ? "never incremented"
4455 REG_IV_TYPE (ivs, bl->regno) = NOT_BASIC_INDUCT;
4462 if (loop_dump_stream)
4463 fprintf (loop_dump_stream, "Biv %d: verified\n", bl->regno);
4469 /* Determine how BIVS are initialized by looking through pre-header
4470 extended basic block. */
4472 loop_bivs_init_find (struct loop *loop)
4474 struct loop_ivs *ivs = LOOP_IVS (loop);
4475 /* Temporary list pointers for traversing ivs->list. */
4476 struct iv_class *bl;
4480 /* Find initial value for each biv by searching backwards from loop_start,
4481 halting at first label. Also record any test condition. */
4484 for (p = loop->start; p && GET_CODE (p) != CODE_LABEL; p = PREV_INSN (p))
4490 if (GET_CODE (p) == CALL_INSN)
4494 note_stores (PATTERN (p), record_initial, ivs);
4496 /* Record any test of a biv that branches around the loop if no store
4497 between it and the start of loop. We only care about tests with
4498 constants and registers and only certain of those. */
4499 if (GET_CODE (p) == JUMP_INSN
4500 && JUMP_LABEL (p) != 0
4501 && next_real_insn (JUMP_LABEL (p)) == next_real_insn (loop->end)
4502 && (test = get_condition_for_loop (loop, p)) != 0
4503 && GET_CODE (XEXP (test, 0)) == REG
4504 && REGNO (XEXP (test, 0)) < max_reg_before_loop
4505 && (bl = REG_IV_CLASS (ivs, REGNO (XEXP (test, 0)))) != 0
4506 && valid_initial_value_p (XEXP (test, 1), p, call_seen, loop->start)
4507 && bl->init_insn == 0)
4509 /* If an NE test, we have an initial value! */
4510 if (GET_CODE (test) == NE)
4513 bl->init_set = gen_rtx_SET (VOIDmode,
4514 XEXP (test, 0), XEXP (test, 1));
4517 bl->initial_test = test;
4523 /* Look at the each biv and see if we can say anything better about its
4524 initial value from any initializing insns set up above. (This is done
4525 in two passes to avoid missing SETs in a PARALLEL.) */
4527 loop_bivs_check (struct loop *loop)
4529 struct loop_ivs *ivs = LOOP_IVS (loop);
4530 /* Temporary list pointers for traversing ivs->list. */
4531 struct iv_class *bl;
4532 struct iv_class **backbl;
4534 for (backbl = &ivs->list; (bl = *backbl); backbl = &bl->next)
4539 if (! bl->init_insn)
4542 /* IF INIT_INSN has a REG_EQUAL or REG_EQUIV note and the value
4543 is a constant, use the value of that. */
4544 if (((note = find_reg_note (bl->init_insn, REG_EQUAL, 0)) != NULL
4545 && CONSTANT_P (XEXP (note, 0)))
4546 || ((note = find_reg_note (bl->init_insn, REG_EQUIV, 0)) != NULL
4547 && CONSTANT_P (XEXP (note, 0))))
4548 src = XEXP (note, 0);
4550 src = SET_SRC (bl->init_set);
4552 if (loop_dump_stream)
4553 fprintf (loop_dump_stream,
4554 "Biv %d: initialized at insn %d: initial value ",
4555 bl->regno, INSN_UID (bl->init_insn));
4557 if ((GET_MODE (src) == GET_MODE (regno_reg_rtx[bl->regno])
4558 || GET_MODE (src) == VOIDmode)
4559 && valid_initial_value_p (src, bl->init_insn,
4560 LOOP_INFO (loop)->pre_header_has_call,
4563 bl->initial_value = src;
4565 if (loop_dump_stream)
4567 print_simple_rtl (loop_dump_stream, src);
4568 fputc ('\n', loop_dump_stream);
4571 /* If we can't make it a giv,
4572 let biv keep initial value of "itself". */
4573 else if (loop_dump_stream)
4574 fprintf (loop_dump_stream, "is complex\n");
4579 /* Search the loop for general induction variables. */
4582 loop_givs_find (struct loop* loop)
4584 for_each_insn_in_loop (loop, check_insn_for_givs);
4588 /* For each giv for which we still don't know whether or not it is
4589 replaceable, check to see if it is replaceable because its final value
4590 can be calculated. */
4593 loop_givs_check (struct loop *loop)
4595 struct loop_ivs *ivs = LOOP_IVS (loop);
4596 struct iv_class *bl;
4598 for (bl = ivs->list; bl; bl = bl->next)
4600 struct induction *v;
4602 for (v = bl->giv; v; v = v->next_iv)
4603 if (! v->replaceable && ! v->not_replaceable)
4604 check_final_value (loop, v);
4609 /* Return nonzero if it is possible to eliminate the biv BL provided
4610 all givs are reduced. This is possible if either the reg is not
4611 used outside the loop, or we can compute what its final value will
4615 loop_biv_eliminable_p (struct loop *loop, struct iv_class *bl,
4616 int threshold, int insn_count)
4618 /* For architectures with a decrement_and_branch_until_zero insn,
4619 don't do this if we put a REG_NONNEG note on the endtest for this
4622 #ifdef HAVE_decrement_and_branch_until_zero
4625 if (loop_dump_stream)
4626 fprintf (loop_dump_stream,
4627 "Cannot eliminate nonneg biv %d.\n", bl->regno);
4632 /* Check that biv is used outside loop or if it has a final value.
4633 Compare against bl->init_insn rather than loop->start. We aren't
4634 concerned with any uses of the biv between init_insn and
4635 loop->start since these won't be affected by the value of the biv
4636 elsewhere in the function, so long as init_insn doesn't use the
4639 if ((REGNO_LAST_LUID (bl->regno) < INSN_LUID (loop->end)
4641 && INSN_UID (bl->init_insn) < max_uid_for_loop
4642 && REGNO_FIRST_LUID (bl->regno) >= INSN_LUID (bl->init_insn)
4643 && ! reg_mentioned_p (bl->biv->dest_reg, SET_SRC (bl->init_set)))
4644 || (bl->final_value = final_biv_value (loop, bl)))
4645 return maybe_eliminate_biv (loop, bl, 0, threshold, insn_count);
4647 if (loop_dump_stream)
4649 fprintf (loop_dump_stream,
4650 "Cannot eliminate biv %d.\n",
4652 fprintf (loop_dump_stream,
4653 "First use: insn %d, last use: insn %d.\n",
4654 REGNO_FIRST_UID (bl->regno),
4655 REGNO_LAST_UID (bl->regno));
4661 /* Reduce each giv of BL that we have decided to reduce. */
4664 loop_givs_reduce (struct loop *loop, struct iv_class *bl)
4666 struct induction *v;
4668 for (v = bl->giv; v; v = v->next_iv)
4670 struct induction *tv;
4671 if (! v->ignore && v->same == 0)
4673 int auto_inc_opt = 0;
4675 /* If the code for derived givs immediately below has already
4676 allocated a new_reg, we must keep it. */
4678 v->new_reg = gen_reg_rtx (v->mode);
4681 /* If the target has auto-increment addressing modes, and
4682 this is an address giv, then try to put the increment
4683 immediately after its use, so that flow can create an
4684 auto-increment addressing mode. */
4685 if (v->giv_type == DEST_ADDR && bl->biv_count == 1
4686 && bl->biv->always_executed && ! bl->biv->maybe_multiple
4687 /* We don't handle reversed biv's because bl->biv->insn
4688 does not have a valid INSN_LUID. */
4690 && v->always_executed && ! v->maybe_multiple
4691 && INSN_UID (v->insn) < max_uid_for_loop)
4693 /* If other giv's have been combined with this one, then
4694 this will work only if all uses of the other giv's occur
4695 before this giv's insn. This is difficult to check.
4697 We simplify this by looking for the common case where
4698 there is one DEST_REG giv, and this giv's insn is the
4699 last use of the dest_reg of that DEST_REG giv. If the
4700 increment occurs after the address giv, then we can
4701 perform the optimization. (Otherwise, the increment
4702 would have to go before other_giv, and we would not be
4703 able to combine it with the address giv to get an
4704 auto-inc address.) */
4705 if (v->combined_with)
4707 struct induction *other_giv = 0;
4709 for (tv = bl->giv; tv; tv = tv->next_iv)
4717 if (! tv && other_giv
4718 && REGNO (other_giv->dest_reg) < max_reg_before_loop
4719 && (REGNO_LAST_UID (REGNO (other_giv->dest_reg))
4720 == INSN_UID (v->insn))
4721 && INSN_LUID (v->insn) < INSN_LUID (bl->biv->insn))
4724 /* Check for case where increment is before the address
4725 giv. Do this test in "loop order". */
4726 else if ((INSN_LUID (v->insn) > INSN_LUID (bl->biv->insn)
4727 && (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4728 || (INSN_LUID (bl->biv->insn)
4729 > INSN_LUID (loop->scan_start))))
4730 || (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4731 && (INSN_LUID (loop->scan_start)
4732 < INSN_LUID (bl->biv->insn))))
4741 /* We can't put an insn immediately after one setting
4742 cc0, or immediately before one using cc0. */
4743 if ((auto_inc_opt == 1 && sets_cc0_p (PATTERN (v->insn)))
4744 || (auto_inc_opt == -1
4745 && (prev = prev_nonnote_insn (v->insn)) != 0
4747 && sets_cc0_p (PATTERN (prev))))
4753 v->auto_inc_opt = 1;
4757 /* For each place where the biv is incremented, add an insn
4758 to increment the new, reduced reg for the giv. */
4759 for (tv = bl->biv; tv; tv = tv->next_iv)
4763 /* Skip if location is the same as a previous one. */
4767 insert_before = NEXT_INSN (tv->insn);
4768 else if (auto_inc_opt == 1)
4769 insert_before = NEXT_INSN (v->insn);
4771 insert_before = v->insn;
4773 if (tv->mult_val == const1_rtx)
4774 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4775 v->new_reg, v->new_reg,
4777 else /* tv->mult_val == const0_rtx */
4778 /* A multiply is acceptable here
4779 since this is presumed to be seldom executed. */
4780 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4781 v->add_val, v->new_reg,
4785 /* Add code at loop start to initialize giv's reduced reg. */
4787 loop_iv_add_mult_hoist (loop,
4788 extend_value_for_giv (v, bl->initial_value),
4789 v->mult_val, v->add_val, v->new_reg);
4795 /* Check for givs whose first use is their definition and whose
4796 last use is the definition of another giv. If so, it is likely
4797 dead and should not be used to derive another giv nor to
4801 loop_givs_dead_check (struct loop *loop ATTRIBUTE_UNUSED, struct iv_class *bl)
4803 struct induction *v;
4805 for (v = bl->giv; v; v = v->next_iv)
4808 || (v->same && v->same->ignore))
4811 if (v->giv_type == DEST_REG
4812 && REGNO_FIRST_UID (REGNO (v->dest_reg)) == INSN_UID (v->insn))
4814 struct induction *v1;
4816 for (v1 = bl->giv; v1; v1 = v1->next_iv)
4817 if (REGNO_LAST_UID (REGNO (v->dest_reg)) == INSN_UID (v1->insn))
4825 loop_givs_rescan (struct loop *loop, struct iv_class *bl, rtx *reg_map)
4827 struct induction *v;
4829 for (v = bl->giv; v; v = v->next_iv)
4831 if (v->same && v->same->ignore)
4837 /* Update expression if this was combined, in case other giv was
4840 v->new_reg = replace_rtx (v->new_reg,
4841 v->same->dest_reg, v->same->new_reg);
4843 /* See if this register is known to be a pointer to something. If
4844 so, see if we can find the alignment. First see if there is a
4845 destination register that is a pointer. If so, this shares the
4846 alignment too. Next see if we can deduce anything from the
4847 computational information. If not, and this is a DEST_ADDR
4848 giv, at least we know that it's a pointer, though we don't know
4850 if (GET_CODE (v->new_reg) == REG
4851 && v->giv_type == DEST_REG
4852 && REG_POINTER (v->dest_reg))
4853 mark_reg_pointer (v->new_reg,
4854 REGNO_POINTER_ALIGN (REGNO (v->dest_reg)));
4855 else if (GET_CODE (v->new_reg) == REG
4856 && REG_POINTER (v->src_reg))
4858 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->src_reg));
4861 || GET_CODE (v->add_val) != CONST_INT
4862 || INTVAL (v->add_val) % (align / BITS_PER_UNIT) != 0)
4865 mark_reg_pointer (v->new_reg, align);
4867 else if (GET_CODE (v->new_reg) == REG
4868 && GET_CODE (v->add_val) == REG
4869 && REG_POINTER (v->add_val))
4871 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->add_val));
4873 if (align == 0 || GET_CODE (v->mult_val) != CONST_INT
4874 || INTVAL (v->mult_val) % (align / BITS_PER_UNIT) != 0)
4877 mark_reg_pointer (v->new_reg, align);
4879 else if (GET_CODE (v->new_reg) == REG && v->giv_type == DEST_ADDR)
4880 mark_reg_pointer (v->new_reg, 0);
4882 if (v->giv_type == DEST_ADDR)
4883 /* Store reduced reg as the address in the memref where we found
4885 validate_change (v->insn, v->location, v->new_reg, 0);
4886 else if (v->replaceable)
4888 reg_map[REGNO (v->dest_reg)] = v->new_reg;
4892 rtx original_insn = v->insn;
4895 /* Not replaceable; emit an insn to set the original giv reg from
4896 the reduced giv, same as above. */
4897 v->insn = loop_insn_emit_after (loop, 0, original_insn,
4898 gen_move_insn (v->dest_reg,
4901 /* The original insn may have a REG_EQUAL note. This note is
4902 now incorrect and may result in invalid substitutions later.
4903 The original insn is dead, but may be part of a libcall
4904 sequence, which doesn't seem worth the bother of handling. */
4905 note = find_reg_note (original_insn, REG_EQUAL, NULL_RTX);
4907 remove_note (original_insn, note);
4910 /* When a loop is reversed, givs which depend on the reversed
4911 biv, and which are live outside the loop, must be set to their
4912 correct final value. This insn is only needed if the giv is
4913 not replaceable. The correct final value is the same as the
4914 value that the giv starts the reversed loop with. */
4915 if (bl->reversed && ! v->replaceable)
4916 loop_iv_add_mult_sink (loop,
4917 extend_value_for_giv (v, bl->initial_value),
4918 v->mult_val, v->add_val, v->dest_reg);
4919 else if (v->final_value)
4920 loop_insn_sink_or_swim (loop,
4921 gen_load_of_final_value (v->dest_reg,
4924 if (loop_dump_stream)
4926 fprintf (loop_dump_stream, "giv at %d reduced to ",
4927 INSN_UID (v->insn));
4928 print_simple_rtl (loop_dump_stream, v->new_reg);
4929 fprintf (loop_dump_stream, "\n");
4936 loop_giv_reduce_benefit (struct loop *loop ATTRIBUTE_UNUSED,
4937 struct iv_class *bl, struct induction *v,
4943 benefit = v->benefit;
4944 PUT_MODE (test_reg, v->mode);
4945 add_cost = iv_add_mult_cost (bl->biv->add_val, v->mult_val,
4946 test_reg, test_reg);
4948 /* Reduce benefit if not replaceable, since we will insert a
4949 move-insn to replace the insn that calculates this giv. Don't do
4950 this unless the giv is a user variable, since it will often be
4951 marked non-replaceable because of the duplication of the exit
4952 code outside the loop. In such a case, the copies we insert are
4953 dead and will be deleted. So they don't have a cost. Similar
4954 situations exist. */
4955 /* ??? The new final_[bg]iv_value code does a much better job of
4956 finding replaceable giv's, and hence this code may no longer be
4958 if (! v->replaceable && ! bl->eliminable
4959 && REG_USERVAR_P (v->dest_reg))
4960 benefit -= copy_cost;
4962 /* Decrease the benefit to count the add-insns that we will insert
4963 to increment the reduced reg for the giv. ??? This can
4964 overestimate the run-time cost of the additional insns, e.g. if
4965 there are multiple basic blocks that increment the biv, but only
4966 one of these blocks is executed during each iteration. There is
4967 no good way to detect cases like this with the current structure
4968 of the loop optimizer. This code is more accurate for
4969 determining code size than run-time benefits. */
4970 benefit -= add_cost * bl->biv_count;
4972 /* Decide whether to strength-reduce this giv or to leave the code
4973 unchanged (recompute it from the biv each time it is used). This
4974 decision can be made independently for each giv. */
4977 /* Attempt to guess whether autoincrement will handle some of the
4978 new add insns; if so, increase BENEFIT (undo the subtraction of
4979 add_cost that was done above). */
4980 if (v->giv_type == DEST_ADDR
4981 /* Increasing the benefit is risky, since this is only a guess.
4982 Avoid increasing register pressure in cases where there would
4983 be no other benefit from reducing this giv. */
4985 && GET_CODE (v->mult_val) == CONST_INT)
4987 int size = GET_MODE_SIZE (GET_MODE (v->mem));
4989 if (HAVE_POST_INCREMENT
4990 && INTVAL (v->mult_val) == size)
4991 benefit += add_cost * bl->biv_count;
4992 else if (HAVE_PRE_INCREMENT
4993 && INTVAL (v->mult_val) == size)
4994 benefit += add_cost * bl->biv_count;
4995 else if (HAVE_POST_DECREMENT
4996 && -INTVAL (v->mult_val) == size)
4997 benefit += add_cost * bl->biv_count;
4998 else if (HAVE_PRE_DECREMENT
4999 && -INTVAL (v->mult_val) == size)
5000 benefit += add_cost * bl->biv_count;
5008 /* Free IV structures for LOOP. */
5011 loop_ivs_free (struct loop *loop)
5013 struct loop_ivs *ivs = LOOP_IVS (loop);
5014 struct iv_class *iv = ivs->list;
5020 struct iv_class *next = iv->next;
5021 struct induction *induction;
5022 struct induction *next_induction;
5024 for (induction = iv->biv; induction; induction = next_induction)
5026 next_induction = induction->next_iv;
5029 for (induction = iv->giv; induction; induction = next_induction)
5031 next_induction = induction->next_iv;
5041 /* Perform strength reduction and induction variable elimination.
5043 Pseudo registers created during this function will be beyond the
5044 last valid index in several tables including
5045 REGS->ARRAY[I].N_TIMES_SET and REGNO_LAST_UID. This does not cause a
5046 problem here, because the added registers cannot be givs outside of
5047 their loop, and hence will never be reconsidered. But scan_loop
5048 must check regnos to make sure they are in bounds. */
5051 strength_reduce (struct loop *loop, int flags)
5053 struct loop_info *loop_info = LOOP_INFO (loop);
5054 struct loop_regs *regs = LOOP_REGS (loop);
5055 struct loop_ivs *ivs = LOOP_IVS (loop);
5057 /* Temporary list pointer for traversing ivs->list. */
5058 struct iv_class *bl;
5059 /* Ratio of extra register life span we can justify
5060 for saving an instruction. More if loop doesn't call subroutines
5061 since in that case saving an insn makes more difference
5062 and more registers are available. */
5063 /* ??? could set this to last value of threshold in move_movables */
5064 int threshold = (loop_info->has_call ? 1 : 2) * (3 + n_non_fixed_regs);
5065 /* Map of pseudo-register replacements. */
5066 rtx *reg_map = NULL;
5068 int unrolled_insn_copies = 0;
5069 rtx test_reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
5070 int insn_count = count_insns_in_loop (loop);
5072 addr_placeholder = gen_reg_rtx (Pmode);
5074 ivs->n_regs = max_reg_before_loop;
5075 ivs->regs = xcalloc (ivs->n_regs, sizeof (struct iv));
5077 /* Find all BIVs in loop. */
5078 loop_bivs_find (loop);
5080 /* Exit if there are no bivs. */
5083 /* Can still unroll the loop anyways, but indicate that there is no
5084 strength reduction info available. */
5085 if (flags & LOOP_UNROLL)
5086 unroll_loop (loop, insn_count, 0);
5088 loop_ivs_free (loop);
5092 /* Determine how BIVS are initialized by looking through pre-header
5093 extended basic block. */
5094 loop_bivs_init_find (loop);
5096 /* Look at the each biv and see if we can say anything better about its
5097 initial value from any initializing insns set up above. */
5098 loop_bivs_check (loop);
5100 /* Search the loop for general induction variables. */
5101 loop_givs_find (loop);
5103 /* Try to calculate and save the number of loop iterations. This is
5104 set to zero if the actual number can not be calculated. This must
5105 be called after all giv's have been identified, since otherwise it may
5106 fail if the iteration variable is a giv. */
5107 loop_iterations (loop);
5109 #ifdef HAVE_prefetch
5110 if (flags & LOOP_PREFETCH)
5111 emit_prefetch_instructions (loop);
5114 /* Now for each giv for which we still don't know whether or not it is
5115 replaceable, check to see if it is replaceable because its final value
5116 can be calculated. This must be done after loop_iterations is called,
5117 so that final_giv_value will work correctly. */
5118 loop_givs_check (loop);
5120 /* Try to prove that the loop counter variable (if any) is always
5121 nonnegative; if so, record that fact with a REG_NONNEG note
5122 so that "decrement and branch until zero" insn can be used. */
5123 check_dbra_loop (loop, insn_count);
5125 /* Create reg_map to hold substitutions for replaceable giv regs.
5126 Some givs might have been made from biv increments, so look at
5127 ivs->reg_iv_type for a suitable size. */
5128 reg_map_size = ivs->n_regs;
5129 reg_map = xcalloc (reg_map_size, sizeof (rtx));
5131 /* Examine each iv class for feasibility of strength reduction/induction
5132 variable elimination. */
5134 for (bl = ivs->list; bl; bl = bl->next)
5136 struct induction *v;
5139 /* Test whether it will be possible to eliminate this biv
5140 provided all givs are reduced. */
5141 bl->eliminable = loop_biv_eliminable_p (loop, bl, threshold, insn_count);
5143 /* This will be true at the end, if all givs which depend on this
5144 biv have been strength reduced.
5145 We can't (currently) eliminate the biv unless this is so. */
5146 bl->all_reduced = 1;
5148 /* Check each extension dependent giv in this class to see if its
5149 root biv is safe from wrapping in the interior mode. */
5150 check_ext_dependent_givs (loop, bl);
5152 /* Combine all giv's for this iv_class. */
5153 combine_givs (regs, bl);
5155 for (v = bl->giv; v; v = v->next_iv)
5157 struct induction *tv;
5159 if (v->ignore || v->same)
5162 benefit = loop_giv_reduce_benefit (loop, bl, v, test_reg);
5164 /* If an insn is not to be strength reduced, then set its ignore
5165 flag, and clear bl->all_reduced. */
5167 /* A giv that depends on a reversed biv must be reduced if it is
5168 used after the loop exit, otherwise, it would have the wrong
5169 value after the loop exit. To make it simple, just reduce all
5170 of such giv's whether or not we know they are used after the loop
5173 if (! flag_reduce_all_givs
5174 && v->lifetime * threshold * benefit < insn_count
5177 if (loop_dump_stream)
5178 fprintf (loop_dump_stream,
5179 "giv of insn %d not worth while, %d vs %d.\n",
5181 v->lifetime * threshold * benefit, insn_count);
5183 bl->all_reduced = 0;
5187 /* Check that we can increment the reduced giv without a
5188 multiply insn. If not, reject it. */
5190 for (tv = bl->biv; tv; tv = tv->next_iv)
5191 if (tv->mult_val == const1_rtx
5192 && ! product_cheap_p (tv->add_val, v->mult_val))
5194 if (loop_dump_stream)
5195 fprintf (loop_dump_stream,
5196 "giv of insn %d: would need a multiply.\n",
5197 INSN_UID (v->insn));
5199 bl->all_reduced = 0;
5205 /* Check for givs whose first use is their definition and whose
5206 last use is the definition of another giv. If so, it is likely
5207 dead and should not be used to derive another giv nor to
5209 loop_givs_dead_check (loop, bl);
5211 /* Reduce each giv that we decided to reduce. */
5212 loop_givs_reduce (loop, bl);
5214 /* Rescan all givs. If a giv is the same as a giv not reduced, mark it
5217 For each giv register that can be reduced now: if replaceable,
5218 substitute reduced reg wherever the old giv occurs;
5219 else add new move insn "giv_reg = reduced_reg". */
5220 loop_givs_rescan (loop, bl, reg_map);
5222 /* All the givs based on the biv bl have been reduced if they
5225 /* For each giv not marked as maybe dead that has been combined with a
5226 second giv, clear any "maybe dead" mark on that second giv.
5227 v->new_reg will either be or refer to the register of the giv it
5230 Doing this clearing avoids problems in biv elimination where
5231 a giv's new_reg is a complex value that can't be put in the
5232 insn but the giv combined with (with a reg as new_reg) is
5233 marked maybe_dead. Since the register will be used in either
5234 case, we'd prefer it be used from the simpler giv. */
5236 for (v = bl->giv; v; v = v->next_iv)
5237 if (! v->maybe_dead && v->same)
5238 v->same->maybe_dead = 0;
5240 /* Try to eliminate the biv, if it is a candidate.
5241 This won't work if ! bl->all_reduced,
5242 since the givs we planned to use might not have been reduced.
5244 We have to be careful that we didn't initially think we could
5245 eliminate this biv because of a giv that we now think may be
5246 dead and shouldn't be used as a biv replacement.
5248 Also, there is the possibility that we may have a giv that looks
5249 like it can be used to eliminate a biv, but the resulting insn
5250 isn't valid. This can happen, for example, on the 88k, where a
5251 JUMP_INSN can compare a register only with zero. Attempts to
5252 replace it with a compare with a constant will fail.
5254 Note that in cases where this call fails, we may have replaced some
5255 of the occurrences of the biv with a giv, but no harm was done in
5256 doing so in the rare cases where it can occur. */
5258 if (bl->all_reduced == 1 && bl->eliminable
5259 && maybe_eliminate_biv (loop, bl, 1, threshold, insn_count))
5261 /* ?? If we created a new test to bypass the loop entirely,
5262 or otherwise drop straight in, based on this test, then
5263 we might want to rewrite it also. This way some later
5264 pass has more hope of removing the initialization of this
5267 /* If final_value != 0, then the biv may be used after loop end
5268 and we must emit an insn to set it just in case.
5270 Reversed bivs already have an insn after the loop setting their
5271 value, so we don't need another one. We can't calculate the
5272 proper final value for such a biv here anyways. */
5273 if (bl->final_value && ! bl->reversed)
5274 loop_insn_sink_or_swim (loop,
5275 gen_load_of_final_value (bl->biv->dest_reg,
5278 if (loop_dump_stream)
5279 fprintf (loop_dump_stream, "Reg %d: biv eliminated\n",
5282 /* See above note wrt final_value. But since we couldn't eliminate
5283 the biv, we must set the value after the loop instead of before. */
5284 else if (bl->final_value && ! bl->reversed)
5285 loop_insn_sink (loop, gen_load_of_final_value (bl->biv->dest_reg,
5289 /* Go through all the instructions in the loop, making all the
5290 register substitutions scheduled in REG_MAP. */
5292 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
5293 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5294 || GET_CODE (p) == CALL_INSN)
5296 replace_regs (PATTERN (p), reg_map, reg_map_size, 0);
5297 replace_regs (REG_NOTES (p), reg_map, reg_map_size, 0);
5301 if (loop_info->n_iterations > 0)
5303 /* When we completely unroll a loop we will likely not need the increment
5304 of the loop BIV and we will not need the conditional branch at the
5306 unrolled_insn_copies = insn_count - 2;
5309 /* When we completely unroll a loop on a HAVE_cc0 machine we will not
5310 need the comparison before the conditional branch at the end of the
5312 unrolled_insn_copies -= 1;
5315 /* We'll need one copy for each loop iteration. */
5316 unrolled_insn_copies *= loop_info->n_iterations;
5318 /* A little slop to account for the ability to remove initialization
5319 code, better CSE, and other secondary benefits of completely
5320 unrolling some loops. */
5321 unrolled_insn_copies -= 1;
5323 /* Clamp the value. */
5324 if (unrolled_insn_copies < 0)
5325 unrolled_insn_copies = 0;
5328 /* Unroll loops from within strength reduction so that we can use the
5329 induction variable information that strength_reduce has already
5330 collected. Always unroll loops that would be as small or smaller
5331 unrolled than when rolled. */
5332 if ((flags & LOOP_UNROLL)
5333 || ((flags & LOOP_AUTO_UNROLL)
5334 && loop_info->n_iterations > 0
5335 && unrolled_insn_copies <= insn_count))
5336 unroll_loop (loop, insn_count, 1);
5338 #ifdef HAVE_doloop_end
5339 if (HAVE_doloop_end && (flags & LOOP_BCT) && flag_branch_on_count_reg)
5340 doloop_optimize (loop);
5341 #endif /* HAVE_doloop_end */
5343 /* In case number of iterations is known, drop branch prediction note
5344 in the branch. Do that only in second loop pass, as loop unrolling
5345 may change the number of iterations performed. */
5346 if (flags & LOOP_BCT)
5348 unsigned HOST_WIDE_INT n
5349 = loop_info->n_iterations / loop_info->unroll_number;
5351 predict_insn (prev_nonnote_insn (loop->end), PRED_LOOP_ITERATIONS,
5352 REG_BR_PROB_BASE - REG_BR_PROB_BASE / n);
5355 if (loop_dump_stream)
5356 fprintf (loop_dump_stream, "\n");
5358 loop_ivs_free (loop);
5363 /*Record all basic induction variables calculated in the insn. */
5365 check_insn_for_bivs (struct loop *loop, rtx p, int not_every_iteration,
5368 struct loop_ivs *ivs = LOOP_IVS (loop);
5375 if (GET_CODE (p) == INSN
5376 && (set = single_set (p))
5377 && GET_CODE (SET_DEST (set)) == REG)
5379 dest_reg = SET_DEST (set);
5380 if (REGNO (dest_reg) < max_reg_before_loop
5381 && REGNO (dest_reg) >= FIRST_PSEUDO_REGISTER
5382 && REG_IV_TYPE (ivs, REGNO (dest_reg)) != NOT_BASIC_INDUCT)
5384 if (basic_induction_var (loop, SET_SRC (set),
5385 GET_MODE (SET_SRC (set)),
5386 dest_reg, p, &inc_val, &mult_val,
5389 /* It is a possible basic induction variable.
5390 Create and initialize an induction structure for it. */
5392 struct induction *v = xmalloc (sizeof (struct induction));
5394 record_biv (loop, v, p, dest_reg, inc_val, mult_val, location,
5395 not_every_iteration, maybe_multiple);
5396 REG_IV_TYPE (ivs, REGNO (dest_reg)) = BASIC_INDUCT;
5398 else if (REGNO (dest_reg) < ivs->n_regs)
5399 REG_IV_TYPE (ivs, REGNO (dest_reg)) = NOT_BASIC_INDUCT;
5405 /* Record all givs calculated in the insn.
5406 A register is a giv if: it is only set once, it is a function of a
5407 biv and a constant (or invariant), and it is not a biv. */
5409 check_insn_for_givs (struct loop *loop, rtx p, int not_every_iteration,
5412 struct loop_regs *regs = LOOP_REGS (loop);
5415 /* Look for a general induction variable in a register. */
5416 if (GET_CODE (p) == INSN
5417 && (set = single_set (p))
5418 && GET_CODE (SET_DEST (set)) == REG
5419 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
5428 rtx last_consec_insn;
5430 dest_reg = SET_DEST (set);
5431 if (REGNO (dest_reg) < FIRST_PSEUDO_REGISTER)
5434 if (/* SET_SRC is a giv. */
5435 (general_induction_var (loop, SET_SRC (set), &src_reg, &add_val,
5436 &mult_val, &ext_val, 0, &benefit, VOIDmode)
5437 /* Equivalent expression is a giv. */
5438 || ((regnote = find_reg_note (p, REG_EQUAL, NULL_RTX))
5439 && general_induction_var (loop, XEXP (regnote, 0), &src_reg,
5440 &add_val, &mult_val, &ext_val, 0,
5441 &benefit, VOIDmode)))
5442 /* Don't try to handle any regs made by loop optimization.
5443 We have nothing on them in regno_first_uid, etc. */
5444 && REGNO (dest_reg) < max_reg_before_loop
5445 /* Don't recognize a BASIC_INDUCT_VAR here. */
5446 && dest_reg != src_reg
5447 /* This must be the only place where the register is set. */
5448 && (regs->array[REGNO (dest_reg)].n_times_set == 1
5449 /* or all sets must be consecutive and make a giv. */
5450 || (benefit = consec_sets_giv (loop, benefit, p,
5452 &add_val, &mult_val, &ext_val,
5453 &last_consec_insn))))
5455 struct induction *v = xmalloc (sizeof (struct induction));
5457 /* If this is a library call, increase benefit. */
5458 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
5459 benefit += libcall_benefit (p);
5461 /* Skip the consecutive insns, if there are any. */
5462 if (regs->array[REGNO (dest_reg)].n_times_set != 1)
5463 p = last_consec_insn;
5465 record_giv (loop, v, p, src_reg, dest_reg, mult_val, add_val,
5466 ext_val, benefit, DEST_REG, not_every_iteration,
5467 maybe_multiple, (rtx*) 0);
5472 /* Look for givs which are memory addresses. */
5473 if (GET_CODE (p) == INSN)
5474 find_mem_givs (loop, PATTERN (p), p, not_every_iteration,
5477 /* Update the status of whether giv can derive other givs. This can
5478 change when we pass a label or an insn that updates a biv. */
5479 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5480 || GET_CODE (p) == CODE_LABEL)
5481 update_giv_derive (loop, p);
5485 /* Return 1 if X is a valid source for an initial value (or as value being
5486 compared against in an initial test).
5488 X must be either a register or constant and must not be clobbered between
5489 the current insn and the start of the loop.
5491 INSN is the insn containing X. */
5494 valid_initial_value_p (rtx x, rtx insn, int call_seen, rtx loop_start)
5499 /* Only consider pseudos we know about initialized in insns whose luids
5501 if (GET_CODE (x) != REG
5502 || REGNO (x) >= max_reg_before_loop)
5505 /* Don't use call-clobbered registers across a call which clobbers it. On
5506 some machines, don't use any hard registers at all. */
5507 if (REGNO (x) < FIRST_PSEUDO_REGISTER
5508 && (SMALL_REGISTER_CLASSES
5509 || (call_used_regs[REGNO (x)] && call_seen)))
5512 /* Don't use registers that have been clobbered before the start of the
5514 if (reg_set_between_p (x, insn, loop_start))
5520 /* Scan X for memory refs and check each memory address
5521 as a possible giv. INSN is the insn whose pattern X comes from.
5522 NOT_EVERY_ITERATION is 1 if the insn might not be executed during
5523 every loop iteration. MAYBE_MULTIPLE is 1 if the insn might be executed
5524 more than once in each loop iteration. */
5527 find_mem_givs (const struct loop *loop, rtx x, rtx insn,
5528 int not_every_iteration, int maybe_multiple)
5537 code = GET_CODE (x);
5562 /* This code used to disable creating GIVs with mult_val == 1 and
5563 add_val == 0. However, this leads to lost optimizations when
5564 it comes time to combine a set of related DEST_ADDR GIVs, since
5565 this one would not be seen. */
5567 if (general_induction_var (loop, XEXP (x, 0), &src_reg, &add_val,
5568 &mult_val, &ext_val, 1, &benefit,
5571 /* Found one; record it. */
5572 struct induction *v = xmalloc (sizeof (struct induction));
5574 record_giv (loop, v, insn, src_reg, addr_placeholder, mult_val,
5575 add_val, ext_val, benefit, DEST_ADDR,
5576 not_every_iteration, maybe_multiple, &XEXP (x, 0));
5587 /* Recursively scan the subexpressions for other mem refs. */
5589 fmt = GET_RTX_FORMAT (code);
5590 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5592 find_mem_givs (loop, XEXP (x, i), insn, not_every_iteration,
5594 else if (fmt[i] == 'E')
5595 for (j = 0; j < XVECLEN (x, i); j++)
5596 find_mem_givs (loop, XVECEXP (x, i, j), insn, not_every_iteration,
5600 /* Fill in the data about one biv update.
5601 V is the `struct induction' in which we record the biv. (It is
5602 allocated by the caller, with alloca.)
5603 INSN is the insn that sets it.
5604 DEST_REG is the biv's reg.
5606 MULT_VAL is const1_rtx if the biv is being incremented here, in which case
5607 INC_VAL is the increment. Otherwise, MULT_VAL is const0_rtx and the biv is
5608 being set to INC_VAL.
5610 NOT_EVERY_ITERATION is nonzero if this biv update is not know to be
5611 executed every iteration; MAYBE_MULTIPLE is nonzero if this biv update
5612 can be executed more than once per iteration. If MAYBE_MULTIPLE
5613 and NOT_EVERY_ITERATION are both zero, we know that the biv update is
5614 executed exactly once per iteration. */
5617 record_biv (struct loop *loop, struct induction *v, rtx insn, rtx dest_reg,
5618 rtx inc_val, rtx mult_val, rtx *location,
5619 int not_every_iteration, int maybe_multiple)
5621 struct loop_ivs *ivs = LOOP_IVS (loop);
5622 struct iv_class *bl;
5625 v->src_reg = dest_reg;
5626 v->dest_reg = dest_reg;
5627 v->mult_val = mult_val;
5628 v->add_val = inc_val;
5629 v->ext_dependent = NULL_RTX;
5630 v->location = location;
5631 v->mode = GET_MODE (dest_reg);
5632 v->always_computable = ! not_every_iteration;
5633 v->always_executed = ! not_every_iteration;
5634 v->maybe_multiple = maybe_multiple;
5637 /* Add this to the reg's iv_class, creating a class
5638 if this is the first incrementation of the reg. */
5640 bl = REG_IV_CLASS (ivs, REGNO (dest_reg));
5643 /* Create and initialize new iv_class. */
5645 bl = xmalloc (sizeof (struct iv_class));
5647 bl->regno = REGNO (dest_reg);
5653 /* Set initial value to the reg itself. */
5654 bl->initial_value = dest_reg;
5655 bl->final_value = 0;
5656 /* We haven't seen the initializing insn yet */
5659 bl->initial_test = 0;
5660 bl->incremented = 0;
5664 bl->total_benefit = 0;
5666 /* Add this class to ivs->list. */
5667 bl->next = ivs->list;
5670 /* Put it in the array of biv register classes. */
5671 REG_IV_CLASS (ivs, REGNO (dest_reg)) = bl;
5675 /* Check if location is the same as a previous one. */
5676 struct induction *induction;
5677 for (induction = bl->biv; induction; induction = induction->next_iv)
5678 if (location == induction->location)
5680 v->same = induction;
5685 /* Update IV_CLASS entry for this biv. */
5686 v->next_iv = bl->biv;
5689 if (mult_val == const1_rtx)
5690 bl->incremented = 1;
5692 if (loop_dump_stream)
5693 loop_biv_dump (v, loop_dump_stream, 0);
5696 /* Fill in the data about one giv.
5697 V is the `struct induction' in which we record the giv. (It is
5698 allocated by the caller, with alloca.)
5699 INSN is the insn that sets it.
5700 BENEFIT estimates the savings from deleting this insn.
5701 TYPE is DEST_REG or DEST_ADDR; it says whether the giv is computed
5702 into a register or is used as a memory address.
5704 SRC_REG is the biv reg which the giv is computed from.
5705 DEST_REG is the giv's reg (if the giv is stored in a reg).
5706 MULT_VAL and ADD_VAL are the coefficients used to compute the giv.
5707 LOCATION points to the place where this giv's value appears in INSN. */
5710 record_giv (const struct loop *loop, struct induction *v, rtx insn,
5711 rtx src_reg, rtx dest_reg, rtx mult_val, rtx add_val,
5712 rtx ext_val, int benefit, enum g_types type,
5713 int not_every_iteration, int maybe_multiple, rtx *location)
5715 struct loop_ivs *ivs = LOOP_IVS (loop);
5716 struct induction *b;
5717 struct iv_class *bl;
5718 rtx set = single_set (insn);
5721 /* Attempt to prove constantness of the values. Don't let simplify_rtx
5722 undo the MULT canonicalization that we performed earlier. */
5723 temp = simplify_rtx (add_val);
5725 && ! (GET_CODE (add_val) == MULT
5726 && GET_CODE (temp) == ASHIFT))
5730 v->src_reg = src_reg;
5732 v->dest_reg = dest_reg;
5733 v->mult_val = mult_val;
5734 v->add_val = add_val;
5735 v->ext_dependent = ext_val;
5736 v->benefit = benefit;
5737 v->location = location;
5739 v->combined_with = 0;
5740 v->maybe_multiple = maybe_multiple;
5742 v->derive_adjustment = 0;
5748 v->auto_inc_opt = 0;
5752 /* The v->always_computable field is used in update_giv_derive, to
5753 determine whether a giv can be used to derive another giv. For a
5754 DEST_REG giv, INSN computes a new value for the giv, so its value
5755 isn't computable if INSN insn't executed every iteration.
5756 However, for a DEST_ADDR giv, INSN merely uses the value of the giv;
5757 it does not compute a new value. Hence the value is always computable
5758 regardless of whether INSN is executed each iteration. */
5760 if (type == DEST_ADDR)
5761 v->always_computable = 1;
5763 v->always_computable = ! not_every_iteration;
5765 v->always_executed = ! not_every_iteration;
5767 if (type == DEST_ADDR)
5769 v->mode = GET_MODE (*location);
5772 else /* type == DEST_REG */
5774 v->mode = GET_MODE (SET_DEST (set));
5776 v->lifetime = LOOP_REG_LIFETIME (loop, REGNO (dest_reg));
5778 /* If the lifetime is zero, it means that this register is
5779 really a dead store. So mark this as a giv that can be
5780 ignored. This will not prevent the biv from being eliminated. */
5781 if (v->lifetime == 0)
5784 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
5785 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
5788 /* Add the giv to the class of givs computed from one biv. */
5790 bl = REG_IV_CLASS (ivs, REGNO (src_reg));
5793 v->next_iv = bl->giv;
5795 /* Don't count DEST_ADDR. This is supposed to count the number of
5796 insns that calculate givs. */
5797 if (type == DEST_REG)
5799 bl->total_benefit += benefit;
5802 /* Fatal error, biv missing for this giv? */
5805 if (type == DEST_ADDR)
5808 v->not_replaceable = 0;
5812 /* The giv can be replaced outright by the reduced register only if all
5813 of the following conditions are true:
5814 - the insn that sets the giv is always executed on any iteration
5815 on which the giv is used at all
5816 (there are two ways to deduce this:
5817 either the insn is executed on every iteration,
5818 or all uses follow that insn in the same basic block),
5819 - the giv is not used outside the loop
5820 - no assignments to the biv occur during the giv's lifetime. */
5822 if (REGNO_FIRST_UID (REGNO (dest_reg)) == INSN_UID (insn)
5823 /* Previous line always fails if INSN was moved by loop opt. */
5824 && REGNO_LAST_LUID (REGNO (dest_reg))
5825 < INSN_LUID (loop->end)
5826 && (! not_every_iteration
5827 || last_use_this_basic_block (dest_reg, insn)))
5829 /* Now check that there are no assignments to the biv within the
5830 giv's lifetime. This requires two separate checks. */
5832 /* Check each biv update, and fail if any are between the first
5833 and last use of the giv.
5835 If this loop contains an inner loop that was unrolled, then
5836 the insn modifying the biv may have been emitted by the loop
5837 unrolling code, and hence does not have a valid luid. Just
5838 mark the biv as not replaceable in this case. It is not very
5839 useful as a biv, because it is used in two different loops.
5840 It is very unlikely that we would be able to optimize the giv
5841 using this biv anyways. */
5844 v->not_replaceable = 0;
5845 for (b = bl->biv; b; b = b->next_iv)
5847 if (INSN_UID (b->insn) >= max_uid_for_loop
5848 || ((INSN_LUID (b->insn)
5849 >= REGNO_FIRST_LUID (REGNO (dest_reg)))
5850 && (INSN_LUID (b->insn)
5851 <= REGNO_LAST_LUID (REGNO (dest_reg)))))
5854 v->not_replaceable = 1;
5859 /* If there are any backwards branches that go from after the
5860 biv update to before it, then this giv is not replaceable. */
5862 for (b = bl->biv; b; b = b->next_iv)
5863 if (back_branch_in_range_p (loop, b->insn))
5866 v->not_replaceable = 1;
5872 /* May still be replaceable, we don't have enough info here to
5875 v->not_replaceable = 0;
5879 /* Record whether the add_val contains a const_int, for later use by
5884 v->no_const_addval = 1;
5885 if (tem == const0_rtx)
5887 else if (CONSTANT_P (add_val))
5888 v->no_const_addval = 0;
5889 if (GET_CODE (tem) == PLUS)
5893 if (GET_CODE (XEXP (tem, 0)) == PLUS)
5894 tem = XEXP (tem, 0);
5895 else if (GET_CODE (XEXP (tem, 1)) == PLUS)
5896 tem = XEXP (tem, 1);
5900 if (CONSTANT_P (XEXP (tem, 1)))
5901 v->no_const_addval = 0;
5905 if (loop_dump_stream)
5906 loop_giv_dump (v, loop_dump_stream, 0);
5909 /* All this does is determine whether a giv can be made replaceable because
5910 its final value can be calculated. This code can not be part of record_giv
5911 above, because final_giv_value requires that the number of loop iterations
5912 be known, and that can not be accurately calculated until after all givs
5913 have been identified. */
5916 check_final_value (const struct loop *loop, struct induction *v)
5918 rtx final_value = 0;
5920 /* DEST_ADDR givs will never reach here, because they are always marked
5921 replaceable above in record_giv. */
5923 /* The giv can be replaced outright by the reduced register only if all
5924 of the following conditions are true:
5925 - the insn that sets the giv is always executed on any iteration
5926 on which the giv is used at all
5927 (there are two ways to deduce this:
5928 either the insn is executed on every iteration,
5929 or all uses follow that insn in the same basic block),
5930 - its final value can be calculated (this condition is different
5931 than the one above in record_giv)
5932 - it's not used before the it's set
5933 - no assignments to the biv occur during the giv's lifetime. */
5936 /* This is only called now when replaceable is known to be false. */
5937 /* Clear replaceable, so that it won't confuse final_giv_value. */
5941 if ((final_value = final_giv_value (loop, v))
5942 && (v->always_executed
5943 || last_use_this_basic_block (v->dest_reg, v->insn)))
5945 int biv_increment_seen = 0, before_giv_insn = 0;
5950 v->not_replaceable = 0;
5952 /* When trying to determine whether or not a biv increment occurs
5953 during the lifetime of the giv, we can ignore uses of the variable
5954 outside the loop because final_value is true. Hence we can not
5955 use regno_last_uid and regno_first_uid as above in record_giv. */
5957 /* Search the loop to determine whether any assignments to the
5958 biv occur during the giv's lifetime. Start with the insn
5959 that sets the giv, and search around the loop until we come
5960 back to that insn again.
5962 Also fail if there is a jump within the giv's lifetime that jumps
5963 to somewhere outside the lifetime but still within the loop. This
5964 catches spaghetti code where the execution order is not linear, and
5965 hence the above test fails. Here we assume that the giv lifetime
5966 does not extend from one iteration of the loop to the next, so as
5967 to make the test easier. Since the lifetime isn't known yet,
5968 this requires two loops. See also record_giv above. */
5970 last_giv_use = v->insn;
5977 before_giv_insn = 1;
5978 p = NEXT_INSN (loop->start);
5983 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5984 || GET_CODE (p) == CALL_INSN)
5986 /* It is possible for the BIV increment to use the GIV if we
5987 have a cycle. Thus we must be sure to check each insn for
5988 both BIV and GIV uses, and we must check for BIV uses
5991 if (! biv_increment_seen
5992 && reg_set_p (v->src_reg, PATTERN (p)))
5993 biv_increment_seen = 1;
5995 if (reg_mentioned_p (v->dest_reg, PATTERN (p)))
5997 if (biv_increment_seen || before_giv_insn)
6000 v->not_replaceable = 1;
6008 /* Now that the lifetime of the giv is known, check for branches
6009 from within the lifetime to outside the lifetime if it is still
6019 p = NEXT_INSN (loop->start);
6020 if (p == last_giv_use)
6023 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
6024 && LABEL_NAME (JUMP_LABEL (p))
6025 && ((loop_insn_first_p (JUMP_LABEL (p), v->insn)
6026 && loop_insn_first_p (loop->start, JUMP_LABEL (p)))
6027 || (loop_insn_first_p (last_giv_use, JUMP_LABEL (p))
6028 && loop_insn_first_p (JUMP_LABEL (p), loop->end))))
6031 v->not_replaceable = 1;
6033 if (loop_dump_stream)
6034 fprintf (loop_dump_stream,
6035 "Found branch outside giv lifetime.\n");
6042 /* If it is replaceable, then save the final value. */
6044 v->final_value = final_value;
6047 if (loop_dump_stream && v->replaceable)
6048 fprintf (loop_dump_stream, "Insn %d: giv reg %d final_value replaceable\n",
6049 INSN_UID (v->insn), REGNO (v->dest_reg));
6052 /* Update the status of whether a giv can derive other givs.
6054 We need to do something special if there is or may be an update to the biv
6055 between the time the giv is defined and the time it is used to derive
6058 In addition, a giv that is only conditionally set is not allowed to
6059 derive another giv once a label has been passed.
6061 The cases we look at are when a label or an update to a biv is passed. */
6064 update_giv_derive (const struct loop *loop, rtx p)
6066 struct loop_ivs *ivs = LOOP_IVS (loop);
6067 struct iv_class *bl;
6068 struct induction *biv, *giv;
6072 /* Search all IV classes, then all bivs, and finally all givs.
6074 There are three cases we are concerned with. First we have the situation
6075 of a giv that is only updated conditionally. In that case, it may not
6076 derive any givs after a label is passed.
6078 The second case is when a biv update occurs, or may occur, after the
6079 definition of a giv. For certain biv updates (see below) that are
6080 known to occur between the giv definition and use, we can adjust the
6081 giv definition. For others, or when the biv update is conditional,
6082 we must prevent the giv from deriving any other givs. There are two
6083 sub-cases within this case.
6085 If this is a label, we are concerned with any biv update that is done
6086 conditionally, since it may be done after the giv is defined followed by
6087 a branch here (actually, we need to pass both a jump and a label, but
6088 this extra tracking doesn't seem worth it).
6090 If this is a jump, we are concerned about any biv update that may be
6091 executed multiple times. We are actually only concerned about
6092 backward jumps, but it is probably not worth performing the test
6093 on the jump again here.
6095 If this is a biv update, we must adjust the giv status to show that a
6096 subsequent biv update was performed. If this adjustment cannot be done,
6097 the giv cannot derive further givs. */
6099 for (bl = ivs->list; bl; bl = bl->next)
6100 for (biv = bl->biv; biv; biv = biv->next_iv)
6101 if (GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN
6104 /* Skip if location is the same as a previous one. */
6108 for (giv = bl->giv; giv; giv = giv->next_iv)
6110 /* If cant_derive is already true, there is no point in
6111 checking all of these conditions again. */
6112 if (giv->cant_derive)
6115 /* If this giv is conditionally set and we have passed a label,
6116 it cannot derive anything. */
6117 if (GET_CODE (p) == CODE_LABEL && ! giv->always_computable)
6118 giv->cant_derive = 1;
6120 /* Skip givs that have mult_val == 0, since
6121 they are really invariants. Also skip those that are
6122 replaceable, since we know their lifetime doesn't contain
6124 else if (giv->mult_val == const0_rtx || giv->replaceable)
6127 /* The only way we can allow this giv to derive another
6128 is if this is a biv increment and we can form the product
6129 of biv->add_val and giv->mult_val. In this case, we will
6130 be able to compute a compensation. */
6131 else if (biv->insn == p)
6136 if (biv->mult_val == const1_rtx)
6137 tem = simplify_giv_expr (loop,
6138 gen_rtx_MULT (giv->mode,
6141 &ext_val_dummy, &dummy);
6143 if (tem && giv->derive_adjustment)
6144 tem = simplify_giv_expr
6146 gen_rtx_PLUS (giv->mode, tem, giv->derive_adjustment),
6147 &ext_val_dummy, &dummy);
6150 giv->derive_adjustment = tem;
6152 giv->cant_derive = 1;
6154 else if ((GET_CODE (p) == CODE_LABEL && ! biv->always_computable)
6155 || (GET_CODE (p) == JUMP_INSN && biv->maybe_multiple))
6156 giv->cant_derive = 1;
6161 /* Check whether an insn is an increment legitimate for a basic induction var.
6162 X is the source of insn P, or a part of it.
6163 MODE is the mode in which X should be interpreted.
6165 DEST_REG is the putative biv, also the destination of the insn.
6166 We accept patterns of these forms:
6167 REG = REG + INVARIANT (includes REG = REG - CONSTANT)
6168 REG = INVARIANT + REG
6170 If X is suitable, we return 1, set *MULT_VAL to CONST1_RTX,
6171 store the additive term into *INC_VAL, and store the place where
6172 we found the additive term into *LOCATION.
6174 If X is an assignment of an invariant into DEST_REG, we set
6175 *MULT_VAL to CONST0_RTX, and store the invariant into *INC_VAL.
6177 We also want to detect a BIV when it corresponds to a variable
6178 whose mode was promoted via PROMOTED_MODE. In that case, an increment
6179 of the variable may be a PLUS that adds a SUBREG of that variable to
6180 an invariant and then sign- or zero-extends the result of the PLUS
6183 Most GIVs in such cases will be in the promoted mode, since that is the
6184 probably the natural computation mode (and almost certainly the mode
6185 used for addresses) on the machine. So we view the pseudo-reg containing
6186 the variable as the BIV, as if it were simply incremented.
6188 Note that treating the entire pseudo as a BIV will result in making
6189 simple increments to any GIVs based on it. However, if the variable
6190 overflows in its declared mode but not its promoted mode, the result will
6191 be incorrect. This is acceptable if the variable is signed, since
6192 overflows in such cases are undefined, but not if it is unsigned, since
6193 those overflows are defined. So we only check for SIGN_EXTEND and
6196 If we cannot find a biv, we return 0. */
6199 basic_induction_var (const struct loop *loop, rtx x, enum machine_mode mode,
6200 rtx dest_reg, rtx p, rtx *inc_val, rtx *mult_val,
6205 rtx insn, set = 0, last, inc;
6207 code = GET_CODE (x);
6212 if (rtx_equal_p (XEXP (x, 0), dest_reg)
6213 || (GET_CODE (XEXP (x, 0)) == SUBREG
6214 && SUBREG_PROMOTED_VAR_P (XEXP (x, 0))
6215 && SUBREG_REG (XEXP (x, 0)) == dest_reg))
6217 argp = &XEXP (x, 1);
6219 else if (rtx_equal_p (XEXP (x, 1), dest_reg)
6220 || (GET_CODE (XEXP (x, 1)) == SUBREG
6221 && SUBREG_PROMOTED_VAR_P (XEXP (x, 1))
6222 && SUBREG_REG (XEXP (x, 1)) == dest_reg))
6224 argp = &XEXP (x, 0);
6230 if (loop_invariant_p (loop, arg) != 1)
6233 /* convert_modes can emit new instructions, e.g. when arg is a loop
6234 invariant MEM and dest_reg has a different mode.
6235 These instructions would be emitted after the end of the function
6236 and then *inc_val would be an uninitialized pseudo.
6237 Detect this and bail in this case.
6238 Other alternatives to solve this can be introducing a convert_modes
6239 variant which is allowed to fail but not allowed to emit new
6240 instructions, emit these instructions before loop start and let
6241 it be garbage collected if *inc_val is never used or saving the
6242 *inc_val initialization sequence generated here and when *inc_val
6243 is going to be actually used, emit it at some suitable place. */
6244 last = get_last_insn ();
6245 inc = convert_modes (GET_MODE (dest_reg), GET_MODE (x), arg, 0);
6246 if (get_last_insn () != last)
6248 delete_insns_since (last);
6253 *mult_val = const1_rtx;
6258 /* If what's inside the SUBREG is a BIV, then the SUBREG. This will
6259 handle addition of promoted variables.
6260 ??? The comment at the start of this function is wrong: promoted
6261 variable increments don't look like it says they do. */
6262 return basic_induction_var (loop, SUBREG_REG (x),
6263 GET_MODE (SUBREG_REG (x)),
6264 dest_reg, p, inc_val, mult_val, location);
6267 /* If this register is assigned in a previous insn, look at its
6268 source, but don't go outside the loop or past a label. */
6270 /* If this sets a register to itself, we would repeat any previous
6271 biv increment if we applied this strategy blindly. */
6272 if (rtx_equal_p (dest_reg, x))
6281 insn = PREV_INSN (insn);
6283 while (insn && GET_CODE (insn) == NOTE
6284 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6288 set = single_set (insn);
6291 dest = SET_DEST (set);
6293 || (GET_CODE (dest) == SUBREG
6294 && (GET_MODE_SIZE (GET_MODE (dest)) <= UNITS_PER_WORD)
6295 && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_INT)
6296 && SUBREG_REG (dest) == x))
6297 return basic_induction_var (loop, SET_SRC (set),
6298 (GET_MODE (SET_SRC (set)) == VOIDmode
6300 : GET_MODE (SET_SRC (set))),
6302 inc_val, mult_val, location);
6304 while (GET_CODE (dest) == SIGN_EXTRACT
6305 || GET_CODE (dest) == ZERO_EXTRACT
6306 || GET_CODE (dest) == SUBREG
6307 || GET_CODE (dest) == STRICT_LOW_PART)
6308 dest = XEXP (dest, 0);
6314 /* Can accept constant setting of biv only when inside inner most loop.
6315 Otherwise, a biv of an inner loop may be incorrectly recognized
6316 as a biv of the outer loop,
6317 causing code to be moved INTO the inner loop. */
6319 if (loop_invariant_p (loop, x) != 1)
6324 /* convert_modes aborts if we try to convert to or from CCmode, so just
6325 exclude that case. It is very unlikely that a condition code value
6326 would be a useful iterator anyways. convert_modes aborts if we try to
6327 convert a float mode to non-float or vice versa too. */
6328 if (loop->level == 1
6329 && GET_MODE_CLASS (mode) == GET_MODE_CLASS (GET_MODE (dest_reg))
6330 && GET_MODE_CLASS (mode) != MODE_CC)
6332 /* Possible bug here? Perhaps we don't know the mode of X. */
6333 last = get_last_insn ();
6334 inc = convert_modes (GET_MODE (dest_reg), mode, x, 0);
6335 if (get_last_insn () != last)
6337 delete_insns_since (last);
6342 *mult_val = const0_rtx;
6349 /* Ignore this BIV if signed arithmetic overflow is defined. */
6352 return basic_induction_var (loop, XEXP (x, 0), GET_MODE (XEXP (x, 0)),
6353 dest_reg, p, inc_val, mult_val, location);
6356 /* Similar, since this can be a sign extension. */
6357 for (insn = PREV_INSN (p);
6358 (insn && GET_CODE (insn) == NOTE
6359 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6360 insn = PREV_INSN (insn))
6364 set = single_set (insn);
6366 if (! rtx_equal_p (dest_reg, XEXP (x, 0))
6367 && set && SET_DEST (set) == XEXP (x, 0)
6368 && GET_CODE (XEXP (x, 1)) == CONST_INT
6369 && INTVAL (XEXP (x, 1)) >= 0
6370 && GET_CODE (SET_SRC (set)) == ASHIFT
6371 && XEXP (x, 1) == XEXP (SET_SRC (set), 1))
6372 return basic_induction_var (loop, XEXP (SET_SRC (set), 0),
6373 GET_MODE (XEXP (x, 0)),
6374 dest_reg, insn, inc_val, mult_val,
6383 /* A general induction variable (giv) is any quantity that is a linear
6384 function of a basic induction variable,
6385 i.e. giv = biv * mult_val + add_val.
6386 The coefficients can be any loop invariant quantity.
6387 A giv need not be computed directly from the biv;
6388 it can be computed by way of other givs. */
6390 /* Determine whether X computes a giv.
6391 If it does, return a nonzero value
6392 which is the benefit from eliminating the computation of X;
6393 set *SRC_REG to the register of the biv that it is computed from;
6394 set *ADD_VAL and *MULT_VAL to the coefficients,
6395 such that the value of X is biv * mult + add; */
6398 general_induction_var (const struct loop *loop, rtx x, rtx *src_reg,
6399 rtx *add_val, rtx *mult_val, rtx *ext_val,
6400 int is_addr, int *pbenefit,
6401 enum machine_mode addr_mode)
6403 struct loop_ivs *ivs = LOOP_IVS (loop);
6406 /* If this is an invariant, forget it, it isn't a giv. */
6407 if (loop_invariant_p (loop, x) == 1)
6411 *ext_val = NULL_RTX;
6412 x = simplify_giv_expr (loop, x, ext_val, pbenefit);
6416 switch (GET_CODE (x))
6420 /* Since this is now an invariant and wasn't before, it must be a giv
6421 with MULT_VAL == 0. It doesn't matter which BIV we associate this
6423 *src_reg = ivs->list->biv->dest_reg;
6424 *mult_val = const0_rtx;
6429 /* This is equivalent to a BIV. */
6431 *mult_val = const1_rtx;
6432 *add_val = const0_rtx;
6436 /* Either (plus (biv) (invar)) or
6437 (plus (mult (biv) (invar_1)) (invar_2)). */
6438 if (GET_CODE (XEXP (x, 0)) == MULT)
6440 *src_reg = XEXP (XEXP (x, 0), 0);
6441 *mult_val = XEXP (XEXP (x, 0), 1);
6445 *src_reg = XEXP (x, 0);
6446 *mult_val = const1_rtx;
6448 *add_val = XEXP (x, 1);
6452 /* ADD_VAL is zero. */
6453 *src_reg = XEXP (x, 0);
6454 *mult_val = XEXP (x, 1);
6455 *add_val = const0_rtx;
6462 /* Remove any enclosing USE from ADD_VAL and MULT_VAL (there will be
6463 unless they are CONST_INT). */
6464 if (GET_CODE (*add_val) == USE)
6465 *add_val = XEXP (*add_val, 0);
6466 if (GET_CODE (*mult_val) == USE)
6467 *mult_val = XEXP (*mult_val, 0);
6470 *pbenefit += address_cost (orig_x, addr_mode) - reg_address_cost;
6472 *pbenefit += rtx_cost (orig_x, SET);
6474 /* Always return true if this is a giv so it will be detected as such,
6475 even if the benefit is zero or negative. This allows elimination
6476 of bivs that might otherwise not be eliminated. */
6480 /* Given an expression, X, try to form it as a linear function of a biv.
6481 We will canonicalize it to be of the form
6482 (plus (mult (BIV) (invar_1))
6484 with possible degeneracies.
6486 The invariant expressions must each be of a form that can be used as a
6487 machine operand. We surround then with a USE rtx (a hack, but localized
6488 and certainly unambiguous!) if not a CONST_INT for simplicity in this
6489 routine; it is the caller's responsibility to strip them.
6491 If no such canonicalization is possible (i.e., two biv's are used or an
6492 expression that is neither invariant nor a biv or giv), this routine
6495 For a nonzero return, the result will have a code of CONST_INT, USE,
6496 REG (for a BIV), PLUS, or MULT. No other codes will occur.
6498 *BENEFIT will be incremented by the benefit of any sub-giv encountered. */
6500 static rtx sge_plus (enum machine_mode, rtx, rtx);
6501 static rtx sge_plus_constant (rtx, rtx);
6504 simplify_giv_expr (const struct loop *loop, rtx x, rtx *ext_val, int *benefit)
6506 struct loop_ivs *ivs = LOOP_IVS (loop);
6507 struct loop_regs *regs = LOOP_REGS (loop);
6508 enum machine_mode mode = GET_MODE (x);
6512 /* If this is not an integer mode, or if we cannot do arithmetic in this
6513 mode, this can't be a giv. */
6514 if (mode != VOIDmode
6515 && (GET_MODE_CLASS (mode) != MODE_INT
6516 || GET_MODE_BITSIZE (mode) > HOST_BITS_PER_WIDE_INT))
6519 switch (GET_CODE (x))
6522 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6523 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6524 if (arg0 == 0 || arg1 == 0)
6527 /* Put constant last, CONST_INT last if both constant. */
6528 if ((GET_CODE (arg0) == USE
6529 || GET_CODE (arg0) == CONST_INT)
6530 && ! ((GET_CODE (arg0) == USE
6531 && GET_CODE (arg1) == USE)
6532 || GET_CODE (arg1) == CONST_INT))
6533 tem = arg0, arg0 = arg1, arg1 = tem;
6535 /* Handle addition of zero, then addition of an invariant. */
6536 if (arg1 == const0_rtx)
6538 else if (GET_CODE (arg1) == CONST_INT || GET_CODE (arg1) == USE)
6539 switch (GET_CODE (arg0))
6543 /* Adding two invariants must result in an invariant, so enclose
6544 addition operation inside a USE and return it. */
6545 if (GET_CODE (arg0) == USE)
6546 arg0 = XEXP (arg0, 0);
6547 if (GET_CODE (arg1) == USE)
6548 arg1 = XEXP (arg1, 0);
6550 if (GET_CODE (arg0) == CONST_INT)
6551 tem = arg0, arg0 = arg1, arg1 = tem;
6552 if (GET_CODE (arg1) == CONST_INT)
6553 tem = sge_plus_constant (arg0, arg1);
6555 tem = sge_plus (mode, arg0, arg1);
6557 if (GET_CODE (tem) != CONST_INT)
6558 tem = gen_rtx_USE (mode, tem);
6563 /* biv + invar or mult + invar. Return sum. */
6564 return gen_rtx_PLUS (mode, arg0, arg1);
6567 /* (a + invar_1) + invar_2. Associate. */
6569 simplify_giv_expr (loop,
6581 /* Each argument must be either REG, PLUS, or MULT. Convert REG to
6582 MULT to reduce cases. */
6583 if (GET_CODE (arg0) == REG)
6584 arg0 = gen_rtx_MULT (mode, arg0, const1_rtx);
6585 if (GET_CODE (arg1) == REG)
6586 arg1 = gen_rtx_MULT (mode, arg1, const1_rtx);
6588 /* Now have PLUS + PLUS, PLUS + MULT, MULT + PLUS, or MULT + MULT.
6589 Put a MULT first, leaving PLUS + PLUS, MULT + PLUS, or MULT + MULT.
6590 Recurse to associate the second PLUS. */
6591 if (GET_CODE (arg1) == MULT)
6592 tem = arg0, arg0 = arg1, arg1 = tem;
6594 if (GET_CODE (arg1) == PLUS)
6596 simplify_giv_expr (loop,
6598 gen_rtx_PLUS (mode, arg0,
6603 /* Now must have MULT + MULT. Distribute if same biv, else not giv. */
6604 if (GET_CODE (arg0) != MULT || GET_CODE (arg1) != MULT)
6607 if (!rtx_equal_p (arg0, arg1))
6610 return simplify_giv_expr (loop,
6619 /* Handle "a - b" as "a + b * (-1)". */
6620 return simplify_giv_expr (loop,
6629 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6630 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6631 if (arg0 == 0 || arg1 == 0)
6634 /* Put constant last, CONST_INT last if both constant. */
6635 if ((GET_CODE (arg0) == USE || GET_CODE (arg0) == CONST_INT)
6636 && GET_CODE (arg1) != CONST_INT)
6637 tem = arg0, arg0 = arg1, arg1 = tem;
6639 /* If second argument is not now constant, not giv. */
6640 if (GET_CODE (arg1) != USE && GET_CODE (arg1) != CONST_INT)
6643 /* Handle multiply by 0 or 1. */
6644 if (arg1 == const0_rtx)
6647 else if (arg1 == const1_rtx)
6650 switch (GET_CODE (arg0))
6653 /* biv * invar. Done. */
6654 return gen_rtx_MULT (mode, arg0, arg1);
6657 /* Product of two constants. */
6658 return GEN_INT (INTVAL (arg0) * INTVAL (arg1));
6661 /* invar * invar is a giv, but attempt to simplify it somehow. */
6662 if (GET_CODE (arg1) != CONST_INT)
6665 arg0 = XEXP (arg0, 0);
6666 if (GET_CODE (arg0) == MULT)
6668 /* (invar_0 * invar_1) * invar_2. Associate. */
6669 return simplify_giv_expr (loop,
6678 /* Propagate the MULT expressions to the innermost nodes. */
6679 else if (GET_CODE (arg0) == PLUS)
6681 /* (invar_0 + invar_1) * invar_2. Distribute. */
6682 return simplify_giv_expr (loop,
6694 return gen_rtx_USE (mode, gen_rtx_MULT (mode, arg0, arg1));
6697 /* (a * invar_1) * invar_2. Associate. */
6698 return simplify_giv_expr (loop,
6707 /* (a + invar_1) * invar_2. Distribute. */
6708 return simplify_giv_expr (loop,
6723 /* Shift by constant is multiply by power of two. */
6724 if (GET_CODE (XEXP (x, 1)) != CONST_INT)
6728 simplify_giv_expr (loop,
6731 GEN_INT ((HOST_WIDE_INT) 1
6732 << INTVAL (XEXP (x, 1)))),
6736 /* "-a" is "a * (-1)" */
6737 return simplify_giv_expr (loop,
6738 gen_rtx_MULT (mode, XEXP (x, 0), constm1_rtx),
6742 /* "~a" is "-a - 1". Silly, but easy. */
6743 return simplify_giv_expr (loop,
6744 gen_rtx_MINUS (mode,
6745 gen_rtx_NEG (mode, XEXP (x, 0)),
6750 /* Already in proper form for invariant. */
6756 /* Conditionally recognize extensions of simple IVs. After we've
6757 computed loop traversal counts and verified the range of the
6758 source IV, we'll reevaluate this as a GIV. */
6759 if (*ext_val == NULL_RTX)
6761 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6762 if (arg0 && *ext_val == NULL_RTX && GET_CODE (arg0) == REG)
6764 *ext_val = gen_rtx_fmt_e (GET_CODE (x), mode, arg0);
6771 /* If this is a new register, we can't deal with it. */
6772 if (REGNO (x) >= max_reg_before_loop)
6775 /* Check for biv or giv. */
6776 switch (REG_IV_TYPE (ivs, REGNO (x)))
6780 case GENERAL_INDUCT:
6782 struct induction *v = REG_IV_INFO (ivs, REGNO (x));
6784 /* Form expression from giv and add benefit. Ensure this giv
6785 can derive another and subtract any needed adjustment if so. */
6787 /* Increasing the benefit here is risky. The only case in which it
6788 is arguably correct is if this is the only use of V. In other
6789 cases, this will artificially inflate the benefit of the current
6790 giv, and lead to suboptimal code. Thus, it is disabled, since
6791 potentially not reducing an only marginally beneficial giv is
6792 less harmful than reducing many givs that are not really
6795 rtx single_use = regs->array[REGNO (x)].single_usage;
6796 if (single_use && single_use != const0_rtx)
6797 *benefit += v->benefit;
6803 tem = gen_rtx_PLUS (mode, gen_rtx_MULT (mode,
6804 v->src_reg, v->mult_val),
6807 if (v->derive_adjustment)
6808 tem = gen_rtx_MINUS (mode, tem, v->derive_adjustment);
6809 arg0 = simplify_giv_expr (loop, tem, ext_val, benefit);
6812 if (!v->ext_dependent)
6817 *ext_val = v->ext_dependent;
6825 /* If it isn't an induction variable, and it is invariant, we
6826 may be able to simplify things further by looking through
6827 the bits we just moved outside the loop. */
6828 if (loop_invariant_p (loop, x) == 1)
6831 struct loop_movables *movables = LOOP_MOVABLES (loop);
6833 for (m = movables->head; m; m = m->next)
6834 if (rtx_equal_p (x, m->set_dest))
6836 /* Ok, we found a match. Substitute and simplify. */
6838 /* If we match another movable, we must use that, as
6839 this one is going away. */
6841 return simplify_giv_expr (loop, m->match->set_dest,
6844 /* If consec is nonzero, this is a member of a group of
6845 instructions that were moved together. We handle this
6846 case only to the point of seeking to the last insn and
6847 looking for a REG_EQUAL. Fail if we don't find one. */
6854 tem = NEXT_INSN (tem);
6858 tem = find_reg_note (tem, REG_EQUAL, NULL_RTX);
6860 tem = XEXP (tem, 0);
6864 tem = single_set (m->insn);
6866 tem = SET_SRC (tem);
6871 /* What we are most interested in is pointer
6872 arithmetic on invariants -- only take
6873 patterns we may be able to do something with. */
6874 if (GET_CODE (tem) == PLUS
6875 || GET_CODE (tem) == MULT
6876 || GET_CODE (tem) == ASHIFT
6877 || GET_CODE (tem) == CONST_INT
6878 || GET_CODE (tem) == SYMBOL_REF)
6880 tem = simplify_giv_expr (loop, tem, ext_val,
6885 else if (GET_CODE (tem) == CONST
6886 && GET_CODE (XEXP (tem, 0)) == PLUS
6887 && GET_CODE (XEXP (XEXP (tem, 0), 0)) == SYMBOL_REF
6888 && GET_CODE (XEXP (XEXP (tem, 0), 1)) == CONST_INT)
6890 tem = simplify_giv_expr (loop, XEXP (tem, 0),
6902 /* Fall through to general case. */
6904 /* If invariant, return as USE (unless CONST_INT).
6905 Otherwise, not giv. */
6906 if (GET_CODE (x) == USE)
6909 if (loop_invariant_p (loop, x) == 1)
6911 if (GET_CODE (x) == CONST_INT)
6913 if (GET_CODE (x) == CONST
6914 && GET_CODE (XEXP (x, 0)) == PLUS
6915 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
6916 && GET_CODE (XEXP (XEXP (x, 0), 1)) == CONST_INT)
6918 return gen_rtx_USE (mode, x);
6925 /* This routine folds invariants such that there is only ever one
6926 CONST_INT in the summation. It is only used by simplify_giv_expr. */
6929 sge_plus_constant (rtx x, rtx c)
6931 if (GET_CODE (x) == CONST_INT)
6932 return GEN_INT (INTVAL (x) + INTVAL (c));
6933 else if (GET_CODE (x) != PLUS)
6934 return gen_rtx_PLUS (GET_MODE (x), x, c);
6935 else if (GET_CODE (XEXP (x, 1)) == CONST_INT)
6937 return gen_rtx_PLUS (GET_MODE (x), XEXP (x, 0),
6938 GEN_INT (INTVAL (XEXP (x, 1)) + INTVAL (c)));
6940 else if (GET_CODE (XEXP (x, 0)) == PLUS
6941 || GET_CODE (XEXP (x, 1)) != PLUS)
6943 return gen_rtx_PLUS (GET_MODE (x),
6944 sge_plus_constant (XEXP (x, 0), c), XEXP (x, 1));
6948 return gen_rtx_PLUS (GET_MODE (x),
6949 sge_plus_constant (XEXP (x, 1), c), XEXP (x, 0));
6954 sge_plus (enum machine_mode mode, rtx x, rtx y)
6956 while (GET_CODE (y) == PLUS)
6958 rtx a = XEXP (y, 0);
6959 if (GET_CODE (a) == CONST_INT)
6960 x = sge_plus_constant (x, a);
6962 x = gen_rtx_PLUS (mode, x, a);
6965 if (GET_CODE (y) == CONST_INT)
6966 x = sge_plus_constant (x, y);
6968 x = gen_rtx_PLUS (mode, x, y);
6972 /* Help detect a giv that is calculated by several consecutive insns;
6976 The caller has already identified the first insn P as having a giv as dest;
6977 we check that all other insns that set the same register follow
6978 immediately after P, that they alter nothing else,
6979 and that the result of the last is still a giv.
6981 The value is 0 if the reg set in P is not really a giv.
6982 Otherwise, the value is the amount gained by eliminating
6983 all the consecutive insns that compute the value.
6985 FIRST_BENEFIT is the amount gained by eliminating the first insn, P.
6986 SRC_REG is the reg of the biv; DEST_REG is the reg of the giv.
6988 The coefficients of the ultimate giv value are stored in
6989 *MULT_VAL and *ADD_VAL. */
6992 consec_sets_giv (const struct loop *loop, int first_benefit, rtx p,
6993 rtx src_reg, rtx dest_reg, rtx *add_val, rtx *mult_val,
6994 rtx *ext_val, rtx *last_consec_insn)
6996 struct loop_ivs *ivs = LOOP_IVS (loop);
6997 struct loop_regs *regs = LOOP_REGS (loop);
7004 /* Indicate that this is a giv so that we can update the value produced in
7005 each insn of the multi-insn sequence.
7007 This induction structure will be used only by the call to
7008 general_induction_var below, so we can allocate it on our stack.
7009 If this is a giv, our caller will replace the induct var entry with
7010 a new induction structure. */
7011 struct induction *v;
7013 if (REG_IV_TYPE (ivs, REGNO (dest_reg)) != UNKNOWN_INDUCT)
7016 v = alloca (sizeof (struct induction));
7017 v->src_reg = src_reg;
7018 v->mult_val = *mult_val;
7019 v->add_val = *add_val;
7020 v->benefit = first_benefit;
7022 v->derive_adjustment = 0;
7023 v->ext_dependent = NULL_RTX;
7025 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
7026 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
7028 count = regs->array[REGNO (dest_reg)].n_times_set - 1;
7033 code = GET_CODE (p);
7035 /* If libcall, skip to end of call sequence. */
7036 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
7040 && (set = single_set (p))
7041 && GET_CODE (SET_DEST (set)) == REG
7042 && SET_DEST (set) == dest_reg
7043 && (general_induction_var (loop, SET_SRC (set), &src_reg,
7044 add_val, mult_val, ext_val, 0,
7046 /* Giv created by equivalent expression. */
7047 || ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX))
7048 && general_induction_var (loop, XEXP (temp, 0), &src_reg,
7049 add_val, mult_val, ext_val, 0,
7050 &benefit, VOIDmode)))
7051 && src_reg == v->src_reg)
7053 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
7054 benefit += libcall_benefit (p);
7057 v->mult_val = *mult_val;
7058 v->add_val = *add_val;
7059 v->benefit += benefit;
7061 else if (code != NOTE)
7063 /* Allow insns that set something other than this giv to a
7064 constant. Such insns are needed on machines which cannot
7065 include long constants and should not disqualify a giv. */
7067 && (set = single_set (p))
7068 && SET_DEST (set) != dest_reg
7069 && CONSTANT_P (SET_SRC (set)))
7072 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7077 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7078 *last_consec_insn = p;
7082 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7083 represented by G1. If no such expression can be found, or it is clear that
7084 it cannot possibly be a valid address, 0 is returned.
7086 To perform the computation, we note that
7089 where `v' is the biv.
7091 So G2 = (y/b) * G1 + (b - a*y/x).
7093 Note that MULT = y/x.
7095 Update: A and B are now allowed to be additive expressions such that
7096 B contains all variables in A. That is, computing B-A will not require
7097 subtracting variables. */
7100 express_from_1 (rtx a, rtx b, rtx mult)
7102 /* If MULT is zero, then A*MULT is zero, and our expression is B. */
7104 if (mult == const0_rtx)
7107 /* If MULT is not 1, we cannot handle A with non-constants, since we
7108 would then be required to subtract multiples of the registers in A.
7109 This is theoretically possible, and may even apply to some Fortran
7110 constructs, but it is a lot of work and we do not attempt it here. */
7112 if (mult != const1_rtx && GET_CODE (a) != CONST_INT)
7115 /* In general these structures are sorted top to bottom (down the PLUS
7116 chain), but not left to right across the PLUS. If B is a higher
7117 order giv than A, we can strip one level and recurse. If A is higher
7118 order, we'll eventually bail out, but won't know that until the end.
7119 If they are the same, we'll strip one level around this loop. */
7121 while (GET_CODE (a) == PLUS && GET_CODE (b) == PLUS)
7123 rtx ra, rb, oa, ob, tmp;
7125 ra = XEXP (a, 0), oa = XEXP (a, 1);
7126 if (GET_CODE (ra) == PLUS)
7127 tmp = ra, ra = oa, oa = tmp;
7129 rb = XEXP (b, 0), ob = XEXP (b, 1);
7130 if (GET_CODE (rb) == PLUS)
7131 tmp = rb, rb = ob, ob = tmp;
7133 if (rtx_equal_p (ra, rb))
7134 /* We matched: remove one reg completely. */
7136 else if (GET_CODE (ob) != PLUS && rtx_equal_p (ra, ob))
7137 /* An alternate match. */
7139 else if (GET_CODE (oa) != PLUS && rtx_equal_p (oa, rb))
7140 /* An alternate match. */
7144 /* Indicates an extra register in B. Strip one level from B and
7145 recurse, hoping B was the higher order expression. */
7146 ob = express_from_1 (a, ob, mult);
7149 return gen_rtx_PLUS (GET_MODE (b), rb, ob);
7153 /* Here we are at the last level of A, go through the cases hoping to
7154 get rid of everything but a constant. */
7156 if (GET_CODE (a) == PLUS)
7160 ra = XEXP (a, 0), oa = XEXP (a, 1);
7161 if (rtx_equal_p (oa, b))
7163 else if (!rtx_equal_p (ra, b))
7166 if (GET_CODE (oa) != CONST_INT)
7169 return GEN_INT (-INTVAL (oa) * INTVAL (mult));
7171 else if (GET_CODE (a) == CONST_INT)
7173 return plus_constant (b, -INTVAL (a) * INTVAL (mult));
7175 else if (CONSTANT_P (a))
7177 enum machine_mode mode_a = GET_MODE (a);
7178 enum machine_mode mode_b = GET_MODE (b);
7179 enum machine_mode mode = mode_b == VOIDmode ? mode_a : mode_b;
7180 return simplify_gen_binary (MINUS, mode, b, a);
7182 else if (GET_CODE (b) == PLUS)
7184 if (rtx_equal_p (a, XEXP (b, 0)))
7186 else if (rtx_equal_p (a, XEXP (b, 1)))
7191 else if (rtx_equal_p (a, b))
7198 express_from (struct induction *g1, struct induction *g2)
7202 /* The value that G1 will be multiplied by must be a constant integer. Also,
7203 the only chance we have of getting a valid address is if b*c/a (see above
7204 for notation) is also an integer. */
7205 if (GET_CODE (g1->mult_val) == CONST_INT
7206 && GET_CODE (g2->mult_val) == CONST_INT)
7208 if (g1->mult_val == const0_rtx
7209 || (g1->mult_val == constm1_rtx
7210 && INTVAL (g2->mult_val)
7211 == (HOST_WIDE_INT) 1 << (HOST_BITS_PER_WIDE_INT - 1))
7212 || INTVAL (g2->mult_val) % INTVAL (g1->mult_val) != 0)
7214 mult = GEN_INT (INTVAL (g2->mult_val) / INTVAL (g1->mult_val));
7216 else if (rtx_equal_p (g1->mult_val, g2->mult_val))
7220 /* ??? Find out if the one is a multiple of the other? */
7224 add = express_from_1 (g1->add_val, g2->add_val, mult);
7225 if (add == NULL_RTX)
7227 /* Failed. If we've got a multiplication factor between G1 and G2,
7228 scale G1's addend and try again. */
7229 if (INTVAL (mult) > 1)
7231 rtx g1_add_val = g1->add_val;
7232 if (GET_CODE (g1_add_val) == MULT
7233 && GET_CODE (XEXP (g1_add_val, 1)) == CONST_INT)
7236 m = INTVAL (mult) * INTVAL (XEXP (g1_add_val, 1));
7237 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val),
7238 XEXP (g1_add_val, 0), GEN_INT (m));
7242 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val), g1_add_val,
7246 add = express_from_1 (g1_add_val, g2->add_val, const1_rtx);
7249 if (add == NULL_RTX)
7252 /* Form simplified final result. */
7253 if (mult == const0_rtx)
7255 else if (mult == const1_rtx)
7256 mult = g1->dest_reg;
7258 mult = gen_rtx_MULT (g2->mode, g1->dest_reg, mult);
7260 if (add == const0_rtx)
7264 if (GET_CODE (add) == PLUS
7265 && CONSTANT_P (XEXP (add, 1)))
7267 rtx tem = XEXP (add, 1);
7268 mult = gen_rtx_PLUS (g2->mode, mult, XEXP (add, 0));
7272 return gen_rtx_PLUS (g2->mode, mult, add);
7276 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7277 represented by G1. This indicates that G2 should be combined with G1 and
7278 that G2 can use (either directly or via an address expression) a register
7279 used to represent G1. */
7282 combine_givs_p (struct induction *g1, struct induction *g2)
7286 /* With the introduction of ext dependent givs, we must care for modes.
7287 G2 must not use a wider mode than G1. */
7288 if (GET_MODE_SIZE (g1->mode) < GET_MODE_SIZE (g2->mode))
7291 ret = comb = express_from (g1, g2);
7292 if (comb == NULL_RTX)
7294 if (g1->mode != g2->mode)
7295 ret = gen_lowpart (g2->mode, comb);
7297 /* If these givs are identical, they can be combined. We use the results
7298 of express_from because the addends are not in a canonical form, so
7299 rtx_equal_p is a weaker test. */
7300 /* But don't combine a DEST_REG giv with a DEST_ADDR giv; we want the
7301 combination to be the other way round. */
7302 if (comb == g1->dest_reg
7303 && (g1->giv_type == DEST_REG || g2->giv_type == DEST_ADDR))
7308 /* If G2 can be expressed as a function of G1 and that function is valid
7309 as an address and no more expensive than using a register for G2,
7310 the expression of G2 in terms of G1 can be used. */
7312 && g2->giv_type == DEST_ADDR
7313 && memory_address_p (GET_MODE (g2->mem), ret))
7319 /* Check each extension dependent giv in this class to see if its
7320 root biv is safe from wrapping in the interior mode, which would
7321 make the giv illegal. */
7324 check_ext_dependent_givs (const struct loop *loop, struct iv_class *bl)
7326 struct loop_info *loop_info = LOOP_INFO (loop);
7327 int ze_ok = 0, se_ok = 0, info_ok = 0;
7328 enum machine_mode biv_mode = GET_MODE (bl->biv->src_reg);
7329 HOST_WIDE_INT start_val;
7330 unsigned HOST_WIDE_INT u_end_val = 0;
7331 unsigned HOST_WIDE_INT u_start_val = 0;
7333 struct induction *v;
7335 /* Make sure the iteration data is available. We must have
7336 constants in order to be certain of no overflow. */
7337 if (loop_info->n_iterations > 0
7338 && bl->initial_value
7339 && GET_CODE (bl->initial_value) == CONST_INT
7340 && (incr = biv_total_increment (bl))
7341 && GET_CODE (incr) == CONST_INT
7342 /* Make sure the host can represent the arithmetic. */
7343 && HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (biv_mode))
7345 unsigned HOST_WIDE_INT abs_incr, total_incr;
7346 HOST_WIDE_INT s_end_val;
7350 start_val = INTVAL (bl->initial_value);
7351 u_start_val = start_val;
7353 neg_incr = 0, abs_incr = INTVAL (incr);
7354 if (INTVAL (incr) < 0)
7355 neg_incr = 1, abs_incr = -abs_incr;
7356 total_incr = abs_incr * loop_info->n_iterations;
7358 /* Check for host arithmetic overflow. */
7359 if (total_incr / loop_info->n_iterations == abs_incr)
7361 unsigned HOST_WIDE_INT u_max;
7362 HOST_WIDE_INT s_max;
7364 u_end_val = start_val + (neg_incr ? -total_incr : total_incr);
7365 s_end_val = u_end_val;
7366 u_max = GET_MODE_MASK (biv_mode);
7369 /* Check zero extension of biv ok. */
7371 /* Check for host arithmetic overflow. */
7373 ? u_end_val < u_start_val
7374 : u_end_val > u_start_val)
7375 /* Check for target arithmetic overflow. */
7377 ? 1 /* taken care of with host overflow */
7378 : u_end_val <= u_max))
7383 /* Check sign extension of biv ok. */
7384 /* ??? While it is true that overflow with signed and pointer
7385 arithmetic is undefined, I fear too many programmers don't
7386 keep this fact in mind -- myself included on occasion.
7387 So leave alone with the signed overflow optimizations. */
7388 if (start_val >= -s_max - 1
7389 /* Check for host arithmetic overflow. */
7391 ? s_end_val < start_val
7392 : s_end_val > start_val)
7393 /* Check for target arithmetic overflow. */
7395 ? s_end_val >= -s_max - 1
7396 : s_end_val <= s_max))
7403 /* If we know the BIV is compared at run-time against an
7404 invariant value, and the increment is +/- 1, we may also
7405 be able to prove that the BIV cannot overflow. */
7406 else if (bl->biv->src_reg == loop_info->iteration_var
7407 && loop_info->comparison_value
7408 && loop_invariant_p (loop, loop_info->comparison_value)
7409 && (incr = biv_total_increment (bl))
7410 && GET_CODE (incr) == CONST_INT)
7412 /* If the increment is +1, and the exit test is a <,
7413 the BIV cannot overflow. (For <=, we have the
7414 problematic case that the comparison value might
7415 be the maximum value of the range.) */
7416 if (INTVAL (incr) == 1)
7418 if (loop_info->comparison_code == LT)
7420 else if (loop_info->comparison_code == LTU)
7424 /* Likewise for increment -1 and exit test >. */
7425 if (INTVAL (incr) == -1)
7427 if (loop_info->comparison_code == GT)
7429 else if (loop_info->comparison_code == GTU)
7434 /* Invalidate givs that fail the tests. */
7435 for (v = bl->giv; v; v = v->next_iv)
7436 if (v->ext_dependent)
7438 enum rtx_code code = GET_CODE (v->ext_dependent);
7451 /* We don't know whether this value is being used as either
7452 signed or unsigned, so to safely truncate we must satisfy
7453 both. The initial check here verifies the BIV itself;
7454 once that is successful we may check its range wrt the
7455 derived GIV. This works only if we were able to determine
7456 constant start and end values above. */
7457 if (se_ok && ze_ok && info_ok)
7459 enum machine_mode outer_mode = GET_MODE (v->ext_dependent);
7460 unsigned HOST_WIDE_INT max = GET_MODE_MASK (outer_mode) >> 1;
7462 /* We know from the above that both endpoints are nonnegative,
7463 and that there is no wrapping. Verify that both endpoints
7464 are within the (signed) range of the outer mode. */
7465 if (u_start_val <= max && u_end_val <= max)
7476 if (loop_dump_stream)
7478 fprintf (loop_dump_stream,
7479 "Verified ext dependent giv at %d of reg %d\n",
7480 INSN_UID (v->insn), bl->regno);
7485 if (loop_dump_stream)
7490 why = "biv iteration values overflowed";
7494 incr = biv_total_increment (bl);
7495 if (incr == const1_rtx)
7496 why = "biv iteration info incomplete; incr by 1";
7498 why = "biv iteration info incomplete";
7501 fprintf (loop_dump_stream,
7502 "Failed ext dependent giv at %d, %s\n",
7503 INSN_UID (v->insn), why);
7506 bl->all_reduced = 0;
7511 /* Generate a version of VALUE in a mode appropriate for initializing V. */
7514 extend_value_for_giv (struct induction *v, rtx value)
7516 rtx ext_dep = v->ext_dependent;
7521 /* Recall that check_ext_dependent_givs verified that the known bounds
7522 of a biv did not overflow or wrap with respect to the extension for
7523 the giv. Therefore, constants need no additional adjustment. */
7524 if (CONSTANT_P (value) && GET_MODE (value) == VOIDmode)
7527 /* Otherwise, we must adjust the value to compensate for the
7528 differing modes of the biv and the giv. */
7529 return gen_rtx_fmt_e (GET_CODE (ext_dep), GET_MODE (ext_dep), value);
7532 struct combine_givs_stats
7539 cmp_combine_givs_stats (const void *xp, const void *yp)
7541 const struct combine_givs_stats * const x =
7542 (const struct combine_givs_stats *) xp;
7543 const struct combine_givs_stats * const y =
7544 (const struct combine_givs_stats *) yp;
7546 d = y->total_benefit - x->total_benefit;
7547 /* Stabilize the sort. */
7549 d = x->giv_number - y->giv_number;
7553 /* Check all pairs of givs for iv_class BL and see if any can be combined with
7554 any other. If so, point SAME to the giv combined with and set NEW_REG to
7555 be an expression (in terms of the other giv's DEST_REG) equivalent to the
7556 giv. Also, update BENEFIT and related fields for cost/benefit analysis. */
7559 combine_givs (struct loop_regs *regs, struct iv_class *bl)
7561 /* Additional benefit to add for being combined multiple times. */
7562 const int extra_benefit = 3;
7564 struct induction *g1, *g2, **giv_array;
7565 int i, j, k, giv_count;
7566 struct combine_givs_stats *stats;
7569 /* Count givs, because bl->giv_count is incorrect here. */
7571 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7575 giv_array = alloca (giv_count * sizeof (struct induction *));
7577 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7579 giv_array[i++] = g1;
7581 stats = xcalloc (giv_count, sizeof (*stats));
7582 can_combine = xcalloc (giv_count, giv_count * sizeof (rtx));
7584 for (i = 0; i < giv_count; i++)
7590 stats[i].giv_number = i;
7592 /* If a DEST_REG GIV is used only once, do not allow it to combine
7593 with anything, for in doing so we will gain nothing that cannot
7594 be had by simply letting the GIV with which we would have combined
7595 to be reduced on its own. The losage shows up in particular with
7596 DEST_ADDR targets on hosts with reg+reg addressing, though it can
7597 be seen elsewhere as well. */
7598 if (g1->giv_type == DEST_REG
7599 && (single_use = regs->array[REGNO (g1->dest_reg)].single_usage)
7600 && single_use != const0_rtx)
7603 this_benefit = g1->benefit;
7604 /* Add an additional weight for zero addends. */
7605 if (g1->no_const_addval)
7608 for (j = 0; j < giv_count; j++)
7614 && (this_combine = combine_givs_p (g1, g2)) != NULL_RTX)
7616 can_combine[i * giv_count + j] = this_combine;
7617 this_benefit += g2->benefit + extra_benefit;
7620 stats[i].total_benefit = this_benefit;
7623 /* Iterate, combining until we can't. */
7625 qsort (stats, giv_count, sizeof (*stats), cmp_combine_givs_stats);
7627 if (loop_dump_stream)
7629 fprintf (loop_dump_stream, "Sorted combine statistics:\n");
7630 for (k = 0; k < giv_count; k++)
7632 g1 = giv_array[stats[k].giv_number];
7633 if (!g1->combined_with && !g1->same)
7634 fprintf (loop_dump_stream, " {%d, %d}",
7635 INSN_UID (giv_array[stats[k].giv_number]->insn),
7636 stats[k].total_benefit);
7638 putc ('\n', loop_dump_stream);
7641 for (k = 0; k < giv_count; k++)
7643 int g1_add_benefit = 0;
7645 i = stats[k].giv_number;
7648 /* If it has already been combined, skip. */
7649 if (g1->combined_with || g1->same)
7652 for (j = 0; j < giv_count; j++)
7655 if (g1 != g2 && can_combine[i * giv_count + j]
7656 /* If it has already been combined, skip. */
7657 && ! g2->same && ! g2->combined_with)
7661 g2->new_reg = can_combine[i * giv_count + j];
7663 /* For destination, we now may replace by mem expression instead
7664 of register. This changes the costs considerably, so add the
7666 if (g2->giv_type == DEST_ADDR)
7667 g2->benefit = (g2->benefit + reg_address_cost
7668 - address_cost (g2->new_reg,
7669 GET_MODE (g2->mem)));
7670 g1->combined_with++;
7671 g1->lifetime += g2->lifetime;
7673 g1_add_benefit += g2->benefit;
7675 /* ??? The new final_[bg]iv_value code does a much better job
7676 of finding replaceable giv's, and hence this code may no
7677 longer be necessary. */
7678 if (! g2->replaceable && REG_USERVAR_P (g2->dest_reg))
7679 g1_add_benefit -= copy_cost;
7681 /* To help optimize the next set of combinations, remove
7682 this giv from the benefits of other potential mates. */
7683 for (l = 0; l < giv_count; ++l)
7685 int m = stats[l].giv_number;
7686 if (can_combine[m * giv_count + j])
7687 stats[l].total_benefit -= g2->benefit + extra_benefit;
7690 if (loop_dump_stream)
7691 fprintf (loop_dump_stream,
7692 "giv at %d combined with giv at %d; new benefit %d + %d, lifetime %d\n",
7693 INSN_UID (g2->insn), INSN_UID (g1->insn),
7694 g1->benefit, g1_add_benefit, g1->lifetime);
7698 /* To help optimize the next set of combinations, remove
7699 this giv from the benefits of other potential mates. */
7700 if (g1->combined_with)
7702 for (j = 0; j < giv_count; ++j)
7704 int m = stats[j].giv_number;
7705 if (can_combine[m * giv_count + i])
7706 stats[j].total_benefit -= g1->benefit + extra_benefit;
7709 g1->benefit += g1_add_benefit;
7711 /* We've finished with this giv, and everything it touched.
7712 Restart the combination so that proper weights for the
7713 rest of the givs are properly taken into account. */
7714 /* ??? Ideally we would compact the arrays at this point, so
7715 as to not cover old ground. But sanely compacting
7716 can_combine is tricky. */
7726 /* Generate sequence for REG = B * M + A. B is the initial value of
7727 the basic induction variable, M a multiplicative constant, A an
7728 additive constant and REG the destination register. */
7731 gen_add_mult (rtx b, rtx m, rtx a, rtx reg)
7737 /* Use unsigned arithmetic. */
7738 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7740 emit_move_insn (reg, result);
7748 /* Update registers created in insn sequence SEQ. */
7751 loop_regs_update (const struct loop *loop ATTRIBUTE_UNUSED, rtx seq)
7755 /* Update register info for alias analysis. */
7758 while (insn != NULL_RTX)
7760 rtx set = single_set (insn);
7762 if (set && GET_CODE (SET_DEST (set)) == REG)
7763 record_base_value (REGNO (SET_DEST (set)), SET_SRC (set), 0);
7765 insn = NEXT_INSN (insn);
7770 /* EMIT code before BEFORE_BB/BEFORE_INSN to set REG = B * M + A. B
7771 is the initial value of the basic induction variable, M a
7772 multiplicative constant, A an additive constant and REG the
7773 destination register. */
7776 loop_iv_add_mult_emit_before (const struct loop *loop, rtx b, rtx m, rtx a,
7777 rtx reg, basic_block before_bb, rtx before_insn)
7783 loop_iv_add_mult_hoist (loop, b, m, a, reg);
7787 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7788 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7790 /* Increase the lifetime of any invariants moved further in code. */
7791 update_reg_last_use (a, before_insn);
7792 update_reg_last_use (b, before_insn);
7793 update_reg_last_use (m, before_insn);
7795 /* It is possible that the expansion created lots of new registers.
7796 Iterate over the sequence we just created and record them all. We
7797 must do this before inserting the sequence. */
7798 loop_regs_update (loop, seq);
7800 loop_insn_emit_before (loop, before_bb, before_insn, seq);
7804 /* Emit insns in loop pre-header to set REG = B * M + A. B is the
7805 initial value of the basic induction variable, M a multiplicative
7806 constant, A an additive constant and REG the destination
7810 loop_iv_add_mult_sink (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7814 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7815 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7817 /* Increase the lifetime of any invariants moved further in code.
7818 ???? Is this really necessary? */
7819 update_reg_last_use (a, loop->sink);
7820 update_reg_last_use (b, loop->sink);
7821 update_reg_last_use (m, loop->sink);
7823 /* It is possible that the expansion created lots of new registers.
7824 Iterate over the sequence we just created and record them all. We
7825 must do this before inserting the sequence. */
7826 loop_regs_update (loop, seq);
7828 loop_insn_sink (loop, seq);
7832 /* Emit insns after loop to set REG = B * M + A. B is the initial
7833 value of the basic induction variable, M a multiplicative constant,
7834 A an additive constant and REG the destination register. */
7837 loop_iv_add_mult_hoist (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7841 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7842 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7844 /* It is possible that the expansion created lots of new registers.
7845 Iterate over the sequence we just created and record them all. We
7846 must do this before inserting the sequence. */
7847 loop_regs_update (loop, seq);
7849 loop_insn_hoist (loop, seq);
7854 /* Similar to gen_add_mult, but compute cost rather than generating
7858 iv_add_mult_cost (rtx b, rtx m, rtx a, rtx reg)
7864 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7866 emit_move_insn (reg, result);
7867 last = get_last_insn ();
7870 rtx t = single_set (last);
7872 cost += rtx_cost (SET_SRC (t), SET);
7873 last = PREV_INSN (last);
7879 /* Test whether A * B can be computed without
7880 an actual multiply insn. Value is 1 if so.
7882 ??? This function stinks because it generates a ton of wasted RTL
7883 ??? and as a result fragments GC memory to no end. There are other
7884 ??? places in the compiler which are invoked a lot and do the same
7885 ??? thing, generate wasted RTL just to see if something is possible. */
7888 product_cheap_p (rtx a, rtx b)
7893 /* If only one is constant, make it B. */
7894 if (GET_CODE (a) == CONST_INT)
7895 tmp = a, a = b, b = tmp;
7897 /* If first constant, both constant, so don't need multiply. */
7898 if (GET_CODE (a) == CONST_INT)
7901 /* If second not constant, neither is constant, so would need multiply. */
7902 if (GET_CODE (b) != CONST_INT)
7905 /* One operand is constant, so might not need multiply insn. Generate the
7906 code for the multiply and see if a call or multiply, or long sequence
7907 of insns is generated. */
7910 expand_mult (GET_MODE (a), a, b, NULL_RTX, 1);
7918 while (tmp != NULL_RTX)
7920 rtx next = NEXT_INSN (tmp);
7923 || GET_CODE (tmp) != INSN
7924 || (GET_CODE (PATTERN (tmp)) == SET
7925 && GET_CODE (SET_SRC (PATTERN (tmp))) == MULT)
7926 || (GET_CODE (PATTERN (tmp)) == PARALLEL
7927 && GET_CODE (XVECEXP (PATTERN (tmp), 0, 0)) == SET
7928 && GET_CODE (SET_SRC (XVECEXP (PATTERN (tmp), 0, 0))) == MULT))
7937 else if (GET_CODE (tmp) == SET
7938 && GET_CODE (SET_SRC (tmp)) == MULT)
7940 else if (GET_CODE (tmp) == PARALLEL
7941 && GET_CODE (XVECEXP (tmp, 0, 0)) == SET
7942 && GET_CODE (SET_SRC (XVECEXP (tmp, 0, 0))) == MULT)
7948 /* Check to see if loop can be terminated by a "decrement and branch until
7949 zero" instruction. If so, add a REG_NONNEG note to the branch insn if so.
7950 Also try reversing an increment loop to a decrement loop
7951 to see if the optimization can be performed.
7952 Value is nonzero if optimization was performed. */
7954 /* This is useful even if the architecture doesn't have such an insn,
7955 because it might change a loops which increments from 0 to n to a loop
7956 which decrements from n to 0. A loop that decrements to zero is usually
7957 faster than one that increments from zero. */
7959 /* ??? This could be rewritten to use some of the loop unrolling procedures,
7960 such as approx_final_value, biv_total_increment, loop_iterations, and
7961 final_[bg]iv_value. */
7964 check_dbra_loop (struct loop *loop, int insn_count)
7966 struct loop_info *loop_info = LOOP_INFO (loop);
7967 struct loop_regs *regs = LOOP_REGS (loop);
7968 struct loop_ivs *ivs = LOOP_IVS (loop);
7969 struct iv_class *bl;
7976 rtx before_comparison;
7980 int compare_and_branch;
7981 rtx loop_start = loop->start;
7982 rtx loop_end = loop->end;
7984 /* If last insn is a conditional branch, and the insn before tests a
7985 register value, try to optimize it. Otherwise, we can't do anything. */
7987 jump = PREV_INSN (loop_end);
7988 comparison = get_condition_for_loop (loop, jump);
7989 if (comparison == 0)
7991 if (!onlyjump_p (jump))
7994 /* Try to compute whether the compare/branch at the loop end is one or
7995 two instructions. */
7996 get_condition (jump, &first_compare, false);
7997 if (first_compare == jump)
7998 compare_and_branch = 1;
7999 else if (first_compare == prev_nonnote_insn (jump))
8000 compare_and_branch = 2;
8005 /* If more than one condition is present to control the loop, then
8006 do not proceed, as this function does not know how to rewrite
8007 loop tests with more than one condition.
8009 Look backwards from the first insn in the last comparison
8010 sequence and see if we've got another comparison sequence. */
8013 if ((jump1 = prev_nonnote_insn (first_compare)) != loop->cont)
8014 if (GET_CODE (jump1) == JUMP_INSN)
8018 /* Check all of the bivs to see if the compare uses one of them.
8019 Skip biv's set more than once because we can't guarantee that
8020 it will be zero on the last iteration. Also skip if the biv is
8021 used between its update and the test insn. */
8023 for (bl = ivs->list; bl; bl = bl->next)
8025 if (bl->biv_count == 1
8026 && ! bl->biv->maybe_multiple
8027 && bl->biv->dest_reg == XEXP (comparison, 0)
8028 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8033 /* Try swapping the comparison to identify a suitable biv. */
8035 for (bl = ivs->list; bl; bl = bl->next)
8036 if (bl->biv_count == 1
8037 && ! bl->biv->maybe_multiple
8038 && bl->biv->dest_reg == XEXP (comparison, 1)
8039 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8042 comparison = gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)),
8044 XEXP (comparison, 1),
8045 XEXP (comparison, 0));
8052 /* Look for the case where the basic induction variable is always
8053 nonnegative, and equals zero on the last iteration.
8054 In this case, add a reg_note REG_NONNEG, which allows the
8055 m68k DBRA instruction to be used. */
8057 if (((GET_CODE (comparison) == GT && XEXP (comparison, 1) == constm1_rtx)
8058 || (GET_CODE (comparison) == NE && XEXP (comparison, 1) == const0_rtx))
8059 && GET_CODE (bl->biv->add_val) == CONST_INT
8060 && INTVAL (bl->biv->add_val) < 0)
8062 /* Initial value must be greater than 0,
8063 init_val % -dec_value == 0 to ensure that it equals zero on
8064 the last iteration */
8066 if (GET_CODE (bl->initial_value) == CONST_INT
8067 && INTVAL (bl->initial_value) > 0
8068 && (INTVAL (bl->initial_value)
8069 % (-INTVAL (bl->biv->add_val))) == 0)
8071 /* register always nonnegative, add REG_NOTE to branch */
8072 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8074 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8081 /* If the decrement is 1 and the value was tested as >= 0 before
8082 the loop, then we can safely optimize. */
8083 for (p = loop_start; p; p = PREV_INSN (p))
8085 if (GET_CODE (p) == CODE_LABEL)
8087 if (GET_CODE (p) != JUMP_INSN)
8090 before_comparison = get_condition_for_loop (loop, p);
8091 if (before_comparison
8092 && XEXP (before_comparison, 0) == bl->biv->dest_reg
8093 && (GET_CODE (before_comparison) == LT
8094 || GET_CODE (before_comparison) == LTU)
8095 && XEXP (before_comparison, 1) == const0_rtx
8096 && ! reg_set_between_p (bl->biv->dest_reg, p, loop_start)
8097 && INTVAL (bl->biv->add_val) == -1)
8099 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8101 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8109 else if (GET_CODE (bl->biv->add_val) == CONST_INT
8110 && INTVAL (bl->biv->add_val) > 0)
8112 /* Try to change inc to dec, so can apply above optimization. */
8114 all registers modified are induction variables or invariant,
8115 all memory references have non-overlapping addresses
8116 (obviously true if only one write)
8117 allow 2 insns for the compare/jump at the end of the loop. */
8118 /* Also, we must avoid any instructions which use both the reversed
8119 biv and another biv. Such instructions will fail if the loop is
8120 reversed. We meet this condition by requiring that either
8121 no_use_except_counting is true, or else that there is only
8123 int num_nonfixed_reads = 0;
8124 /* 1 if the iteration var is used only to count iterations. */
8125 int no_use_except_counting = 0;
8126 /* 1 if the loop has no memory store, or it has a single memory store
8127 which is reversible. */
8128 int reversible_mem_store = 1;
8130 if (bl->giv_count == 0
8131 && !loop->exit_count
8132 && !loop_info->has_multiple_exit_targets)
8134 rtx bivreg = regno_reg_rtx[bl->regno];
8135 struct iv_class *blt;
8137 /* If there are no givs for this biv, and the only exit is the
8138 fall through at the end of the loop, then
8139 see if perhaps there are no uses except to count. */
8140 no_use_except_counting = 1;
8141 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8144 rtx set = single_set (p);
8146 if (set && GET_CODE (SET_DEST (set)) == REG
8147 && REGNO (SET_DEST (set)) == bl->regno)
8148 /* An insn that sets the biv is okay. */
8150 else if (!reg_mentioned_p (bivreg, PATTERN (p)))
8151 /* An insn that doesn't mention the biv is okay. */
8153 else if (p == prev_nonnote_insn (prev_nonnote_insn (loop_end))
8154 || p == prev_nonnote_insn (loop_end))
8156 /* If either of these insns uses the biv and sets a pseudo
8157 that has more than one usage, then the biv has uses
8158 other than counting since it's used to derive a value
8159 that is used more than one time. */
8160 note_stores (PATTERN (p), note_set_pseudo_multiple_uses,
8162 if (regs->multiple_uses)
8164 no_use_except_counting = 0;
8170 no_use_except_counting = 0;
8175 /* A biv has uses besides counting if it is used to set
8177 for (blt = ivs->list; blt; blt = blt->next)
8179 && reg_mentioned_p (bivreg, SET_SRC (blt->init_set)))
8181 no_use_except_counting = 0;
8186 if (no_use_except_counting)
8187 /* No need to worry about MEMs. */
8189 else if (loop_info->num_mem_sets <= 1)
8191 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8193 num_nonfixed_reads += count_nonfixed_reads (loop, PATTERN (p));
8195 /* If the loop has a single store, and the destination address is
8196 invariant, then we can't reverse the loop, because this address
8197 might then have the wrong value at loop exit.
8198 This would work if the source was invariant also, however, in that
8199 case, the insn should have been moved out of the loop. */
8201 if (loop_info->num_mem_sets == 1)
8203 struct induction *v;
8205 /* If we could prove that each of the memory locations
8206 written to was different, then we could reverse the
8207 store -- but we don't presently have any way of
8209 reversible_mem_store = 0;
8211 /* If the store depends on a register that is set after the
8212 store, it depends on the initial value, and is thus not
8214 for (v = bl->giv; reversible_mem_store && v; v = v->next_iv)
8216 if (v->giv_type == DEST_REG
8217 && reg_mentioned_p (v->dest_reg,
8218 PATTERN (loop_info->first_loop_store_insn))
8219 && loop_insn_first_p (loop_info->first_loop_store_insn,
8221 reversible_mem_store = 0;
8228 /* This code only acts for innermost loops. Also it simplifies
8229 the memory address check by only reversing loops with
8230 zero or one memory access.
8231 Two memory accesses could involve parts of the same array,
8232 and that can't be reversed.
8233 If the biv is used only for counting, than we don't need to worry
8234 about all these things. */
8236 if ((num_nonfixed_reads <= 1
8237 && ! loop_info->has_nonconst_call
8238 && ! loop_info->has_prefetch
8239 && ! loop_info->has_volatile
8240 && reversible_mem_store
8241 && (bl->giv_count + bl->biv_count + loop_info->num_mem_sets
8242 + num_unmoved_movables (loop) + compare_and_branch == insn_count)
8243 && (bl == ivs->list && bl->next == 0))
8244 || (no_use_except_counting && ! loop_info->has_prefetch))
8248 /* Loop can be reversed. */
8249 if (loop_dump_stream)
8250 fprintf (loop_dump_stream, "Can reverse loop\n");
8252 /* Now check other conditions:
8254 The increment must be a constant, as must the initial value,
8255 and the comparison code must be LT.
8257 This test can probably be improved since +/- 1 in the constant
8258 can be obtained by changing LT to LE and vice versa; this is
8262 /* for constants, LE gets turned into LT */
8263 && (GET_CODE (comparison) == LT
8264 || (GET_CODE (comparison) == LE
8265 && no_use_except_counting)
8266 || GET_CODE (comparison) == LTU))
8268 HOST_WIDE_INT add_val, add_adjust, comparison_val = 0;
8269 rtx initial_value, comparison_value;
8271 enum rtx_code cmp_code;
8272 int comparison_const_width;
8273 unsigned HOST_WIDE_INT comparison_sign_mask;
8275 add_val = INTVAL (bl->biv->add_val);
8276 comparison_value = XEXP (comparison, 1);
8277 if (GET_MODE (comparison_value) == VOIDmode)
8278 comparison_const_width
8279 = GET_MODE_BITSIZE (GET_MODE (XEXP (comparison, 0)));
8281 comparison_const_width
8282 = GET_MODE_BITSIZE (GET_MODE (comparison_value));
8283 if (comparison_const_width > HOST_BITS_PER_WIDE_INT)
8284 comparison_const_width = HOST_BITS_PER_WIDE_INT;
8285 comparison_sign_mask
8286 = (unsigned HOST_WIDE_INT) 1 << (comparison_const_width - 1);
8288 /* If the comparison value is not a loop invariant, then we
8289 can not reverse this loop.
8291 ??? If the insns which initialize the comparison value as
8292 a whole compute an invariant result, then we could move
8293 them out of the loop and proceed with loop reversal. */
8294 if (! loop_invariant_p (loop, comparison_value))
8297 if (GET_CODE (comparison_value) == CONST_INT)
8298 comparison_val = INTVAL (comparison_value);
8299 initial_value = bl->initial_value;
8301 /* Normalize the initial value if it is an integer and
8302 has no other use except as a counter. This will allow
8303 a few more loops to be reversed. */
8304 if (no_use_except_counting
8305 && GET_CODE (comparison_value) == CONST_INT
8306 && GET_CODE (initial_value) == CONST_INT)
8308 comparison_val = comparison_val - INTVAL (bl->initial_value);
8309 /* The code below requires comparison_val to be a multiple
8310 of add_val in order to do the loop reversal, so
8311 round up comparison_val to a multiple of add_val.
8312 Since comparison_value is constant, we know that the
8313 current comparison code is LT. */
8314 comparison_val = comparison_val + add_val - 1;
8316 -= (unsigned HOST_WIDE_INT) comparison_val % add_val;
8317 /* We postpone overflow checks for COMPARISON_VAL here;
8318 even if there is an overflow, we might still be able to
8319 reverse the loop, if converting the loop exit test to
8321 initial_value = const0_rtx;
8324 /* First check if we can do a vanilla loop reversal. */
8325 if (initial_value == const0_rtx
8326 /* If we have a decrement_and_branch_on_count,
8327 prefer the NE test, since this will allow that
8328 instruction to be generated. Note that we must
8329 use a vanilla loop reversal if the biv is used to
8330 calculate a giv or has a non-counting use. */
8331 #if ! defined (HAVE_decrement_and_branch_until_zero) \
8332 && defined (HAVE_decrement_and_branch_on_count)
8333 && (! (add_val == 1 && loop->vtop
8334 && (bl->biv_count == 0
8335 || no_use_except_counting)))
8337 && GET_CODE (comparison_value) == CONST_INT
8338 /* Now do postponed overflow checks on COMPARISON_VAL. */
8339 && ! (((comparison_val - add_val) ^ INTVAL (comparison_value))
8340 & comparison_sign_mask))
8342 /* Register will always be nonnegative, with value
8343 0 on last iteration */
8344 add_adjust = add_val;
8348 else if (add_val == 1 && loop->vtop
8349 && (bl->biv_count == 0
8350 || no_use_except_counting))
8358 if (GET_CODE (comparison) == LE)
8359 add_adjust -= add_val;
8361 /* If the initial value is not zero, or if the comparison
8362 value is not an exact multiple of the increment, then we
8363 can not reverse this loop. */
8364 if (initial_value == const0_rtx
8365 && GET_CODE (comparison_value) == CONST_INT)
8367 if (((unsigned HOST_WIDE_INT) comparison_val % add_val) != 0)
8372 if (! no_use_except_counting || add_val != 1)
8376 final_value = comparison_value;
8378 /* Reset these in case we normalized the initial value
8379 and comparison value above. */
8380 if (GET_CODE (comparison_value) == CONST_INT
8381 && GET_CODE (initial_value) == CONST_INT)
8383 comparison_value = GEN_INT (comparison_val);
8385 = GEN_INT (comparison_val + INTVAL (bl->initial_value));
8387 bl->initial_value = initial_value;
8389 /* Save some info needed to produce the new insns. */
8390 reg = bl->biv->dest_reg;
8391 jump_label = condjump_label (PREV_INSN (loop_end));
8392 new_add_val = GEN_INT (-INTVAL (bl->biv->add_val));
8394 /* Set start_value; if this is not a CONST_INT, we need
8396 Initialize biv to start_value before loop start.
8397 The old initializing insn will be deleted as a
8398 dead store by flow.c. */
8399 if (initial_value == const0_rtx
8400 && GET_CODE (comparison_value) == CONST_INT)
8402 start_value = GEN_INT (comparison_val - add_adjust);
8403 loop_insn_hoist (loop, gen_move_insn (reg, start_value));
8405 else if (GET_CODE (initial_value) == CONST_INT)
8407 enum machine_mode mode = GET_MODE (reg);
8408 rtx offset = GEN_INT (-INTVAL (initial_value) - add_adjust);
8409 rtx add_insn = gen_add3_insn (reg, comparison_value, offset);
8415 = gen_rtx_PLUS (mode, comparison_value, offset);
8416 loop_insn_hoist (loop, add_insn);
8417 if (GET_CODE (comparison) == LE)
8418 final_value = gen_rtx_PLUS (mode, comparison_value,
8421 else if (! add_adjust)
8423 enum machine_mode mode = GET_MODE (reg);
8424 rtx sub_insn = gen_sub3_insn (reg, comparison_value,
8430 = gen_rtx_MINUS (mode, comparison_value, initial_value);
8431 loop_insn_hoist (loop, sub_insn);
8434 /* We could handle the other cases too, but it'll be
8435 better to have a testcase first. */
8438 /* We may not have a single insn which can increment a reg, so
8439 create a sequence to hold all the insns from expand_inc. */
8441 expand_inc (reg, new_add_val);
8445 p = loop_insn_emit_before (loop, 0, bl->biv->insn, tem);
8446 delete_insn (bl->biv->insn);
8448 /* Update biv info to reflect its new status. */
8450 bl->initial_value = start_value;
8451 bl->biv->add_val = new_add_val;
8453 /* Update loop info. */
8454 loop_info->initial_value = reg;
8455 loop_info->initial_equiv_value = reg;
8456 loop_info->final_value = const0_rtx;
8457 loop_info->final_equiv_value = const0_rtx;
8458 loop_info->comparison_value = const0_rtx;
8459 loop_info->comparison_code = cmp_code;
8460 loop_info->increment = new_add_val;
8462 /* Inc LABEL_NUSES so that delete_insn will
8463 not delete the label. */
8464 LABEL_NUSES (XEXP (jump_label, 0))++;
8466 /* Emit an insn after the end of the loop to set the biv's
8467 proper exit value if it is used anywhere outside the loop. */
8468 if ((REGNO_LAST_UID (bl->regno) != INSN_UID (first_compare))
8470 || REGNO_FIRST_UID (bl->regno) != INSN_UID (bl->init_insn))
8471 loop_insn_sink (loop, gen_load_of_final_value (reg, final_value));
8473 /* Delete compare/branch at end of loop. */
8474 delete_related_insns (PREV_INSN (loop_end));
8475 if (compare_and_branch == 2)
8476 delete_related_insns (first_compare);
8478 /* Add new compare/branch insn at end of loop. */
8480 emit_cmp_and_jump_insns (reg, const0_rtx, cmp_code, NULL_RTX,
8482 XEXP (jump_label, 0));
8485 emit_jump_insn_before (tem, loop_end);
8487 for (tem = PREV_INSN (loop_end);
8488 tem && GET_CODE (tem) != JUMP_INSN;
8489 tem = PREV_INSN (tem))
8493 JUMP_LABEL (tem) = XEXP (jump_label, 0);
8499 /* Increment of LABEL_NUSES done above. */
8500 /* Register is now always nonnegative,
8501 so add REG_NONNEG note to the branch. */
8502 REG_NOTES (tem) = gen_rtx_EXPR_LIST (REG_NONNEG, reg,
8508 /* No insn may reference both the reversed and another biv or it
8509 will fail (see comment near the top of the loop reversal
8511 Earlier on, we have verified that the biv has no use except
8512 counting, or it is the only biv in this function.
8513 However, the code that computes no_use_except_counting does
8514 not verify reg notes. It's possible to have an insn that
8515 references another biv, and has a REG_EQUAL note with an
8516 expression based on the reversed biv. To avoid this case,
8517 remove all REG_EQUAL notes based on the reversed biv
8519 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8523 rtx set = single_set (p);
8524 /* If this is a set of a GIV based on the reversed biv, any
8525 REG_EQUAL notes should still be correct. */
8527 || GET_CODE (SET_DEST (set)) != REG
8528 || (size_t) REGNO (SET_DEST (set)) >= ivs->n_regs
8529 || REG_IV_TYPE (ivs, REGNO (SET_DEST (set))) != GENERAL_INDUCT
8530 || REG_IV_INFO (ivs, REGNO (SET_DEST (set)))->src_reg != bl->biv->src_reg)
8531 for (pnote = ®_NOTES (p); *pnote;)
8533 if (REG_NOTE_KIND (*pnote) == REG_EQUAL
8534 && reg_mentioned_p (regno_reg_rtx[bl->regno],
8536 *pnote = XEXP (*pnote, 1);
8538 pnote = &XEXP (*pnote, 1);
8542 /* Mark that this biv has been reversed. Each giv which depends
8543 on this biv, and which is also live past the end of the loop
8544 will have to be fixed up. */
8548 if (loop_dump_stream)
8550 fprintf (loop_dump_stream, "Reversed loop");
8552 fprintf (loop_dump_stream, " and added reg_nonneg\n");
8554 fprintf (loop_dump_stream, "\n");
8565 /* Verify whether the biv BL appears to be eliminable,
8566 based on the insns in the loop that refer to it.
8568 If ELIMINATE_P is nonzero, actually do the elimination.
8570 THRESHOLD and INSN_COUNT are from loop_optimize and are used to
8571 determine whether invariant insns should be placed inside or at the
8572 start of the loop. */
8575 maybe_eliminate_biv (const struct loop *loop, struct iv_class *bl,
8576 int eliminate_p, int threshold, int insn_count)
8578 struct loop_ivs *ivs = LOOP_IVS (loop);
8579 rtx reg = bl->biv->dest_reg;
8582 /* Scan all insns in the loop, stopping if we find one that uses the
8583 biv in a way that we cannot eliminate. */
8585 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
8587 enum rtx_code code = GET_CODE (p);
8588 basic_block where_bb = 0;
8589 rtx where_insn = threshold >= insn_count ? 0 : p;
8592 /* If this is a libcall that sets a giv, skip ahead to its end. */
8593 if (GET_RTX_CLASS (code) == 'i')
8595 note = find_reg_note (p, REG_LIBCALL, NULL_RTX);
8599 rtx last = XEXP (note, 0);
8600 rtx set = single_set (last);
8602 if (set && GET_CODE (SET_DEST (set)) == REG)
8604 unsigned int regno = REGNO (SET_DEST (set));
8606 if (regno < ivs->n_regs
8607 && REG_IV_TYPE (ivs, regno) == GENERAL_INDUCT
8608 && REG_IV_INFO (ivs, regno)->src_reg == bl->biv->src_reg)
8614 /* Closely examine the insn if the biv is mentioned. */
8615 if ((code == INSN || code == JUMP_INSN || code == CALL_INSN)
8616 && reg_mentioned_p (reg, PATTERN (p))
8617 && ! maybe_eliminate_biv_1 (loop, PATTERN (p), p, bl,
8618 eliminate_p, where_bb, where_insn))
8620 if (loop_dump_stream)
8621 fprintf (loop_dump_stream,
8622 "Cannot eliminate biv %d: biv used in insn %d.\n",
8623 bl->regno, INSN_UID (p));
8627 /* If we are eliminating, kill REG_EQUAL notes mentioning the biv. */
8629 && (note = find_reg_note (p, REG_EQUAL, NULL_RTX)) != NULL_RTX
8630 && reg_mentioned_p (reg, XEXP (note, 0)))
8631 remove_note (p, note);
8636 if (loop_dump_stream)
8637 fprintf (loop_dump_stream, "biv %d %s eliminated.\n",
8638 bl->regno, eliminate_p ? "was" : "can be");
8645 /* INSN and REFERENCE are instructions in the same insn chain.
8646 Return nonzero if INSN is first. */
8649 loop_insn_first_p (rtx insn, rtx reference)
8653 for (p = insn, q = reference;;)
8655 /* Start with test for not first so that INSN == REFERENCE yields not
8657 if (q == insn || ! p)
8659 if (p == reference || ! q)
8662 /* Either of P or Q might be a NOTE. Notes have the same LUID as the
8663 previous insn, hence the <= comparison below does not work if
8665 if (INSN_UID (p) < max_uid_for_loop
8666 && INSN_UID (q) < max_uid_for_loop
8667 && GET_CODE (p) != NOTE)
8668 return INSN_LUID (p) <= INSN_LUID (q);
8670 if (INSN_UID (p) >= max_uid_for_loop
8671 || GET_CODE (p) == NOTE)
8673 if (INSN_UID (q) >= max_uid_for_loop)
8678 /* We are trying to eliminate BIV in INSN using GIV. Return nonzero if
8679 the offset that we have to take into account due to auto-increment /
8680 div derivation is zero. */
8682 biv_elimination_giv_has_0_offset (struct induction *biv,
8683 struct induction *giv, rtx insn)
8685 /* If the giv V had the auto-inc address optimization applied
8686 to it, and INSN occurs between the giv insn and the biv
8687 insn, then we'd have to adjust the value used here.
8688 This is rare, so we don't bother to make this possible. */
8689 if (giv->auto_inc_opt
8690 && ((loop_insn_first_p (giv->insn, insn)
8691 && loop_insn_first_p (insn, biv->insn))
8692 || (loop_insn_first_p (biv->insn, insn)
8693 && loop_insn_first_p (insn, giv->insn))))
8699 /* If BL appears in X (part of the pattern of INSN), see if we can
8700 eliminate its use. If so, return 1. If not, return 0.
8702 If BIV does not appear in X, return 1.
8704 If ELIMINATE_P is nonzero, actually do the elimination.
8705 WHERE_INSN/WHERE_BB indicate where extra insns should be added.
8706 Depending on how many items have been moved out of the loop, it
8707 will either be before INSN (when WHERE_INSN is nonzero) or at the
8708 start of the loop (when WHERE_INSN is zero). */
8711 maybe_eliminate_biv_1 (const struct loop *loop, rtx x, rtx insn,
8712 struct iv_class *bl, int eliminate_p,
8713 basic_block where_bb, rtx where_insn)
8715 enum rtx_code code = GET_CODE (x);
8716 rtx reg = bl->biv->dest_reg;
8717 enum machine_mode mode = GET_MODE (reg);
8718 struct induction *v;
8730 /* If we haven't already been able to do something with this BIV,
8731 we can't eliminate it. */
8737 /* If this sets the BIV, it is not a problem. */
8738 if (SET_DEST (x) == reg)
8741 /* If this is an insn that defines a giv, it is also ok because
8742 it will go away when the giv is reduced. */
8743 for (v = bl->giv; v; v = v->next_iv)
8744 if (v->giv_type == DEST_REG && SET_DEST (x) == v->dest_reg)
8748 if (SET_DEST (x) == cc0_rtx && SET_SRC (x) == reg)
8750 /* Can replace with any giv that was reduced and
8751 that has (MULT_VAL != 0) and (ADD_VAL == 0).
8752 Require a constant for MULT_VAL, so we know it's nonzero.
8753 ??? We disable this optimization to avoid potential
8756 for (v = bl->giv; v; v = v->next_iv)
8757 if (GET_CODE (v->mult_val) == CONST_INT && v->mult_val != const0_rtx
8758 && v->add_val == const0_rtx
8759 && ! v->ignore && ! v->maybe_dead && v->always_computable
8763 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8769 /* If the giv has the opposite direction of change,
8770 then reverse the comparison. */
8771 if (INTVAL (v->mult_val) < 0)
8772 new = gen_rtx_COMPARE (GET_MODE (v->new_reg),
8773 const0_rtx, v->new_reg);
8777 /* We can probably test that giv's reduced reg. */
8778 if (validate_change (insn, &SET_SRC (x), new, 0))
8782 /* Look for a giv with (MULT_VAL != 0) and (ADD_VAL != 0);
8783 replace test insn with a compare insn (cmp REDUCED_GIV ADD_VAL).
8784 Require a constant for MULT_VAL, so we know it's nonzero.
8785 ??? Do this only if ADD_VAL is a pointer to avoid a potential
8786 overflow problem. */
8788 for (v = bl->giv; v; v = v->next_iv)
8789 if (GET_CODE (v->mult_val) == CONST_INT
8790 && v->mult_val != const0_rtx
8791 && ! v->ignore && ! v->maybe_dead && v->always_computable
8793 && (GET_CODE (v->add_val) == SYMBOL_REF
8794 || GET_CODE (v->add_val) == LABEL_REF
8795 || GET_CODE (v->add_val) == CONST
8796 || (GET_CODE (v->add_val) == REG
8797 && REG_POINTER (v->add_val))))
8799 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8805 /* If the giv has the opposite direction of change,
8806 then reverse the comparison. */
8807 if (INTVAL (v->mult_val) < 0)
8808 new = gen_rtx_COMPARE (VOIDmode, copy_rtx (v->add_val),
8811 new = gen_rtx_COMPARE (VOIDmode, v->new_reg,
8812 copy_rtx (v->add_val));
8814 /* Replace biv with the giv's reduced register. */
8815 update_reg_last_use (v->add_val, insn);
8816 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8819 /* Insn doesn't support that constant or invariant. Copy it
8820 into a register (it will be a loop invariant.) */
8821 tem = gen_reg_rtx (GET_MODE (v->new_reg));
8823 loop_insn_emit_before (loop, 0, where_insn,
8825 copy_rtx (v->add_val)));
8827 /* Substitute the new register for its invariant value in
8828 the compare expression. */
8829 XEXP (new, (INTVAL (v->mult_val) < 0) ? 0 : 1) = tem;
8830 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8839 case GT: case GE: case GTU: case GEU:
8840 case LT: case LE: case LTU: case LEU:
8841 /* See if either argument is the biv. */
8842 if (XEXP (x, 0) == reg)
8843 arg = XEXP (x, 1), arg_operand = 1;
8844 else if (XEXP (x, 1) == reg)
8845 arg = XEXP (x, 0), arg_operand = 0;
8849 if (CONSTANT_P (arg))
8851 /* First try to replace with any giv that has constant positive
8852 mult_val and constant add_val. We might be able to support
8853 negative mult_val, but it seems complex to do it in general. */
8855 for (v = bl->giv; v; v = v->next_iv)
8856 if (GET_CODE (v->mult_val) == CONST_INT
8857 && INTVAL (v->mult_val) > 0
8858 && (GET_CODE (v->add_val) == SYMBOL_REF
8859 || GET_CODE (v->add_val) == LABEL_REF
8860 || GET_CODE (v->add_val) == CONST
8861 || (GET_CODE (v->add_val) == REG
8862 && REG_POINTER (v->add_val)))
8863 && ! v->ignore && ! v->maybe_dead && v->always_computable
8866 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8869 /* Don't eliminate if the linear combination that makes up
8870 the giv overflows when it is applied to ARG. */
8871 if (GET_CODE (arg) == CONST_INT)
8875 if (GET_CODE (v->add_val) == CONST_INT)
8876 add_val = v->add_val;
8878 add_val = const0_rtx;
8880 if (const_mult_add_overflow_p (arg, v->mult_val,
8888 /* Replace biv with the giv's reduced reg. */
8889 validate_change (insn, &XEXP (x, 1 - arg_operand), v->new_reg, 1);
8891 /* If all constants are actually constant integers and
8892 the derived constant can be directly placed in the COMPARE,
8894 if (GET_CODE (arg) == CONST_INT
8895 && GET_CODE (v->add_val) == CONST_INT)
8897 tem = expand_mult_add (arg, NULL_RTX, v->mult_val,
8898 v->add_val, mode, 1);
8902 /* Otherwise, load it into a register. */
8903 tem = gen_reg_rtx (mode);
8904 loop_iv_add_mult_emit_before (loop, arg,
8905 v->mult_val, v->add_val,
8906 tem, where_bb, where_insn);
8909 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8911 if (apply_change_group ())
8915 /* Look for giv with positive constant mult_val and nonconst add_val.
8916 Insert insns to calculate new compare value.
8917 ??? Turn this off due to possible overflow. */
8919 for (v = bl->giv; v; v = v->next_iv)
8920 if (GET_CODE (v->mult_val) == CONST_INT
8921 && INTVAL (v->mult_val) > 0
8922 && ! v->ignore && ! v->maybe_dead && v->always_computable
8928 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8934 tem = gen_reg_rtx (mode);
8936 /* Replace biv with giv's reduced register. */
8937 validate_change (insn, &XEXP (x, 1 - arg_operand),
8940 /* Compute value to compare against. */
8941 loop_iv_add_mult_emit_before (loop, arg,
8942 v->mult_val, v->add_val,
8943 tem, where_bb, where_insn);
8944 /* Use it in this insn. */
8945 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8946 if (apply_change_group ())
8950 else if (GET_CODE (arg) == REG || GET_CODE (arg) == MEM)
8952 if (loop_invariant_p (loop, arg) == 1)
8954 /* Look for giv with constant positive mult_val and nonconst
8955 add_val. Insert insns to compute new compare value.
8956 ??? Turn this off due to possible overflow. */
8958 for (v = bl->giv; v; v = v->next_iv)
8959 if (GET_CODE (v->mult_val) == CONST_INT && INTVAL (v->mult_val) > 0
8960 && ! v->ignore && ! v->maybe_dead && v->always_computable
8966 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8972 tem = gen_reg_rtx (mode);
8974 /* Replace biv with giv's reduced register. */
8975 validate_change (insn, &XEXP (x, 1 - arg_operand),
8978 /* Compute value to compare against. */
8979 loop_iv_add_mult_emit_before (loop, arg,
8980 v->mult_val, v->add_val,
8981 tem, where_bb, where_insn);
8982 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8983 if (apply_change_group ())
8988 /* This code has problems. Basically, you can't know when
8989 seeing if we will eliminate BL, whether a particular giv
8990 of ARG will be reduced. If it isn't going to be reduced,
8991 we can't eliminate BL. We can try forcing it to be reduced,
8992 but that can generate poor code.
8994 The problem is that the benefit of reducing TV, below should
8995 be increased if BL can actually be eliminated, but this means
8996 we might have to do a topological sort of the order in which
8997 we try to process biv. It doesn't seem worthwhile to do
8998 this sort of thing now. */
9001 /* Otherwise the reg compared with had better be a biv. */
9002 if (GET_CODE (arg) != REG
9003 || REG_IV_TYPE (ivs, REGNO (arg)) != BASIC_INDUCT)
9006 /* Look for a pair of givs, one for each biv,
9007 with identical coefficients. */
9008 for (v = bl->giv; v; v = v->next_iv)
9010 struct induction *tv;
9012 if (v->ignore || v->maybe_dead || v->mode != mode)
9015 for (tv = REG_IV_CLASS (ivs, REGNO (arg))->giv; tv;
9017 if (! tv->ignore && ! tv->maybe_dead
9018 && rtx_equal_p (tv->mult_val, v->mult_val)
9019 && rtx_equal_p (tv->add_val, v->add_val)
9020 && tv->mode == mode)
9022 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
9028 /* Replace biv with its giv's reduced reg. */
9029 XEXP (x, 1 - arg_operand) = v->new_reg;
9030 /* Replace other operand with the other giv's
9032 XEXP (x, arg_operand) = tv->new_reg;
9039 /* If we get here, the biv can't be eliminated. */
9043 /* If this address is a DEST_ADDR giv, it doesn't matter if the
9044 biv is used in it, since it will be replaced. */
9045 for (v = bl->giv; v; v = v->next_iv)
9046 if (v->giv_type == DEST_ADDR && v->location == &XEXP (x, 0))
9054 /* See if any subexpression fails elimination. */
9055 fmt = GET_RTX_FORMAT (code);
9056 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
9061 if (! maybe_eliminate_biv_1 (loop, XEXP (x, i), insn, bl,
9062 eliminate_p, where_bb, where_insn))
9067 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9068 if (! maybe_eliminate_biv_1 (loop, XVECEXP (x, i, j), insn, bl,
9069 eliminate_p, where_bb, where_insn))
9078 /* Return nonzero if the last use of REG
9079 is in an insn following INSN in the same basic block. */
9082 last_use_this_basic_block (rtx reg, rtx insn)
9086 n && GET_CODE (n) != CODE_LABEL && GET_CODE (n) != JUMP_INSN;
9089 if (REGNO_LAST_UID (REGNO (reg)) == INSN_UID (n))
9095 /* Called via `note_stores' to record the initial value of a biv. Here we
9096 just record the location of the set and process it later. */
9099 record_initial (rtx dest, rtx set, void *data ATTRIBUTE_UNUSED)
9101 struct loop_ivs *ivs = (struct loop_ivs *) data;
9102 struct iv_class *bl;
9104 if (GET_CODE (dest) != REG
9105 || REGNO (dest) >= ivs->n_regs
9106 || REG_IV_TYPE (ivs, REGNO (dest)) != BASIC_INDUCT)
9109 bl = REG_IV_CLASS (ivs, REGNO (dest));
9111 /* If this is the first set found, record it. */
9112 if (bl->init_insn == 0)
9114 bl->init_insn = note_insn;
9119 /* If any of the registers in X are "old" and currently have a last use earlier
9120 than INSN, update them to have a last use of INSN. Their actual last use
9121 will be the previous insn but it will not have a valid uid_luid so we can't
9122 use it. X must be a source expression only. */
9125 update_reg_last_use (rtx x, rtx insn)
9127 /* Check for the case where INSN does not have a valid luid. In this case,
9128 there is no need to modify the regno_last_uid, as this can only happen
9129 when code is inserted after the loop_end to set a pseudo's final value,
9130 and hence this insn will never be the last use of x.
9131 ???? This comment is not correct. See for example loop_givs_reduce.
9132 This may insert an insn before another new insn. */
9133 if (GET_CODE (x) == REG && REGNO (x) < max_reg_before_loop
9134 && INSN_UID (insn) < max_uid_for_loop
9135 && REGNO_LAST_LUID (REGNO (x)) < INSN_LUID (insn))
9137 REGNO_LAST_UID (REGNO (x)) = INSN_UID (insn);
9142 const char *fmt = GET_RTX_FORMAT (GET_CODE (x));
9143 for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
9146 update_reg_last_use (XEXP (x, i), insn);
9147 else if (fmt[i] == 'E')
9148 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9149 update_reg_last_use (XVECEXP (x, i, j), insn);
9154 /* Given an insn INSN and condition COND, return the condition in a
9155 canonical form to simplify testing by callers. Specifically:
9157 (1) The code will always be a comparison operation (EQ, NE, GT, etc.).
9158 (2) Both operands will be machine operands; (cc0) will have been replaced.
9159 (3) If an operand is a constant, it will be the second operand.
9160 (4) (LE x const) will be replaced with (LT x <const+1>) and similarly
9161 for GE, GEU, and LEU.
9163 If the condition cannot be understood, or is an inequality floating-point
9164 comparison which needs to be reversed, 0 will be returned.
9166 If REVERSE is nonzero, then reverse the condition prior to canonizing it.
9168 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9169 insn used in locating the condition was found. If a replacement test
9170 of the condition is desired, it should be placed in front of that
9171 insn and we will be sure that the inputs are still valid.
9173 If WANT_REG is nonzero, we wish the condition to be relative to that
9174 register, if possible. Therefore, do not canonicalize the condition
9175 further. If ALLOW_CC_MODE is nonzero, allow the condition returned
9176 to be a compare to a CC mode register. */
9179 canonicalize_condition (rtx insn, rtx cond, int reverse, rtx *earliest,
9180 rtx want_reg, int allow_cc_mode)
9187 int reverse_code = 0;
9188 enum machine_mode mode;
9190 code = GET_CODE (cond);
9191 mode = GET_MODE (cond);
9192 op0 = XEXP (cond, 0);
9193 op1 = XEXP (cond, 1);
9196 code = reversed_comparison_code (cond, insn);
9197 if (code == UNKNOWN)
9203 /* If we are comparing a register with zero, see if the register is set
9204 in the previous insn to a COMPARE or a comparison operation. Perform
9205 the same tests as a function of STORE_FLAG_VALUE as find_comparison_args
9208 while (GET_RTX_CLASS (code) == '<'
9209 && op1 == CONST0_RTX (GET_MODE (op0))
9212 /* Set nonzero when we find something of interest. */
9216 /* If comparison with cc0, import actual comparison from compare
9220 if ((prev = prev_nonnote_insn (prev)) == 0
9221 || GET_CODE (prev) != INSN
9222 || (set = single_set (prev)) == 0
9223 || SET_DEST (set) != cc0_rtx)
9226 op0 = SET_SRC (set);
9227 op1 = CONST0_RTX (GET_MODE (op0));
9233 /* If this is a COMPARE, pick up the two things being compared. */
9234 if (GET_CODE (op0) == COMPARE)
9236 op1 = XEXP (op0, 1);
9237 op0 = XEXP (op0, 0);
9240 else if (GET_CODE (op0) != REG)
9243 /* Go back to the previous insn. Stop if it is not an INSN. We also
9244 stop if it isn't a single set or if it has a REG_INC note because
9245 we don't want to bother dealing with it. */
9247 if ((prev = prev_nonnote_insn (prev)) == 0
9248 || GET_CODE (prev) != INSN
9249 || FIND_REG_INC_NOTE (prev, NULL_RTX))
9252 set = set_of (op0, prev);
9255 && (GET_CODE (set) != SET
9256 || !rtx_equal_p (SET_DEST (set), op0)))
9259 /* If this is setting OP0, get what it sets it to if it looks
9263 enum machine_mode inner_mode = GET_MODE (SET_DEST (set));
9264 #ifdef FLOAT_STORE_FLAG_VALUE
9265 REAL_VALUE_TYPE fsfv;
9268 /* ??? We may not combine comparisons done in a CCmode with
9269 comparisons not done in a CCmode. This is to aid targets
9270 like Alpha that have an IEEE compliant EQ instruction, and
9271 a non-IEEE compliant BEQ instruction. The use of CCmode is
9272 actually artificial, simply to prevent the combination, but
9273 should not affect other platforms.
9275 However, we must allow VOIDmode comparisons to match either
9276 CCmode or non-CCmode comparison, because some ports have
9277 modeless comparisons inside branch patterns.
9279 ??? This mode check should perhaps look more like the mode check
9280 in simplify_comparison in combine. */
9282 if ((GET_CODE (SET_SRC (set)) == COMPARE
9285 && GET_MODE_CLASS (inner_mode) == MODE_INT
9286 && (GET_MODE_BITSIZE (inner_mode)
9287 <= HOST_BITS_PER_WIDE_INT)
9288 && (STORE_FLAG_VALUE
9289 & ((HOST_WIDE_INT) 1
9290 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9291 #ifdef FLOAT_STORE_FLAG_VALUE
9293 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9294 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9295 REAL_VALUE_NEGATIVE (fsfv)))
9298 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'))
9299 && (((GET_MODE_CLASS (mode) == MODE_CC)
9300 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9301 || mode == VOIDmode || inner_mode == VOIDmode))
9303 else if (((code == EQ
9305 && (GET_MODE_BITSIZE (inner_mode)
9306 <= HOST_BITS_PER_WIDE_INT)
9307 && GET_MODE_CLASS (inner_mode) == MODE_INT
9308 && (STORE_FLAG_VALUE
9309 & ((HOST_WIDE_INT) 1
9310 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9311 #ifdef FLOAT_STORE_FLAG_VALUE
9313 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9314 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9315 REAL_VALUE_NEGATIVE (fsfv)))
9318 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'
9319 && (((GET_MODE_CLASS (mode) == MODE_CC)
9320 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9321 || mode == VOIDmode || inner_mode == VOIDmode))
9331 else if (reg_set_p (op0, prev))
9332 /* If this sets OP0, but not directly, we have to give up. */
9337 if (GET_RTX_CLASS (GET_CODE (x)) == '<')
9338 code = GET_CODE (x);
9341 code = reversed_comparison_code (x, prev);
9342 if (code == UNKNOWN)
9347 op0 = XEXP (x, 0), op1 = XEXP (x, 1);
9353 /* If constant is first, put it last. */
9354 if (CONSTANT_P (op0))
9355 code = swap_condition (code), tem = op0, op0 = op1, op1 = tem;
9357 /* If OP0 is the result of a comparison, we weren't able to find what
9358 was really being compared, so fail. */
9360 && GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
9363 /* Canonicalize any ordered comparison with integers involving equality
9364 if we can do computations in the relevant mode and we do not
9367 if (GET_MODE_CLASS (GET_MODE (op0)) != MODE_CC
9368 && GET_CODE (op1) == CONST_INT
9369 && GET_MODE (op0) != VOIDmode
9370 && GET_MODE_BITSIZE (GET_MODE (op0)) <= HOST_BITS_PER_WIDE_INT)
9372 HOST_WIDE_INT const_val = INTVAL (op1);
9373 unsigned HOST_WIDE_INT uconst_val = const_val;
9374 unsigned HOST_WIDE_INT max_val
9375 = (unsigned HOST_WIDE_INT) GET_MODE_MASK (GET_MODE (op0));
9380 if ((unsigned HOST_WIDE_INT) const_val != max_val >> 1)
9381 code = LT, op1 = gen_int_mode (const_val + 1, GET_MODE (op0));
9384 /* When cross-compiling, const_val might be sign-extended from
9385 BITS_PER_WORD to HOST_BITS_PER_WIDE_INT */
9387 if ((HOST_WIDE_INT) (const_val & max_val)
9388 != (((HOST_WIDE_INT) 1
9389 << (GET_MODE_BITSIZE (GET_MODE (op0)) - 1))))
9390 code = GT, op1 = gen_int_mode (const_val - 1, GET_MODE (op0));
9394 if (uconst_val < max_val)
9395 code = LTU, op1 = gen_int_mode (uconst_val + 1, GET_MODE (op0));
9399 if (uconst_val != 0)
9400 code = GTU, op1 = gen_int_mode (uconst_val - 1, GET_MODE (op0));
9408 /* Never return CC0; return zero instead. */
9412 return gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
9415 /* Given a jump insn JUMP, return the condition that will cause it to branch
9416 to its JUMP_LABEL. If the condition cannot be understood, or is an
9417 inequality floating-point comparison which needs to be reversed, 0 will
9420 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9421 insn used in locating the condition was found. If a replacement test
9422 of the condition is desired, it should be placed in front of that
9423 insn and we will be sure that the inputs are still valid.
9425 If ALLOW_CC_MODE is nonzero, allow the condition returned to be a
9426 compare CC mode register. */
9429 get_condition (rtx jump, rtx *earliest, int allow_cc_mode)
9435 /* If this is not a standard conditional jump, we can't parse it. */
9436 if (GET_CODE (jump) != JUMP_INSN
9437 || ! any_condjump_p (jump))
9439 set = pc_set (jump);
9441 cond = XEXP (SET_SRC (set), 0);
9443 /* If this branches to JUMP_LABEL when the condition is false, reverse
9446 = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
9447 && XEXP (XEXP (SET_SRC (set), 2), 0) == JUMP_LABEL (jump);
9449 return canonicalize_condition (jump, cond, reverse, earliest, NULL_RTX,
9453 /* Similar to above routine, except that we also put an invariant last
9454 unless both operands are invariants. */
9457 get_condition_for_loop (const struct loop *loop, rtx x)
9459 rtx comparison = get_condition (x, (rtx*) 0, false);
9462 || ! loop_invariant_p (loop, XEXP (comparison, 0))
9463 || loop_invariant_p (loop, XEXP (comparison, 1)))
9466 return gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)), VOIDmode,
9467 XEXP (comparison, 1), XEXP (comparison, 0));
9470 /* Scan the function and determine whether it has indirect (computed) jumps.
9472 This is taken mostly from flow.c; similar code exists elsewhere
9473 in the compiler. It may be useful to put this into rtlanal.c. */
9475 indirect_jump_in_function_p (rtx start)
9479 for (insn = start; insn; insn = NEXT_INSN (insn))
9480 if (computed_jump_p (insn))
9486 /* Add MEM to the LOOP_MEMS array, if appropriate. See the
9487 documentation for LOOP_MEMS for the definition of `appropriate'.
9488 This function is called from prescan_loop via for_each_rtx. */
9491 insert_loop_mem (rtx *mem, void *data ATTRIBUTE_UNUSED)
9493 struct loop_info *loop_info = data;
9500 switch (GET_CODE (m))
9506 /* We're not interested in MEMs that are only clobbered. */
9510 /* We're not interested in the MEM associated with a
9511 CONST_DOUBLE, so there's no need to traverse into this. */
9515 /* We're not interested in any MEMs that only appear in notes. */
9519 /* This is not a MEM. */
9523 /* See if we've already seen this MEM. */
9524 for (i = 0; i < loop_info->mems_idx; ++i)
9525 if (rtx_equal_p (m, loop_info->mems[i].mem))
9527 if (GET_MODE (m) != GET_MODE (loop_info->mems[i].mem))
9528 /* The modes of the two memory accesses are different. If
9529 this happens, something tricky is going on, and we just
9530 don't optimize accesses to this MEM. */
9531 loop_info->mems[i].optimize = 0;
9536 /* Resize the array, if necessary. */
9537 if (loop_info->mems_idx == loop_info->mems_allocated)
9539 if (loop_info->mems_allocated != 0)
9540 loop_info->mems_allocated *= 2;
9542 loop_info->mems_allocated = 32;
9544 loop_info->mems = xrealloc (loop_info->mems,
9545 loop_info->mems_allocated * sizeof (loop_mem_info));
9548 /* Actually insert the MEM. */
9549 loop_info->mems[loop_info->mems_idx].mem = m;
9550 /* We can't hoist this MEM out of the loop if it's a BLKmode MEM
9551 because we can't put it in a register. We still store it in the
9552 table, though, so that if we see the same address later, but in a
9553 non-BLK mode, we'll not think we can optimize it at that point. */
9554 loop_info->mems[loop_info->mems_idx].optimize = (GET_MODE (m) != BLKmode);
9555 loop_info->mems[loop_info->mems_idx].reg = NULL_RTX;
9556 ++loop_info->mems_idx;
9562 /* Allocate REGS->ARRAY or reallocate it if it is too small.
9564 Increment REGS->ARRAY[I].SET_IN_LOOP at the index I of each
9565 register that is modified by an insn between FROM and TO. If the
9566 value of an element of REGS->array[I].SET_IN_LOOP becomes 127 or
9567 more, stop incrementing it, to avoid overflow.
9569 Store in REGS->ARRAY[I].SINGLE_USAGE the single insn in which
9570 register I is used, if it is only used once. Otherwise, it is set
9571 to 0 (for no uses) or const0_rtx for more than one use. This
9572 parameter may be zero, in which case this processing is not done.
9574 Set REGS->ARRAY[I].MAY_NOT_OPTIMIZE nonzero if we should not
9575 optimize register I. */
9578 loop_regs_scan (const struct loop *loop, int extra_size)
9580 struct loop_regs *regs = LOOP_REGS (loop);
9582 /* last_set[n] is nonzero iff reg n has been set in the current
9583 basic block. In that case, it is the insn that last set reg n. */
9588 old_nregs = regs->num;
9589 regs->num = max_reg_num ();
9591 /* Grow the regs array if not allocated or too small. */
9592 if (regs->num >= regs->size)
9594 regs->size = regs->num + extra_size;
9596 regs->array = xrealloc (regs->array, regs->size * sizeof (*regs->array));
9598 /* Zero the new elements. */
9599 memset (regs->array + old_nregs, 0,
9600 (regs->size - old_nregs) * sizeof (*regs->array));
9603 /* Clear previously scanned fields but do not clear n_times_set. */
9604 for (i = 0; i < old_nregs; i++)
9606 regs->array[i].set_in_loop = 0;
9607 regs->array[i].may_not_optimize = 0;
9608 regs->array[i].single_usage = NULL_RTX;
9611 last_set = xcalloc (regs->num, sizeof (rtx));
9613 /* Scan the loop, recording register usage. */
9614 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9615 insn = NEXT_INSN (insn))
9619 /* Record registers that have exactly one use. */
9620 find_single_use_in_loop (regs, insn, PATTERN (insn));
9622 /* Include uses in REG_EQUAL notes. */
9623 if (REG_NOTES (insn))
9624 find_single_use_in_loop (regs, insn, REG_NOTES (insn));
9626 if (GET_CODE (PATTERN (insn)) == SET
9627 || GET_CODE (PATTERN (insn)) == CLOBBER)
9628 count_one_set (regs, insn, PATTERN (insn), last_set);
9629 else if (GET_CODE (PATTERN (insn)) == PARALLEL)
9632 for (i = XVECLEN (PATTERN (insn), 0) - 1; i >= 0; i--)
9633 count_one_set (regs, insn, XVECEXP (PATTERN (insn), 0, i),
9638 if (GET_CODE (insn) == CODE_LABEL || GET_CODE (insn) == JUMP_INSN)
9639 memset (last_set, 0, regs->num * sizeof (rtx));
9641 /* Invalidate all registers used for function argument passing.
9642 We check rtx_varies_p for the same reason as below, to allow
9643 optimizing PIC calculations. */
9644 if (GET_CODE (insn) == CALL_INSN)
9647 for (link = CALL_INSN_FUNCTION_USAGE (insn);
9649 link = XEXP (link, 1))
9653 if (GET_CODE (op = XEXP (link, 0)) == USE
9654 && GET_CODE (reg = XEXP (op, 0)) == REG
9655 && rtx_varies_p (reg, 1))
9656 regs->array[REGNO (reg)].may_not_optimize = 1;
9661 /* Invalidate all hard registers clobbered by calls. With one exception:
9662 a call-clobbered PIC register is still function-invariant for our
9663 purposes, since we can hoist any PIC calculations out of the loop.
9664 Thus the call to rtx_varies_p. */
9665 if (LOOP_INFO (loop)->has_call)
9666 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
9667 if (TEST_HARD_REG_BIT (regs_invalidated_by_call, i)
9668 && rtx_varies_p (regno_reg_rtx[i], 1))
9670 regs->array[i].may_not_optimize = 1;
9671 regs->array[i].set_in_loop = 1;
9674 #ifdef AVOID_CCMODE_COPIES
9675 /* Don't try to move insns which set CC registers if we should not
9676 create CCmode register copies. */
9677 for (i = regs->num - 1; i >= FIRST_PSEUDO_REGISTER; i--)
9678 if (GET_MODE_CLASS (GET_MODE (regno_reg_rtx[i])) == MODE_CC)
9679 regs->array[i].may_not_optimize = 1;
9682 /* Set regs->array[I].n_times_set for the new registers. */
9683 for (i = old_nregs; i < regs->num; i++)
9684 regs->array[i].n_times_set = regs->array[i].set_in_loop;
9689 /* Returns the number of real INSNs in the LOOP. */
9692 count_insns_in_loop (const struct loop *loop)
9697 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9698 insn = NEXT_INSN (insn))
9705 /* Move MEMs into registers for the duration of the loop. */
9708 load_mems (const struct loop *loop)
9710 struct loop_info *loop_info = LOOP_INFO (loop);
9711 struct loop_regs *regs = LOOP_REGS (loop);
9712 int maybe_never = 0;
9714 rtx p, prev_ebb_head;
9715 rtx label = NULL_RTX;
9717 /* Nonzero if the next instruction may never be executed. */
9718 int next_maybe_never = 0;
9719 unsigned int last_max_reg = max_reg_num ();
9721 if (loop_info->mems_idx == 0)
9724 /* We cannot use next_label here because it skips over normal insns. */
9725 end_label = next_nonnote_insn (loop->end);
9726 if (end_label && GET_CODE (end_label) != CODE_LABEL)
9727 end_label = NULL_RTX;
9729 /* Check to see if it's possible that some instructions in the loop are
9730 never executed. Also check if there is a goto out of the loop other
9731 than right after the end of the loop. */
9732 for (p = next_insn_in_loop (loop, loop->scan_start);
9734 p = next_insn_in_loop (loop, p))
9736 if (GET_CODE (p) == CODE_LABEL)
9738 else if (GET_CODE (p) == JUMP_INSN
9739 /* If we enter the loop in the middle, and scan
9740 around to the beginning, don't set maybe_never
9741 for that. This must be an unconditional jump,
9742 otherwise the code at the top of the loop might
9743 never be executed. Unconditional jumps are
9744 followed a by barrier then loop end. */
9745 && ! (GET_CODE (p) == JUMP_INSN
9746 && JUMP_LABEL (p) == loop->top
9747 && NEXT_INSN (NEXT_INSN (p)) == loop->end
9748 && any_uncondjump_p (p)))
9750 /* If this is a jump outside of the loop but not right
9751 after the end of the loop, we would have to emit new fixup
9752 sequences for each such label. */
9753 if (/* If we can't tell where control might go when this
9754 JUMP_INSN is executed, we must be conservative. */
9756 || (JUMP_LABEL (p) != end_label
9757 && (INSN_UID (JUMP_LABEL (p)) >= max_uid_for_loop
9758 || INSN_LUID (JUMP_LABEL (p)) < INSN_LUID (loop->start)
9759 || INSN_LUID (JUMP_LABEL (p)) > INSN_LUID (loop->end))))
9762 if (!any_condjump_p (p))
9763 /* Something complicated. */
9766 /* If there are any more instructions in the loop, they
9767 might not be reached. */
9768 next_maybe_never = 1;
9770 else if (next_maybe_never)
9774 /* Find start of the extended basic block that enters the loop. */
9775 for (p = loop->start;
9776 PREV_INSN (p) && GET_CODE (p) != CODE_LABEL;
9783 /* Build table of mems that get set to constant values before the
9785 for (; p != loop->start; p = NEXT_INSN (p))
9786 cselib_process_insn (p);
9788 /* Actually move the MEMs. */
9789 for (i = 0; i < loop_info->mems_idx; ++i)
9791 regset_head load_copies;
9792 regset_head store_copies;
9795 rtx mem = loop_info->mems[i].mem;
9798 if (MEM_VOLATILE_P (mem)
9799 || loop_invariant_p (loop, XEXP (mem, 0)) != 1)
9800 /* There's no telling whether or not MEM is modified. */
9801 loop_info->mems[i].optimize = 0;
9803 /* Go through the MEMs written to in the loop to see if this
9804 one is aliased by one of them. */
9805 mem_list_entry = loop_info->store_mems;
9806 while (mem_list_entry)
9808 if (rtx_equal_p (mem, XEXP (mem_list_entry, 0)))
9810 else if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
9813 /* MEM is indeed aliased by this store. */
9814 loop_info->mems[i].optimize = 0;
9817 mem_list_entry = XEXP (mem_list_entry, 1);
9820 if (flag_float_store && written
9821 && GET_MODE_CLASS (GET_MODE (mem)) == MODE_FLOAT)
9822 loop_info->mems[i].optimize = 0;
9824 /* If this MEM is written to, we must be sure that there
9825 are no reads from another MEM that aliases this one. */
9826 if (loop_info->mems[i].optimize && written)
9830 for (j = 0; j < loop_info->mems_idx; ++j)
9834 else if (true_dependence (mem,
9836 loop_info->mems[j].mem,
9839 /* It's not safe to hoist loop_info->mems[i] out of
9840 the loop because writes to it might not be
9841 seen by reads from loop_info->mems[j]. */
9842 loop_info->mems[i].optimize = 0;
9848 if (maybe_never && may_trap_p (mem))
9849 /* We can't access the MEM outside the loop; it might
9850 cause a trap that wouldn't have happened otherwise. */
9851 loop_info->mems[i].optimize = 0;
9853 if (!loop_info->mems[i].optimize)
9854 /* We thought we were going to lift this MEM out of the
9855 loop, but later discovered that we could not. */
9858 INIT_REG_SET (&load_copies);
9859 INIT_REG_SET (&store_copies);
9861 /* Allocate a pseudo for this MEM. We set REG_USERVAR_P in
9862 order to keep scan_loop from moving stores to this MEM
9863 out of the loop just because this REG is neither a
9864 user-variable nor used in the loop test. */
9865 reg = gen_reg_rtx (GET_MODE (mem));
9866 REG_USERVAR_P (reg) = 1;
9867 loop_info->mems[i].reg = reg;
9869 /* Now, replace all references to the MEM with the
9870 corresponding pseudos. */
9872 for (p = next_insn_in_loop (loop, loop->scan_start);
9874 p = next_insn_in_loop (loop, p))
9880 set = single_set (p);
9882 /* See if this copies the mem into a register that isn't
9883 modified afterwards. We'll try to do copy propagation
9884 a little further on. */
9886 /* @@@ This test is _way_ too conservative. */
9888 && GET_CODE (SET_DEST (set)) == REG
9889 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER
9890 && REGNO (SET_DEST (set)) < last_max_reg
9891 && regs->array[REGNO (SET_DEST (set))].n_times_set == 1
9892 && rtx_equal_p (SET_SRC (set), mem))
9893 SET_REGNO_REG_SET (&load_copies, REGNO (SET_DEST (set)));
9895 /* See if this copies the mem from a register that isn't
9896 modified afterwards. We'll try to remove the
9897 redundant copy later on by doing a little register
9898 renaming and copy propagation. This will help
9899 to untangle things for the BIV detection code. */
9902 && GET_CODE (SET_SRC (set)) == REG
9903 && REGNO (SET_SRC (set)) >= FIRST_PSEUDO_REGISTER
9904 && REGNO (SET_SRC (set)) < last_max_reg
9905 && regs->array[REGNO (SET_SRC (set))].n_times_set == 1
9906 && rtx_equal_p (SET_DEST (set), mem))
9907 SET_REGNO_REG_SET (&store_copies, REGNO (SET_SRC (set)));
9909 /* If this is a call which uses / clobbers this memory
9910 location, we must not change the interface here. */
9911 if (GET_CODE (p) == CALL_INSN
9912 && reg_mentioned_p (loop_info->mems[i].mem,
9913 CALL_INSN_FUNCTION_USAGE (p)))
9916 loop_info->mems[i].optimize = 0;
9920 /* Replace the memory reference with the shadow register. */
9921 replace_loop_mems (p, loop_info->mems[i].mem,
9922 loop_info->mems[i].reg, written);
9925 if (GET_CODE (p) == CODE_LABEL
9926 || GET_CODE (p) == JUMP_INSN)
9930 if (! loop_info->mems[i].optimize)
9931 ; /* We found we couldn't do the replacement, so do nothing. */
9932 else if (! apply_change_group ())
9933 /* We couldn't replace all occurrences of the MEM. */
9934 loop_info->mems[i].optimize = 0;
9937 /* Load the memory immediately before LOOP->START, which is
9938 the NOTE_LOOP_BEG. */
9939 cselib_val *e = cselib_lookup (mem, VOIDmode, 0);
9943 struct elt_loc_list *const_equiv = 0;
9947 struct elt_loc_list *equiv;
9948 struct elt_loc_list *best_equiv = 0;
9949 for (equiv = e->locs; equiv; equiv = equiv->next)
9951 if (CONSTANT_P (equiv->loc))
9952 const_equiv = equiv;
9953 else if (GET_CODE (equiv->loc) == REG
9954 /* Extending hard register lifetimes causes crash
9955 on SRC targets. Doing so on non-SRC is
9956 probably also not good idea, since we most
9957 probably have pseudoregister equivalence as
9959 && REGNO (equiv->loc) >= FIRST_PSEUDO_REGISTER)
9962 /* Use the constant equivalence if that is cheap enough. */
9964 best_equiv = const_equiv;
9965 else if (const_equiv
9966 && (rtx_cost (const_equiv->loc, SET)
9967 <= rtx_cost (best_equiv->loc, SET)))
9969 best_equiv = const_equiv;
9973 /* If best_equiv is nonzero, we know that MEM is set to a
9974 constant or register before the loop. We will use this
9975 knowledge to initialize the shadow register with that
9976 constant or reg rather than by loading from MEM. */
9978 best = copy_rtx (best_equiv->loc);
9981 set = gen_move_insn (reg, best);
9982 set = loop_insn_hoist (loop, set);
9985 for (p = prev_ebb_head; p != loop->start; p = NEXT_INSN (p))
9986 if (REGNO_LAST_UID (REGNO (best)) == INSN_UID (p))
9988 REGNO_LAST_UID (REGNO (best)) = INSN_UID (set);
9994 set_unique_reg_note (set, REG_EQUAL, copy_rtx (const_equiv->loc));
9998 if (label == NULL_RTX)
10000 label = gen_label_rtx ();
10001 emit_label_after (label, loop->end);
10004 /* Store the memory immediately after END, which is
10005 the NOTE_LOOP_END. */
10006 set = gen_move_insn (copy_rtx (mem), reg);
10007 loop_insn_emit_after (loop, 0, label, set);
10010 if (loop_dump_stream)
10012 fprintf (loop_dump_stream, "Hoisted regno %d %s from ",
10013 REGNO (reg), (written ? "r/w" : "r/o"));
10014 print_rtl (loop_dump_stream, mem);
10015 fputc ('\n', loop_dump_stream);
10018 /* Attempt a bit of copy propagation. This helps untangle the
10019 data flow, and enables {basic,general}_induction_var to find
10021 EXECUTE_IF_SET_IN_REG_SET
10022 (&load_copies, FIRST_PSEUDO_REGISTER, j,
10024 try_copy_prop (loop, reg, j);
10026 CLEAR_REG_SET (&load_copies);
10028 EXECUTE_IF_SET_IN_REG_SET
10029 (&store_copies, FIRST_PSEUDO_REGISTER, j,
10031 try_swap_copy_prop (loop, reg, j);
10033 CLEAR_REG_SET (&store_copies);
10037 /* Now, we need to replace all references to the previous exit
10038 label with the new one. */
10039 if (label != NULL_RTX && end_label != NULL_RTX)
10040 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
10041 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == end_label)
10042 redirect_jump (p, label, false);
10047 /* For communication between note_reg_stored and its caller. */
10048 struct note_reg_stored_arg
10054 /* Called via note_stores, record in SET_SEEN whether X, which is written,
10055 is equal to ARG. */
10057 note_reg_stored (rtx x, rtx setter ATTRIBUTE_UNUSED, void *arg)
10059 struct note_reg_stored_arg *t = (struct note_reg_stored_arg *) arg;
10064 /* Try to replace every occurrence of pseudo REGNO with REPLACEMENT.
10065 There must be exactly one insn that sets this pseudo; it will be
10066 deleted if all replacements succeed and we can prove that the register
10067 is not used after the loop. */
10070 try_copy_prop (const struct loop *loop, rtx replacement, unsigned int regno)
10072 /* This is the reg that we are copying from. */
10073 rtx reg_rtx = regno_reg_rtx[regno];
10076 /* These help keep track of whether we replaced all uses of the reg. */
10077 int replaced_last = 0;
10078 int store_is_first = 0;
10080 for (insn = next_insn_in_loop (loop, loop->scan_start);
10082 insn = next_insn_in_loop (loop, insn))
10086 /* Only substitute within one extended basic block from the initializing
10088 if (GET_CODE (insn) == CODE_LABEL && init_insn)
10091 if (! INSN_P (insn))
10094 /* Is this the initializing insn? */
10095 set = single_set (insn);
10097 && GET_CODE (SET_DEST (set)) == REG
10098 && REGNO (SET_DEST (set)) == regno)
10104 if (REGNO_FIRST_UID (regno) == INSN_UID (insn))
10105 store_is_first = 1;
10108 /* Only substitute after seeing the initializing insn. */
10109 if (init_insn && insn != init_insn)
10111 struct note_reg_stored_arg arg;
10113 replace_loop_regs (insn, reg_rtx, replacement);
10114 if (REGNO_LAST_UID (regno) == INSN_UID (insn))
10117 /* Stop replacing when REPLACEMENT is modified. */
10118 arg.reg = replacement;
10120 note_stores (PATTERN (insn), note_reg_stored, &arg);
10123 rtx note = find_reg_note (insn, REG_EQUAL, NULL);
10125 /* It is possible that we've turned previously valid REG_EQUAL to
10126 invalid, as we change the REGNO to REPLACEMENT and unlike REGNO,
10127 REPLACEMENT is modified, we get different meaning. */
10128 if (note && reg_mentioned_p (replacement, XEXP (note, 0)))
10129 remove_note (insn, note);
10136 if (apply_change_group ())
10138 if (loop_dump_stream)
10139 fprintf (loop_dump_stream, " Replaced reg %d", regno);
10140 if (store_is_first && replaced_last)
10145 /* Assume we're just deleting INIT_INSN. */
10147 /* Look for REG_RETVAL note. If we're deleting the end of
10148 the libcall sequence, the whole sequence can go. */
10149 retval_note = find_reg_note (init_insn, REG_RETVAL, NULL_RTX);
10150 /* If we found a REG_RETVAL note, find the first instruction
10151 in the sequence. */
10153 first = XEXP (retval_note, 0);
10155 /* Delete the instructions. */
10156 loop_delete_insns (first, init_insn);
10158 if (loop_dump_stream)
10159 fprintf (loop_dump_stream, ".\n");
10163 /* Replace all the instructions from FIRST up to and including LAST
10164 with NOTE_INSN_DELETED notes. */
10167 loop_delete_insns (rtx first, rtx last)
10171 if (loop_dump_stream)
10172 fprintf (loop_dump_stream, ", deleting init_insn (%d)",
10174 delete_insn (first);
10176 /* If this was the LAST instructions we're supposed to delete,
10181 first = NEXT_INSN (first);
10185 /* Try to replace occurrences of pseudo REGNO with REPLACEMENT within
10186 loop LOOP if the order of the sets of these registers can be
10187 swapped. There must be exactly one insn within the loop that sets
10188 this pseudo followed immediately by a move insn that sets
10189 REPLACEMENT with REGNO. */
10191 try_swap_copy_prop (const struct loop *loop, rtx replacement,
10192 unsigned int regno)
10195 rtx set = NULL_RTX;
10196 unsigned int new_regno;
10198 new_regno = REGNO (replacement);
10200 for (insn = next_insn_in_loop (loop, loop->scan_start);
10202 insn = next_insn_in_loop (loop, insn))
10204 /* Search for the insn that copies REGNO to NEW_REGNO? */
10206 && (set = single_set (insn))
10207 && GET_CODE (SET_DEST (set)) == REG
10208 && REGNO (SET_DEST (set)) == new_regno
10209 && GET_CODE (SET_SRC (set)) == REG
10210 && REGNO (SET_SRC (set)) == regno)
10214 if (insn != NULL_RTX)
10219 /* Some DEF-USE info would come in handy here to make this
10220 function more general. For now, just check the previous insn
10221 which is the most likely candidate for setting REGNO. */
10223 prev_insn = PREV_INSN (insn);
10226 && (prev_set = single_set (prev_insn))
10227 && GET_CODE (SET_DEST (prev_set)) == REG
10228 && REGNO (SET_DEST (prev_set)) == regno)
10231 (set (reg regno) (expr))
10232 (set (reg new_regno) (reg regno))
10234 so try converting this to:
10235 (set (reg new_regno) (expr))
10236 (set (reg regno) (reg new_regno))
10238 The former construct is often generated when a global
10239 variable used for an induction variable is shadowed by a
10240 register (NEW_REGNO). The latter construct improves the
10241 chances of GIV replacement and BIV elimination. */
10243 validate_change (prev_insn, &SET_DEST (prev_set),
10245 validate_change (insn, &SET_DEST (set),
10247 validate_change (insn, &SET_SRC (set),
10250 if (apply_change_group ())
10252 if (loop_dump_stream)
10253 fprintf (loop_dump_stream,
10254 " Swapped set of reg %d at %d with reg %d at %d.\n",
10255 regno, INSN_UID (insn),
10256 new_regno, INSN_UID (prev_insn));
10258 /* Update first use of REGNO. */
10259 if (REGNO_FIRST_UID (regno) == INSN_UID (prev_insn))
10260 REGNO_FIRST_UID (regno) = INSN_UID (insn);
10262 /* Now perform copy propagation to hopefully
10263 remove all uses of REGNO within the loop. */
10264 try_copy_prop (loop, replacement, regno);
10270 /* Worker function for find_mem_in_note, called via for_each_rtx. */
10273 find_mem_in_note_1 (rtx *x, void *data)
10275 if (*x != NULL_RTX && GET_CODE (*x) == MEM)
10277 rtx *res = (rtx *) data;
10284 /* Returns the first MEM found in NOTE by depth-first search. */
10287 find_mem_in_note (rtx note)
10289 if (note && for_each_rtx (¬e, find_mem_in_note_1, ¬e))
10294 /* Replace MEM with its associated pseudo register. This function is
10295 called from load_mems via for_each_rtx. DATA is actually a pointer
10296 to a structure describing the instruction currently being scanned
10297 and the MEM we are currently replacing. */
10300 replace_loop_mem (rtx *mem, void *data)
10302 loop_replace_args *args = (loop_replace_args *) data;
10308 switch (GET_CODE (m))
10314 /* We're not interested in the MEM associated with a
10315 CONST_DOUBLE, so there's no need to traverse into one. */
10319 /* This is not a MEM. */
10323 if (!rtx_equal_p (args->match, m))
10324 /* This is not the MEM we are currently replacing. */
10327 /* Actually replace the MEM. */
10328 validate_change (args->insn, mem, args->replacement, 1);
10334 replace_loop_mems (rtx insn, rtx mem, rtx reg, int written)
10336 loop_replace_args args;
10340 args.replacement = reg;
10342 for_each_rtx (&insn, replace_loop_mem, &args);
10344 /* If we hoist a mem write out of the loop, then REG_EQUAL
10345 notes referring to the mem are no longer valid. */
10351 for (link = ®_NOTES (insn); (note = *link); link = &XEXP (note, 1))
10353 if (REG_NOTE_KIND (note) == REG_EQUAL
10354 && (sub = find_mem_in_note (note))
10355 && true_dependence (mem, VOIDmode, sub, rtx_varies_p))
10357 /* Remove the note. */
10358 validate_change (NULL_RTX, link, XEXP (note, 1), 1);
10365 /* Replace one register with another. Called through for_each_rtx; PX points
10366 to the rtx being scanned. DATA is actually a pointer to
10367 a structure of arguments. */
10370 replace_loop_reg (rtx *px, void *data)
10373 loop_replace_args *args = (loop_replace_args *) data;
10378 if (x == args->match)
10379 validate_change (args->insn, px, args->replacement, 1);
10385 replace_loop_regs (rtx insn, rtx reg, rtx replacement)
10387 loop_replace_args args;
10391 args.replacement = replacement;
10393 for_each_rtx (&insn, replace_loop_reg, &args);
10396 /* Emit insn for PATTERN after WHERE_INSN in basic block WHERE_BB
10397 (ignored in the interim). */
10400 loop_insn_emit_after (const struct loop *loop ATTRIBUTE_UNUSED,
10401 basic_block where_bb ATTRIBUTE_UNUSED, rtx where_insn,
10404 return emit_insn_after (pattern, where_insn);
10408 /* If WHERE_INSN is nonzero emit insn for PATTERN before WHERE_INSN
10409 in basic block WHERE_BB (ignored in the interim) within the loop
10410 otherwise hoist PATTERN into the loop pre-header. */
10413 loop_insn_emit_before (const struct loop *loop,
10414 basic_block where_bb ATTRIBUTE_UNUSED,
10415 rtx where_insn, rtx pattern)
10418 return loop_insn_hoist (loop, pattern);
10419 return emit_insn_before (pattern, where_insn);
10423 /* Emit call insn for PATTERN before WHERE_INSN in basic block
10424 WHERE_BB (ignored in the interim) within the loop. */
10427 loop_call_insn_emit_before (const struct loop *loop ATTRIBUTE_UNUSED,
10428 basic_block where_bb ATTRIBUTE_UNUSED,
10429 rtx where_insn, rtx pattern)
10431 return emit_call_insn_before (pattern, where_insn);
10435 /* Hoist insn for PATTERN into the loop pre-header. */
10438 loop_insn_hoist (const struct loop *loop, rtx pattern)
10440 return loop_insn_emit_before (loop, 0, loop->start, pattern);
10444 /* Hoist call insn for PATTERN into the loop pre-header. */
10447 loop_call_insn_hoist (const struct loop *loop, rtx pattern)
10449 return loop_call_insn_emit_before (loop, 0, loop->start, pattern);
10453 /* Sink insn for PATTERN after the loop end. */
10456 loop_insn_sink (const struct loop *loop, rtx pattern)
10458 return loop_insn_emit_before (loop, 0, loop->sink, pattern);
10461 /* bl->final_value can be either general_operand or PLUS of general_operand
10462 and constant. Emit sequence of instructions to load it into REG. */
10464 gen_load_of_final_value (rtx reg, rtx final_value)
10468 final_value = force_operand (final_value, reg);
10469 if (final_value != reg)
10470 emit_move_insn (reg, final_value);
10471 seq = get_insns ();
10476 /* If the loop has multiple exits, emit insn for PATTERN before the
10477 loop to ensure that it will always be executed no matter how the
10478 loop exits. Otherwise, emit the insn for PATTERN after the loop,
10479 since this is slightly more efficient. */
10482 loop_insn_sink_or_swim (const struct loop *loop, rtx pattern)
10484 if (loop->exit_count)
10485 return loop_insn_hoist (loop, pattern);
10487 return loop_insn_sink (loop, pattern);
10491 loop_ivs_dump (const struct loop *loop, FILE *file, int verbose)
10493 struct iv_class *bl;
10496 if (! loop || ! file)
10499 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10502 fprintf (file, "Loop %d: %d IV classes\n", loop->num, iv_num);
10504 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10506 loop_iv_class_dump (bl, file, verbose);
10507 fputc ('\n', file);
10513 loop_iv_class_dump (const struct iv_class *bl, FILE *file,
10514 int verbose ATTRIBUTE_UNUSED)
10516 struct induction *v;
10520 if (! bl || ! file)
10523 fprintf (file, "IV class for reg %d, benefit %d\n",
10524 bl->regno, bl->total_benefit);
10526 fprintf (file, " Init insn %d", INSN_UID (bl->init_insn));
10527 if (bl->initial_value)
10529 fprintf (file, ", init val: ");
10530 print_simple_rtl (file, bl->initial_value);
10532 if (bl->initial_test)
10534 fprintf (file, ", init test: ");
10535 print_simple_rtl (file, bl->initial_test);
10537 fputc ('\n', file);
10539 if (bl->final_value)
10541 fprintf (file, " Final val: ");
10542 print_simple_rtl (file, bl->final_value);
10543 fputc ('\n', file);
10546 if ((incr = biv_total_increment (bl)))
10548 fprintf (file, " Total increment: ");
10549 print_simple_rtl (file, incr);
10550 fputc ('\n', file);
10553 /* List the increments. */
10554 for (i = 0, v = bl->biv; v; v = v->next_iv, i++)
10556 fprintf (file, " Inc%d: insn %d, incr: ", i, INSN_UID (v->insn));
10557 print_simple_rtl (file, v->add_val);
10558 fputc ('\n', file);
10561 /* List the givs. */
10562 for (i = 0, v = bl->giv; v; v = v->next_iv, i++)
10564 fprintf (file, " Giv%d: insn %d, benefit %d, ",
10565 i, INSN_UID (v->insn), v->benefit);
10566 if (v->giv_type == DEST_ADDR)
10567 print_simple_rtl (file, v->mem);
10569 print_simple_rtl (file, single_set (v->insn));
10570 fputc ('\n', file);
10576 loop_biv_dump (const struct induction *v, FILE *file, int verbose)
10583 REGNO (v->dest_reg), INSN_UID (v->insn));
10584 fprintf (file, " const ");
10585 print_simple_rtl (file, v->add_val);
10587 if (verbose && v->final_value)
10589 fputc ('\n', file);
10590 fprintf (file, " final ");
10591 print_simple_rtl (file, v->final_value);
10594 fputc ('\n', file);
10599 loop_giv_dump (const struct induction *v, FILE *file, int verbose)
10604 if (v->giv_type == DEST_REG)
10605 fprintf (file, "Giv %d: insn %d",
10606 REGNO (v->dest_reg), INSN_UID (v->insn));
10608 fprintf (file, "Dest address: insn %d",
10609 INSN_UID (v->insn));
10611 fprintf (file, " src reg %d benefit %d",
10612 REGNO (v->src_reg), v->benefit);
10613 fprintf (file, " lifetime %d",
10616 if (v->replaceable)
10617 fprintf (file, " replaceable");
10619 if (v->no_const_addval)
10620 fprintf (file, " ncav");
10622 if (v->ext_dependent)
10624 switch (GET_CODE (v->ext_dependent))
10627 fprintf (file, " ext se");
10630 fprintf (file, " ext ze");
10633 fprintf (file, " ext tr");
10640 fputc ('\n', file);
10641 fprintf (file, " mult ");
10642 print_simple_rtl (file, v->mult_val);
10644 fputc ('\n', file);
10645 fprintf (file, " add ");
10646 print_simple_rtl (file, v->add_val);
10648 if (verbose && v->final_value)
10650 fputc ('\n', file);
10651 fprintf (file, " final ");
10652 print_simple_rtl (file, v->final_value);
10655 fputc ('\n', file);
10660 debug_ivs (const struct loop *loop)
10662 loop_ivs_dump (loop, stderr, 1);
10667 debug_iv_class (const struct iv_class *bl)
10669 loop_iv_class_dump (bl, stderr, 1);
10674 debug_biv (const struct induction *v)
10676 loop_biv_dump (v, stderr, 1);
10681 debug_giv (const struct induction *v)
10683 loop_giv_dump (v, stderr, 1);
10687 #define LOOP_BLOCK_NUM_1(INSN) \
10688 ((INSN) ? (BLOCK_FOR_INSN (INSN) ? BLOCK_NUM (INSN) : - 1) : -1)
10690 /* The notes do not have an assigned block, so look at the next insn. */
10691 #define LOOP_BLOCK_NUM(INSN) \
10692 ((INSN) ? (GET_CODE (INSN) == NOTE \
10693 ? LOOP_BLOCK_NUM_1 (next_nonnote_insn (INSN)) \
10694 : LOOP_BLOCK_NUM_1 (INSN)) \
10697 #define LOOP_INSN_UID(INSN) ((INSN) ? INSN_UID (INSN) : -1)
10700 loop_dump_aux (const struct loop *loop, FILE *file,
10701 int verbose ATTRIBUTE_UNUSED)
10705 if (! loop || ! file)
10708 /* Print diagnostics to compare our concept of a loop with
10709 what the loop notes say. */
10710 if (! PREV_INSN (BB_HEAD (loop->first))
10711 || GET_CODE (PREV_INSN (BB_HEAD (loop->first))) != NOTE
10712 || NOTE_LINE_NUMBER (PREV_INSN (BB_HEAD (loop->first)))
10713 != NOTE_INSN_LOOP_BEG)
10714 fprintf (file, ";; No NOTE_INSN_LOOP_BEG at %d\n",
10715 INSN_UID (PREV_INSN (BB_HEAD (loop->first))));
10716 if (! NEXT_INSN (BB_END (loop->last))
10717 || GET_CODE (NEXT_INSN (BB_END (loop->last))) != NOTE
10718 || NOTE_LINE_NUMBER (NEXT_INSN (BB_END (loop->last)))
10719 != NOTE_INSN_LOOP_END)
10720 fprintf (file, ";; No NOTE_INSN_LOOP_END at %d\n",
10721 INSN_UID (NEXT_INSN (BB_END (loop->last))));
10726 ";; start %d (%d), cont dom %d (%d), cont %d (%d), vtop %d (%d), end %d (%d)\n",
10727 LOOP_BLOCK_NUM (loop->start),
10728 LOOP_INSN_UID (loop->start),
10729 LOOP_BLOCK_NUM (loop->cont),
10730 LOOP_INSN_UID (loop->cont),
10731 LOOP_BLOCK_NUM (loop->cont),
10732 LOOP_INSN_UID (loop->cont),
10733 LOOP_BLOCK_NUM (loop->vtop),
10734 LOOP_INSN_UID (loop->vtop),
10735 LOOP_BLOCK_NUM (loop->end),
10736 LOOP_INSN_UID (loop->end));
10737 fprintf (file, ";; top %d (%d), scan start %d (%d)\n",
10738 LOOP_BLOCK_NUM (loop->top),
10739 LOOP_INSN_UID (loop->top),
10740 LOOP_BLOCK_NUM (loop->scan_start),
10741 LOOP_INSN_UID (loop->scan_start));
10742 fprintf (file, ";; exit_count %d", loop->exit_count);
10743 if (loop->exit_count)
10745 fputs (", labels:", file);
10746 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
10748 fprintf (file, " %d ",
10749 LOOP_INSN_UID (XEXP (label, 0)));
10752 fputs ("\n", file);
10754 /* This can happen when a marked loop appears as two nested loops,
10755 say from while (a || b) {}. The inner loop won't match
10756 the loop markers but the outer one will. */
10757 if (LOOP_BLOCK_NUM (loop->cont) != loop->latch->index)
10758 fprintf (file, ";; NOTE_INSN_LOOP_CONT not in loop latch\n");
10762 /* Call this function from the debugger to dump LOOP. */
10765 debug_loop (const struct loop *loop)
10767 flow_loop_dump (loop, stderr, loop_dump_aux, 1);
10770 /* Call this function from the debugger to dump LOOPS. */
10773 debug_loops (const struct loops *loops)
10775 flow_loops_dump (loops, stderr, loop_dump_aux, 1);
OSDN Git Service
