* tree-data-ref.c (dr_analyze_alias): Handle case smt is NULL.

[pf3gnuchains/gcc-fork.git] / gcc / expr.c

/* Convert tree expression to rtl instructions, for GNU compiler.
   Copyright (C) 1988, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007
   Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 2, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING.  If not, write to the Free
Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "machmode.h"
#include "real.h"
#include "rtl.h"
#include "tree.h"
#include "flags.h"
#include "regs.h"
#include "hard-reg-set.h"
#include "except.h"
#include "function.h"
#include "insn-config.h"
#include "insn-attr.h"
/* Include expr.h after insn-config.h so we get HAVE_conditional_move.  */
#include "expr.h"
#include "optabs.h"
#include "libfuncs.h"
#include "recog.h"
#include "reload.h"
#include "output.h"
#include "typeclass.h"
#include "toplev.h"
#include "ggc.h"
#include "langhooks.h"
#include "intl.h"
#include "tm_p.h"
#include "tree-iterator.h"
#include "tree-pass.h"
#include "tree-flow.h"
#include "target.h"
#include "timevar.h"

/* Decide whether a function's arguments should be processed
   from first to last or from last to first.

   They should if the stack and args grow in opposite directions, but
   only if we have push insns.  */

#ifdef PUSH_ROUNDING

#ifndef PUSH_ARGS_REVERSED
#if defined (STACK_GROWS_DOWNWARD) != defined (ARGS_GROW_DOWNWARD)
#define PUSH_ARGS_REVERSED      /* If it's last to first.  */
#endif
#endif

#endif

#ifndef STACK_PUSH_CODE
#ifdef STACK_GROWS_DOWNWARD
#define STACK_PUSH_CODE PRE_DEC
#else
#define STACK_PUSH_CODE PRE_INC
#endif
#endif


/* If this is nonzero, we do not bother generating VOLATILE
   around volatile memory references, and we are willing to
   output indirect addresses.  If cse is to follow, we reject
   indirect addresses so a useful potential cse is generated;
   if it is used only once, instruction combination will produce
   the same indirect address eventually.  */
int cse_not_expected;

/* This structure is used by move_by_pieces to describe the move to
   be performed.  */
struct move_by_pieces
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  rtx from;
  rtx from_addr;
  int autinc_from;
  int explicit_inc_from;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  int reverse;
};

/* This structure is used by store_by_pieces to describe the clear to
   be performed.  */

struct store_by_pieces
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode);
  void *constfundata;
  int reverse;
};

static unsigned HOST_WIDE_INT move_by_pieces_ninsns (unsigned HOST_WIDE_INT,
                                                     unsigned int,
                                                     unsigned int);
static void move_by_pieces_1 (rtx (*) (rtx, ...), enum machine_mode,
                              struct move_by_pieces *);
static bool block_move_libcall_safe_for_call_parm (void);
static bool emit_block_move_via_movmem (rtx, rtx, rtx, unsigned, unsigned, HOST_WIDE_INT);
static tree emit_block_move_libcall_fn (int);
static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned);
static rtx clear_by_pieces_1 (void *, HOST_WIDE_INT, enum machine_mode);
static void clear_by_pieces (rtx, unsigned HOST_WIDE_INT, unsigned int);
static void store_by_pieces_1 (struct store_by_pieces *, unsigned int);
static void store_by_pieces_2 (rtx (*) (rtx, ...), enum machine_mode,
                               struct store_by_pieces *);
static tree clear_storage_libcall_fn (int);
static rtx compress_float_constant (rtx, rtx);
static rtx get_subtarget (rtx);
static void store_constructor_field (rtx, unsigned HOST_WIDE_INT,
                                     HOST_WIDE_INT, enum machine_mode,
                                     tree, tree, int, int);
static void store_constructor (tree, rtx, int, HOST_WIDE_INT);
static rtx store_field (rtx, HOST_WIDE_INT, HOST_WIDE_INT, enum machine_mode,
                        tree, tree, int, bool);

static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (tree, tree);

static int is_aligning_offset (tree, tree);
static void expand_operands (tree, tree, rtx, rtx*, rtx*,
                             enum expand_modifier);
static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
static rtx do_store_flag (tree, rtx, enum machine_mode, int);
#ifdef PUSH_ROUNDING
static void emit_single_push_insn (enum machine_mode, rtx, tree);
#endif
static void do_tablejump (rtx, enum machine_mode, rtx, rtx, rtx);
static rtx const_vector_from_tree (tree);
static void write_complex_part (rtx, rtx, bool);

/* Record for each mode whether we can move a register directly to or
   from an object of that mode in memory.  If we can't, we won't try
   to use that mode directly when accessing a field of that mode.  */

static char direct_load[NUM_MACHINE_MODES];
static char direct_store[NUM_MACHINE_MODES];

/* Record for each mode whether we can float-extend from memory.  */

static bool float_extend_from_mem[NUM_MACHINE_MODES][NUM_MACHINE_MODES];

/* This macro is used to determine whether move_by_pieces should be called
   to perform a structure copy.  */
#ifndef MOVE_BY_PIECES_P
#define MOVE_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, MOVE_MAX_PIECES + 1) \
   < (unsigned int) MOVE_RATIO)
#endif

/* This macro is used to determine whether clear_by_pieces should be
   called to clear storage.  */
#ifndef CLEAR_BY_PIECES_P
#define CLEAR_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, STORE_MAX_PIECES + 1) \
   < (unsigned int) CLEAR_RATIO)
#endif

/* This macro is used to determine whether store_by_pieces should be
   called to "memset" storage with byte values other than zero, or
   to "memcpy" storage when the source is a constant string.  */
#ifndef STORE_BY_PIECES_P
#define STORE_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, STORE_MAX_PIECES + 1) \
   < (unsigned int) MOVE_RATIO)
#endif

/* This array records the insn_code of insns to perform block moves.  */
enum insn_code movmem_optab[NUM_MACHINE_MODES];

/* This array records the insn_code of insns to perform block sets.  */
enum insn_code setmem_optab[NUM_MACHINE_MODES];

/* These arrays record the insn_code of three different kinds of insns
   to perform block compares.  */
enum insn_code cmpstr_optab[NUM_MACHINE_MODES];
enum insn_code cmpstrn_optab[NUM_MACHINE_MODES];
enum insn_code cmpmem_optab[NUM_MACHINE_MODES];

/* Synchronization primitives.  */
enum insn_code sync_add_optab[NUM_MACHINE_MODES];
enum insn_code sync_sub_optab[NUM_MACHINE_MODES];
enum insn_code sync_ior_optab[NUM_MACHINE_MODES];
enum insn_code sync_and_optab[NUM_MACHINE_MODES];
enum insn_code sync_xor_optab[NUM_MACHINE_MODES];
enum insn_code sync_nand_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_add_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_sub_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_ior_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_and_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_xor_optab[NUM_MACHINE_MODES];
enum insn_code sync_old_nand_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_add_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_sub_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_ior_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_and_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_xor_optab[NUM_MACHINE_MODES];
enum insn_code sync_new_nand_optab[NUM_MACHINE_MODES];
enum insn_code sync_compare_and_swap[NUM_MACHINE_MODES];
enum insn_code sync_compare_and_swap_cc[NUM_MACHINE_MODES];
enum insn_code sync_lock_test_and_set[NUM_MACHINE_MODES];
enum insn_code sync_lock_release[NUM_MACHINE_MODES];

/* SLOW_UNALIGNED_ACCESS is nonzero if unaligned accesses are very slow.  */

#ifndef SLOW_UNALIGNED_ACCESS
#define SLOW_UNALIGNED_ACCESS(MODE, ALIGN) STRICT_ALIGNMENT
#endif
\f
/* This is run once per compilation to set up which modes can be used
   directly in memory and to initialize the block move optab.  */

void
init_expr_once (void)
{
  rtx insn, pat;
  enum machine_mode mode;
  int num_clobbers;
  rtx mem, mem1;
  rtx reg;

  /* Try indexing by frame ptr and try by stack ptr.
     It is known that on the Convex the stack ptr isn't a valid index.
     With luck, one or the other is valid on any machine.  */
  mem = gen_rtx_MEM (VOIDmode, stack_pointer_rtx);
  mem1 = gen_rtx_MEM (VOIDmode, frame_pointer_rtx);

  /* A scratch register we can modify in-place below to avoid
     useless RTL allocations.  */
  reg = gen_rtx_REG (VOIDmode, -1);

  insn = rtx_alloc (INSN);
  pat = gen_rtx_SET (0, NULL_RTX, NULL_RTX);
  PATTERN (insn) = pat;

  for (mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
       mode = (enum machine_mode) ((int) mode + 1))
    {
      int regno;

      direct_load[(int) mode] = direct_store[(int) mode] = 0;
      PUT_MODE (mem, mode);
      PUT_MODE (mem1, mode);
      PUT_MODE (reg, mode);

      /* See if there is some register that can be used in this mode and
         directly loaded or stored from memory.  */

      if (mode != VOIDmode && mode != BLKmode)
        for (regno = 0; regno < FIRST_PSEUDO_REGISTER
             && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
             regno++)
          {
            if (! HARD_REGNO_MODE_OK (regno, mode))
              continue;

            REGNO (reg) = regno;

            SET_SRC (pat) = mem;
            SET_DEST (pat) = reg;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_load[(int) mode] = 1;

            SET_SRC (pat) = mem1;
            SET_DEST (pat) = reg;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_load[(int) mode] = 1;

            SET_SRC (pat) = reg;
            SET_DEST (pat) = mem;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_store[(int) mode] = 1;

            SET_SRC (pat) = reg;
            SET_DEST (pat) = mem1;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_store[(int) mode] = 1;
          }
    }

  mem = gen_rtx_MEM (VOIDmode, gen_rtx_raw_REG (Pmode, 10000));

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum machine_mode srcmode;
      for (srcmode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); srcmode != mode;
           srcmode = GET_MODE_WIDER_MODE (srcmode))
        {
          enum insn_code ic;

          ic = can_extend_p (mode, srcmode, 0);
          if (ic == CODE_FOR_nothing)
            continue;

          PUT_MODE (mem, srcmode);

          if ((*insn_data[ic].operand[1].predicate) (mem, srcmode))
            float_extend_from_mem[mode][srcmode] = true;
        }
    }
}

/* This is run at the start of compiling a function.  */

void
init_expr (void)
{
  cfun->expr = ggc_alloc_cleared (sizeof (struct expr_status));
}
\f
/* Copy data from FROM to TO, where the machine modes are not the same.
   Both modes may be integer, or both may be floating.
   UNSIGNEDP should be nonzero if FROM is an unsigned type.
   This causes zero-extension instead of sign-extension.  */

void
convert_move (rtx to, rtx from, int unsignedp)
{
  enum machine_mode to_mode = GET_MODE (to);
  enum machine_mode from_mode = GET_MODE (from);
  int to_real = SCALAR_FLOAT_MODE_P (to_mode);
  int from_real = SCALAR_FLOAT_MODE_P (from_mode);
  enum insn_code code;
  rtx libcall;

  /* rtx code for making an equivalent value.  */
  enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
                              : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));


  gcc_assert (to_real == from_real);
  gcc_assert (to_mode != BLKmode);
  gcc_assert (from_mode != BLKmode);

  /* If the source and destination are already the same, then there's
     nothing to do.  */
  if (to == from)
    return;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  We don't handle such SUBREGs as
     TO here.  */

  if (GET_CODE (from) == SUBREG && SUBREG_PROMOTED_VAR_P (from)
      && (GET_MODE_SIZE (GET_MODE (SUBREG_REG (from)))
          >= GET_MODE_SIZE (to_mode))
      && SUBREG_PROMOTED_UNSIGNED_P (from) == unsignedp)
    from = gen_lowpart (to_mode, from), from_mode = to_mode;

  gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));

  if (to_mode == from_mode
      || (from_mode == VOIDmode && CONSTANT_P (from)))
    {
      emit_move_insn (to, from);
      return;
    }

  if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
    {
      gcc_assert (GET_MODE_BITSIZE (from_mode) == GET_MODE_BITSIZE (to_mode));

      if (VECTOR_MODE_P (to_mode))
        from = simplify_gen_subreg (to_mode, from, GET_MODE (from), 0);
      else
        to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);

      emit_move_insn (to, from);
      return;
    }

  if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
    {
      convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
      convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
      return;
    }

  if (to_real)
    {
      rtx value, insns;
      convert_optab tab;

      gcc_assert ((GET_MODE_PRECISION (from_mode)
                   != GET_MODE_PRECISION (to_mode))
                  || (DECIMAL_FLOAT_MODE_P (from_mode)
                      != DECIMAL_FLOAT_MODE_P (to_mode)));

      if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
        /* Conversion between decimal float and binary float, same size.  */
        tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
      else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
        tab = sext_optab;
      else
        tab = trunc_optab;

      /* Try converting directly if the insn is supported.  */

      code = tab->handlers[to_mode][from_mode].insn_code;
      if (code != CODE_FOR_nothing)
        {
          emit_unop_insn (code, to, from,
                          tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
          return;
        }

      /* Otherwise use a libcall.  */
      libcall = tab->handlers[to_mode][from_mode].libfunc;

      /* Is this conversion implemented yet?  */
      gcc_assert (libcall);

      start_sequence ();
      value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
                                       1, from, from_mode);
      insns = get_insns ();
      end_sequence ();
      emit_libcall_block (insns, to, value,
                          tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
                                                                       from)
                          : gen_rtx_FLOAT_EXTEND (to_mode, from));
      return;
    }

  /* Handle pointer conversion.  */                     /* SPEE 900220.  */
  /* Targets are expected to provide conversion insns between PxImode and
     xImode for all MODE_PARTIAL_INT modes they use, but no others.  */
  if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
    {
      enum machine_mode full_mode
        = smallest_mode_for_size (GET_MODE_BITSIZE (to_mode), MODE_INT);

      gcc_assert (trunc_optab->handlers[to_mode][full_mode].insn_code
                  != CODE_FOR_nothing);

      if (full_mode != from_mode)
        from = convert_to_mode (full_mode, from, unsignedp);
      emit_unop_insn (trunc_optab->handlers[to_mode][full_mode].insn_code,
                      to, from, UNKNOWN);
      return;
    }
  if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
    {
      rtx new_from;
      enum machine_mode full_mode
        = smallest_mode_for_size (GET_MODE_BITSIZE (from_mode), MODE_INT);

      gcc_assert (sext_optab->handlers[full_mode][from_mode].insn_code
                  != CODE_FOR_nothing);

      if (to_mode == full_mode)
        {
          emit_unop_insn (sext_optab->handlers[full_mode][from_mode].insn_code,
                          to, from, UNKNOWN);
          return;
        }

      new_from = gen_reg_rtx (full_mode);
      emit_unop_insn (sext_optab->handlers[full_mode][from_mode].insn_code,
                      new_from, from, UNKNOWN);

      /* else proceed to integer conversions below.  */
      from_mode = full_mode;
      from = new_from;
    }

  /* Now both modes are integers.  */

  /* Handle expanding beyond a word.  */
  if (GET_MODE_BITSIZE (from_mode) < GET_MODE_BITSIZE (to_mode)
      && GET_MODE_BITSIZE (to_mode) > BITS_PER_WORD)
    {
      rtx insns;
      rtx lowpart;
      rtx fill_value;
      rtx lowfrom;
      int i;
      enum machine_mode lowpart_mode;
      int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);

      /* Try converting directly if the insn is supported.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
          != CODE_FOR_nothing)
        {
          /* If FROM is a SUBREG, put it into a register.  Do this
             so that we always generate the same set of insns for
             better cse'ing; if an intermediate assignment occurred,
             we won't be doing the operation directly on the SUBREG.  */
          if (optimize > 0 && GET_CODE (from) == SUBREG)
            from = force_reg (from_mode, from);
          emit_unop_insn (code, to, from, equiv_code);
          return;
        }
      /* Next, try converting via full word.  */
      else if (GET_MODE_BITSIZE (from_mode) < BITS_PER_WORD
               && ((code = can_extend_p (to_mode, word_mode, unsignedp))
                   != CODE_FOR_nothing))
        {
          if (REG_P (to))
            {
              if (reg_overlap_mentioned_p (to, from))
                from = force_reg (from_mode, from);
              emit_insn (gen_rtx_CLOBBER (VOIDmode, to));
            }
          convert_move (gen_lowpart (word_mode, to), from, unsignedp);
          emit_unop_insn (code, to,
                          gen_lowpart (word_mode, to), equiv_code);
          return;
        }

      /* No special multiword conversion insn; do it by hand.  */
      start_sequence ();

      /* Since we will turn this into a no conflict block, we must ensure
         that the source does not overlap the target.  */

      if (reg_overlap_mentioned_p (to, from))
        from = force_reg (from_mode, from);

      /* Get a copy of FROM widened to a word, if necessary.  */
      if (GET_MODE_BITSIZE (from_mode) < BITS_PER_WORD)
        lowpart_mode = word_mode;
      else
        lowpart_mode = from_mode;

      lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);

      lowpart = gen_lowpart (lowpart_mode, to);
      emit_move_insn (lowpart, lowfrom);

      /* Compute the value to put in each remaining word.  */
      if (unsignedp)
        fill_value = const0_rtx;
      else
        {
#ifdef HAVE_slt
          if (HAVE_slt
              && insn_data[(int) CODE_FOR_slt].operand[0].mode == word_mode
              && STORE_FLAG_VALUE == -1)
            {
              emit_cmp_insn (lowfrom, const0_rtx, NE, NULL_RTX,
                             lowpart_mode, 0);
              fill_value = gen_reg_rtx (word_mode);
              emit_insn (gen_slt (fill_value));
            }
          else
#endif
            {
              fill_value
                = expand_shift (RSHIFT_EXPR, lowpart_mode, lowfrom,
                                size_int (GET_MODE_BITSIZE (lowpart_mode) - 1),
                                NULL_RTX, 0);
              fill_value = convert_to_mode (word_mode, fill_value, 1);
            }
        }

      /* Fill the remaining words.  */
      for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
        {
          int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
          rtx subword = operand_subword (to, index, 1, to_mode);

          gcc_assert (subword);

          if (fill_value != subword)
            emit_move_insn (subword, fill_value);
        }

      insns = get_insns ();
      end_sequence ();

      emit_no_conflict_block (insns, to, from, NULL_RTX,
                              gen_rtx_fmt_e (equiv_code, to_mode, copy_rtx (from)));
      return;
    }

  /* Truncating multi-word to a word or less.  */
  if (GET_MODE_BITSIZE (from_mode) > BITS_PER_WORD
      && GET_MODE_BITSIZE (to_mode) <= BITS_PER_WORD)
    {
      if (!((MEM_P (from)
             && ! MEM_VOLATILE_P (from)
             && direct_load[(int) to_mode]
             && ! mode_dependent_address_p (XEXP (from, 0)))
            || REG_P (from)
            || GET_CODE (from) == SUBREG))
        from = force_reg (from_mode, from);
      convert_move (to, gen_lowpart (word_mode, from), 0);
      return;
    }

  /* Now follow all the conversions between integers
     no more than a word long.  */

  /* For truncation, usually we can just refer to FROM in a narrower mode.  */
  if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
      && TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (to_mode),
                                GET_MODE_BITSIZE (from_mode)))
    {
      if (!((MEM_P (from)
             && ! MEM_VOLATILE_P (from)
             && direct_load[(int) to_mode]
             && ! mode_dependent_address_p (XEXP (from, 0)))
            || REG_P (from)
            || GET_CODE (from) == SUBREG))
        from = force_reg (from_mode, from);
      if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
          && ! HARD_REGNO_MODE_OK (REGNO (from), to_mode))
        from = copy_to_reg (from);
      emit_move_insn (to, gen_lowpart (to_mode, from));
      return;
    }

  /* Handle extension.  */
  if (GET_MODE_BITSIZE (to_mode) > GET_MODE_BITSIZE (from_mode))
    {
      /* Convert directly if that works.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
          != CODE_FOR_nothing)
        {
          emit_unop_insn (code, to, from, equiv_code);
          return;
        }
      else
        {
          enum machine_mode intermediate;
          rtx tmp;
          tree shift_amount;

          /* Search for a mode to convert via.  */
          for (intermediate = from_mode; intermediate != VOIDmode;
               intermediate = GET_MODE_WIDER_MODE (intermediate))
            if (((can_extend_p (to_mode, intermediate, unsignedp)
                  != CODE_FOR_nothing)
                 || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
                     && TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (to_mode),
                                               GET_MODE_BITSIZE (intermediate))))
                && (can_extend_p (intermediate, from_mode, unsignedp)
                    != CODE_FOR_nothing))
              {
                convert_move (to, convert_to_mode (intermediate, from,
                                                   unsignedp), unsignedp);
                return;
              }

          /* No suitable intermediate mode.
             Generate what we need with shifts.  */
          shift_amount = build_int_cst (NULL_TREE,
                                        GET_MODE_BITSIZE (to_mode)
                                        - GET_MODE_BITSIZE (from_mode));
          from = gen_lowpart (to_mode, force_reg (from_mode, from));
          tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
                              to, unsignedp);
          tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
                              to, unsignedp);
          if (tmp != to)
            emit_move_insn (to, tmp);
          return;
        }
    }

  /* Support special truncate insns for certain modes.  */
  if (trunc_optab->handlers[to_mode][from_mode].insn_code != CODE_FOR_nothing)
    {
      emit_unop_insn (trunc_optab->handlers[to_mode][from_mode].insn_code,
                      to, from, UNKNOWN);
      return;
    }

  /* Handle truncation of volatile memrefs, and so on;
     the things that couldn't be truncated directly,
     and for which there was no special instruction.

     ??? Code above formerly short-circuited this, for most integer
     mode pairs, with a force_reg in from_mode followed by a recursive
     call to this routine.  Appears always to have been wrong.  */
  if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode))
    {
      rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
      emit_move_insn (to, temp);
      return;
    }

  /* Mode combination is not recognized.  */
  gcc_unreachable ();
}

/* Return an rtx for a value that would result
   from converting X to mode MODE.
   Both X and MODE may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.
   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.  */

rtx
convert_to_mode (enum machine_mode mode, rtx x, int unsignedp)
{
  return convert_modes (mode, VOIDmode, x, unsignedp);
}

/* Return an rtx for a value that would result
   from converting X from mode OLDMODE to mode MODE.
   Both modes may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.

   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.

   You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode.  */

rtx
convert_modes (enum machine_mode mode, enum machine_mode oldmode, rtx x, int unsignedp)
{
  rtx temp;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  */

  if (GET_CODE (x) == SUBREG && SUBREG_PROMOTED_VAR_P (x)
      && GET_MODE_SIZE (GET_MODE (SUBREG_REG (x))) >= GET_MODE_SIZE (mode)
      && SUBREG_PROMOTED_UNSIGNED_P (x) == unsignedp)
    x = gen_lowpart (mode, x);

  if (GET_MODE (x) != VOIDmode)
    oldmode = GET_MODE (x);

  if (mode == oldmode)
    return x;

  /* There is one case that we must handle specially: If we are converting
     a CONST_INT into a mode whose size is twice HOST_BITS_PER_WIDE_INT and
     we are to interpret the constant as unsigned, gen_lowpart will do
     the wrong if the constant appears negative.  What we want to do is
     make the high-order word of the constant zero, not all ones.  */

  if (unsignedp && GET_MODE_CLASS (mode) == MODE_INT
      && GET_MODE_BITSIZE (mode) == 2 * HOST_BITS_PER_WIDE_INT
      && GET_CODE (x) == CONST_INT && INTVAL (x) < 0)
    {
      HOST_WIDE_INT val = INTVAL (x);

      if (oldmode != VOIDmode
          && HOST_BITS_PER_WIDE_INT > GET_MODE_BITSIZE (oldmode))
        {
          int width = GET_MODE_BITSIZE (oldmode);

          /* We need to zero extend VAL.  */
          val &= ((HOST_WIDE_INT) 1 << width) - 1;
        }

      return immed_double_const (val, (HOST_WIDE_INT) 0, mode);
    }

  /* We can do this with a gen_lowpart if both desired and current modes
     are integer, and this is either a constant integer, a register, or a
     non-volatile MEM.  Except for the constant case where MODE is no
     wider than HOST_BITS_PER_WIDE_INT, we must be narrowing the operand.  */

  if ((GET_CODE (x) == CONST_INT
       && GET_MODE_BITSIZE (mode) <= HOST_BITS_PER_WIDE_INT)
      || (GET_MODE_CLASS (mode) == MODE_INT
          && GET_MODE_CLASS (oldmode) == MODE_INT
          && (GET_CODE (x) == CONST_DOUBLE
              || (GET_MODE_SIZE (mode) <= GET_MODE_SIZE (oldmode)
                  && ((MEM_P (x) && ! MEM_VOLATILE_P (x)
                       && direct_load[(int) mode])
                      || (REG_P (x)
                          && (! HARD_REGISTER_P (x)
                              || HARD_REGNO_MODE_OK (REGNO (x), mode))
                          && TRULY_NOOP_TRUNCATION (GET_MODE_BITSIZE (mode),
                                                    GET_MODE_BITSIZE (GET_MODE (x)))))))))
    {
      /* ?? If we don't know OLDMODE, we have to assume here that
         X does not need sign- or zero-extension.   This may not be
         the case, but it's the best we can do.  */
      if (GET_CODE (x) == CONST_INT && oldmode != VOIDmode
          && GET_MODE_SIZE (mode) > GET_MODE_SIZE (oldmode))
        {
          HOST_WIDE_INT val = INTVAL (x);
          int width = GET_MODE_BITSIZE (oldmode);

          /* We must sign or zero-extend in this case.  Start by
             zero-extending, then sign extend if we need to.  */
          val &= ((HOST_WIDE_INT) 1 << width) - 1;
          if (! unsignedp
              && (val & ((HOST_WIDE_INT) 1 << (width - 1))))
            val |= (HOST_WIDE_INT) (-1) << width;

          return gen_int_mode (val, mode);
        }

      return gen_lowpart (mode, x);
    }

  /* Converting from integer constant into mode is always equivalent to an
     subreg operation.  */
  if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
    {
      gcc_assert (GET_MODE_BITSIZE (mode) == GET_MODE_BITSIZE (oldmode));
      return simplify_gen_subreg (mode, x, oldmode, 0);
    }

  temp = gen_reg_rtx (mode);
  convert_move (temp, x, unsignedp);
  return temp;
}
\f
/* STORE_MAX_PIECES is the number of bytes at a time that we can
   store efficiently.  Due to internal GCC limitations, this is
   MOVE_MAX_PIECES limited by the number of bytes GCC can represent
   for an immediate constant.  */

#define STORE_MAX_PIECES  MIN (MOVE_MAX_PIECES, 2 * sizeof (HOST_WIDE_INT))

/* Determine whether the LEN bytes can be moved by using several move
   instructions.  Return nonzero if a call to move_by_pieces should
   succeed.  */

int
can_move_by_pieces (unsigned HOST_WIDE_INT len,
                    unsigned int align ATTRIBUTE_UNUSED)
{
  return MOVE_BY_PIECES_P (len, align);
}

/* Generate several move instructions to copy LEN bytes from block FROM to
   block TO.  (These are MEM rtx's with BLKmode).

   If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
   used to push FROM to the stack.

   ALIGN is maximum stack alignment we can assume.

   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
                unsigned int align, int endp)
{
  struct move_by_pieces data;
  rtx to_addr, from_addr = XEXP (from, 0);
  unsigned int max_size = MOVE_MAX_PIECES + 1;
  enum machine_mode mode = VOIDmode, tmode;
  enum insn_code icode;

  align = MIN (to ? MEM_ALIGN (to) : align, MEM_ALIGN (from));

  data.offset = 0;
  data.from_addr = from_addr;
  if (to)
    {
      to_addr = XEXP (to, 0);
      data.to = to;
      data.autinc_to
        = (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
           || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);
      data.reverse
        = (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
    }
  else
    {
      to_addr = NULL_RTX;
      data.to = NULL_RTX;
      data.autinc_to = 1;
#ifdef STACK_GROWS_DOWNWARD
      data.reverse = 1;
#else
      data.reverse = 0;
#endif
    }
  data.to_addr = to_addr;
  data.from = from;
  data.autinc_from
    = (GET_CODE (from_addr) == PRE_INC || GET_CODE (from_addr) == PRE_DEC
       || GET_CODE (from_addr) == POST_INC
       || GET_CODE (from_addr) == POST_DEC);

  data.explicit_inc_from = 0;
  data.explicit_inc_to = 0;
  if (data.reverse) data.offset = len;
  data.len = len;

  /* If copying requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!(data.autinc_from && data.autinc_to)
      && move_by_pieces_ninsns (len, align, max_size) > 2)
    {
      /* Find the mode of the largest move...  */
      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) < max_size)
          mode = tmode;

      if (USE_LOAD_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_from)
        {
          data.from_addr = copy_addr_to_reg (plus_constant (from_addr, len));
          data.autinc_from = 1;
          data.explicit_inc_from = -1;
        }
      if (USE_LOAD_POST_INCREMENT (mode) && ! data.autinc_from)
        {
          data.from_addr = copy_addr_to_reg (from_addr);
          data.autinc_from = 1;
          data.explicit_inc_from = 1;
        }
      if (!data.autinc_from && CONSTANT_P (from_addr))
        data.from_addr = copy_addr_to_reg (from_addr);
      if (USE_STORE_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_to)
        {
          data.to_addr = copy_addr_to_reg (plus_constant (to_addr, len));
          data.autinc_to = 1;
          data.explicit_inc_to = -1;
        }
      if (USE_STORE_POST_INCREMENT (mode) && ! data.reverse && ! data.autinc_to)
        {
          data.to_addr = copy_addr_to_reg (to_addr);
          data.autinc_to = 1;
          data.explicit_inc_to = 1;
        }
      if (!data.autinc_to && CONSTANT_P (to_addr))
        data.to_addr = copy_addr_to_reg (to_addr);
    }

  tmode = mode_for_size (MOVE_MAX_PIECES * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      enum machine_mode xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
           tmode != VOIDmode;
           xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) > MOVE_MAX_PIECES
            || SLOW_UNALIGNED_ACCESS (tmode, align))
          break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  /* First move what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1)
    {
      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) < max_size)
          mode = tmode;

      if (mode == VOIDmode)
        break;

      icode = mov_optab->handlers[(int) mode].insn_code;
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        move_by_pieces_1 (GEN_FCN (icode), mode, &data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data.len);

  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
        {
          if (endp == 2)
            {
              if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
                emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
              else
                data.to_addr = copy_addr_to_reg (plus_constant (data.to_addr,
                                                                -1));
            }
          to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
                                           data.offset);
        }
      else
        {
          if (endp == 2)
            --data.offset;
          to1 = adjust_address (data.to, QImode, data.offset);
        }
      return to1;
    }
  else
    return data.to;
}

/* Return number of insns required to move L bytes by pieces.
   ALIGN (in bits) is maximum alignment we can assume.  */

static unsigned HOST_WIDE_INT
move_by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
                       unsigned int max_size)
{
  unsigned HOST_WIDE_INT n_insns = 0;
  enum machine_mode tmode;

  tmode = mode_for_size (MOVE_MAX_PIECES * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      enum machine_mode tmode, xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
           tmode != VOIDmode;
           xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) > MOVE_MAX_PIECES
            || SLOW_UNALIGNED_ACCESS (tmode, align))
          break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  while (max_size > 1)
    {
      enum machine_mode mode = VOIDmode;
      enum insn_code icode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) < max_size)
          mode = tmode;

      if (mode == VOIDmode)
        break;

      icode = mov_optab->handlers[(int) mode].insn_code;
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        n_insns += l / GET_MODE_SIZE (mode), l %= GET_MODE_SIZE (mode);

      max_size = GET_MODE_SIZE (mode);
    }

  gcc_assert (!l);
  return n_insns;
}

/* Subroutine of move_by_pieces.  Move as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
move_by_pieces_1 (rtx (*genfun) (rtx, ...), enum machine_mode mode,
                  struct move_by_pieces *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1 = NULL_RTX, from1;

  while (data->len >= size)
    {
      if (data->reverse)
        data->offset -= size;

      if (data->to)
        {
          if (data->autinc_to)
            to1 = adjust_automodify_address (data->to, mode, data->to_addr,
                                             data->offset);
          else
            to1 = adjust_address (data->to, mode, data->offset);
        }

      if (data->autinc_from)
        from1 = adjust_automodify_address (data->from, mode, data->from_addr,
                                           data->offset);
      else
        from1 = adjust_address (data->from, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
        emit_insn (gen_add2_insn (data->to_addr,
                                  GEN_INT (-(HOST_WIDE_INT)size)));
      if (HAVE_PRE_DECREMENT && data->explicit_inc_from < 0)
        emit_insn (gen_add2_insn (data->from_addr,
                                  GEN_INT (-(HOST_WIDE_INT)size)));

      if (data->to)
        emit_insn ((*genfun) (to1, from1));
      else
        {
#ifdef PUSH_ROUNDING
          emit_single_push_insn (mode, from1, NULL);
#else
          gcc_unreachable ();
#endif
        }

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
        emit_insn (gen_add2_insn (data->to_addr, GEN_INT (size)));
      if (HAVE_POST_INCREMENT && data->explicit_inc_from > 0)
        emit_insn (gen_add2_insn (data->from_addr, GEN_INT (size)));

      if (! data->reverse)
        data->offset += size;

      data->len -= size;
    }
}
\f
/* Emit code to move a block Y to a block X.  This may be done with
   string-move instructions, with multiple scalar move instructions,
   or with a library call.

   Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
   SIZE is an rtx that says how long they are.
   ALIGN is the maximum alignment we can assume they have.
   METHOD describes what kind of copy this is, and what mechanisms may be used.

   Return the address of the new block, if memcpy is called and returns it,
   0 otherwise.  */

rtx
emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
                       unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  bool may_use_call;
  rtx retval = 0;
  unsigned int align;

  switch (method)
    {
    case BLOCK_OP_NORMAL:
    case BLOCK_OP_TAILCALL:
      may_use_call = true;
      break;

    case BLOCK_OP_CALL_PARM:
      may_use_call = block_move_libcall_safe_for_call_parm ();

      /* Make inhibit_defer_pop nonzero around the library call
         to force it to pop the arguments right away.  */
      NO_DEFER_POP;
      break;

    case BLOCK_OP_NO_LIBCALL:
      may_use_call = false;
      break;

    default:
      gcc_unreachable ();
    }

  align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));

  gcc_assert (MEM_P (x));
  gcc_assert (MEM_P (y));
  gcc_assert (size);

  /* Make sure we've got BLKmode addresses; store_one_arg can decide that
     block copy is more efficient for other large modes, e.g. DCmode.  */
  x = adjust_address (x, BLKmode, 0);
  y = adjust_address (y, BLKmode, 0);

  /* Set MEM_SIZE as appropriate for this block copy.  The main place this
     can be incorrect is coming from __builtin_memcpy.  */
  if (GET_CODE (size) == CONST_INT)
    {
      if (INTVAL (size) == 0)
        return 0;

      x = shallow_copy_rtx (x);
      y = shallow_copy_rtx (y);
      set_mem_size (x, size);
      set_mem_size (y, size);
    }

  if (GET_CODE (size) == CONST_INT && MOVE_BY_PIECES_P (INTVAL (size), align))
    move_by_pieces (x, y, INTVAL (size), align, 0);
  else if (emit_block_move_via_movmem (x, y, size, align,
                                       expected_align, expected_size))
    ;
  else if (may_use_call)
    retval = emit_block_move_via_libcall (x, y, size,
                                          method == BLOCK_OP_TAILCALL);
  else
    emit_block_move_via_loop (x, y, size, align);

  if (method == BLOCK_OP_CALL_PARM)
    OK_DEFER_POP;

  return retval;
}

rtx
emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method)
{
  return emit_block_move_hints (x, y, size, method, 0, -1);
}

/* A subroutine of emit_block_move.  Returns true if calling the
   block move libcall will not clobber any parameters which may have
   already been placed on the stack.  */

static bool
block_move_libcall_safe_for_call_parm (void)
{
  /* If arguments are pushed on the stack, then they're safe.  */
  if (PUSH_ARGS)
    return true;

  /* If registers go on the stack anyway, any argument is sure to clobber
     an outgoing argument.  */
#if defined (REG_PARM_STACK_SPACE)
  if (OUTGOING_REG_PARM_STACK_SPACE)
    {
      tree fn;
      fn = emit_block_move_libcall_fn (false);
      if (REG_PARM_STACK_SPACE (fn) != 0)
        return false;
    }
#endif

  /* If any argument goes in memory, then it might clobber an outgoing
     argument.  */
  {
    CUMULATIVE_ARGS args_so_far;
    tree fn, arg;

    fn = emit_block_move_libcall_fn (false);
    INIT_CUMULATIVE_ARGS (args_so_far, TREE_TYPE (fn), NULL_RTX, 0, 3);

    arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
    for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
      {
        enum machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
        rtx tmp = FUNCTION_ARG (args_so_far, mode, NULL_TREE, 1);
        if (!tmp || !REG_P (tmp))
          return false;
        if (targetm.calls.arg_partial_bytes (&args_so_far, mode, NULL, 1))
          return false;
        FUNCTION_ARG_ADVANCE (args_so_far, mode, NULL_TREE, 1);
      }
  }
  return true;
}

/* A subroutine of emit_block_move.  Expand a movmem pattern;
   return true if successful.  */

static bool
emit_block_move_via_movmem (rtx x, rtx y, rtx size, unsigned int align,
                            unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  rtx opalign = GEN_INT (align / BITS_PER_UNIT);
  int save_volatile_ok = volatile_ok;
  enum machine_mode mode;

  if (expected_align < align)
    expected_align = align;

  /* Since this is a move insn, we don't care about volatility.  */
  volatile_ok = 1;

  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = movmem_optab[(int) mode];
      insn_operand_predicate_fn pred;

      if (code != CODE_FOR_nothing
          /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
             here because if SIZE is less than the mode mask, as it is
             returned by the macro, it will definitely be less than the
             actual mode mask.  */
          && ((GET_CODE (size) == CONST_INT
               && ((unsigned HOST_WIDE_INT) INTVAL (size)
                   <= (GET_MODE_MASK (mode) >> 1)))
              || GET_MODE_BITSIZE (mode) >= BITS_PER_WORD)
          && ((pred = insn_data[(int) code].operand[0].predicate) == 0
              || (*pred) (x, BLKmode))
          && ((pred = insn_data[(int) code].operand[1].predicate) == 0
              || (*pred) (y, BLKmode))
          && ((pred = insn_data[(int) code].operand[3].predicate) == 0
              || (*pred) (opalign, VOIDmode)))
        {
          rtx op2;
          rtx last = get_last_insn ();
          rtx pat;

          op2 = convert_to_mode (mode, size, 1);
          pred = insn_data[(int) code].operand[2].predicate;
          if (pred != 0 && ! (*pred) (op2, mode))
            op2 = copy_to_mode_reg (mode, op2);

          /* ??? When called via emit_block_move_for_call, it'd be
             nice if there were some way to inform the backend, so
             that it doesn't fail the expansion because it thinks
             emitting the libcall would be more efficient.  */

          if (insn_data[(int) code].n_operands == 4)
            pat = GEN_FCN ((int) code) (x, y, op2, opalign);
          else
            pat = GEN_FCN ((int) code) (x, y, op2, opalign,
                                        GEN_INT (expected_align),
                                        GEN_INT (expected_size));
          if (pat)
            {
              emit_insn (pat);
              volatile_ok = save_volatile_ok;
              return true;
            }
          else
            delete_insns_since (last);
        }
    }

  volatile_ok = save_volatile_ok;
  return false;
}

/* A subroutine of emit_block_move.  Expand a call to memcpy.
   Return the return value from memcpy, 0 otherwise.  */

rtx
emit_block_move_via_libcall (rtx dst, rtx src, rtx size, bool tailcall)
{
  rtx dst_addr, src_addr;
  tree call_expr, fn, src_tree, dst_tree, size_tree;
  enum machine_mode size_mode;
  rtx retval;

  /* Emit code to copy the addresses of DST and SRC and SIZE into new
     pseudos.  We can then place those new pseudos into a VAR_DECL and
     use them later.  */

  dst_addr = copy_to_mode_reg (Pmode, XEXP (dst, 0));
  src_addr = copy_to_mode_reg (Pmode, XEXP (src, 0));

  dst_addr = convert_memory_address (ptr_mode, dst_addr);
  src_addr = convert_memory_address (ptr_mode, src_addr);

  dst_tree = make_tree (ptr_type_node, dst_addr);
  src_tree = make_tree (ptr_type_node, src_addr);

  size_mode = TYPE_MODE (sizetype);

  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memcpy in this context.  This could be a user call to memcpy and
     the user may wish to examine the return value from memcpy.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  size_tree = make_tree (sizetype, size);

  fn = emit_block_move_libcall_fn (true);
  call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of emit_block_move_via_libcall.  Create the tree node
   for the function we use for block copies.  The first time FOR_CALL
   is true, we call assemble_external.  */

static GTY(()) tree block_move_fn;

void
init_block_move_fn (const char *asmspec)
{
  if (!block_move_fn)
    {
      tree args, fn;

      fn = get_identifier ("memcpy");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
                                       const_ptr_type_node, sizetype,
                                       NULL_TREE);

      fn = build_decl (FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      block_move_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_move_fn, asmspec);
}

static tree
emit_block_move_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_move_fn)
    init_block_move_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_move_fn);
      assemble_external (block_move_fn);
    }

  return block_move_fn;
}

/* A subroutine of emit_block_move.  Copy the data via an explicit
   loop.  This is used only when libcalls are forbidden.  */
/* ??? It'd be nice to copy in hunks larger than QImode.  */

static void
emit_block_move_via_loop (rtx x, rtx y, rtx size,
                          unsigned int align ATTRIBUTE_UNUSED)
{
  rtx cmp_label, top_label, iter, x_addr, y_addr, tmp;
  enum machine_mode iter_mode;

  iter_mode = GET_MODE (size);
  if (iter_mode == VOIDmode)
    iter_mode = word_mode;

  top_label = gen_label_rtx ();
  cmp_label = gen_label_rtx ();
  iter = gen_reg_rtx (iter_mode);

  emit_move_insn (iter, const0_rtx);

  x_addr = force_operand (XEXP (x, 0), NULL_RTX);
  y_addr = force_operand (XEXP (y, 0), NULL_RTX);
  do_pending_stack_adjust ();

  emit_jump (cmp_label);
  emit_label (top_label);

  tmp = convert_modes (Pmode, iter_mode, iter, true);
  x_addr = gen_rtx_PLUS (Pmode, x_addr, tmp);
  y_addr = gen_rtx_PLUS (Pmode, y_addr, tmp);
  x = change_address (x, QImode, x_addr);
  y = change_address (y, QImode, y_addr);

  emit_move_insn (x, y);

  tmp = expand_simple_binop (iter_mode, PLUS, iter, const1_rtx, iter,
                             true, OPTAB_LIB_WIDEN);
  if (tmp != iter)
    emit_move_insn (iter, tmp);

  emit_label (cmp_label);

  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
                           true, top_label);
}
\f
/* Copy all or part of a value X into registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_to_reg (int regno, rtx x, int nregs, enum machine_mode mode)
{
  int i;
#ifdef HAVE_load_multiple
  rtx pat;
  rtx last;
#endif

  if (nregs == 0)
    return;

  if (CONSTANT_P (x) && ! LEGITIMATE_CONSTANT_P (x))
    x = validize_mem (force_const_mem (mode, x));

  /* See if the machine can do this with a load multiple insn.  */
#ifdef HAVE_load_multiple
  if (HAVE_load_multiple)
    {
      last = get_last_insn ();
      pat = gen_load_multiple (gen_rtx_REG (word_mode, regno), x,
                               GEN_INT (nregs));
      if (pat)
        {
          emit_insn (pat);
          return;
        }
      else
        delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    emit_move_insn (gen_rtx_REG (word_mode, regno + i),
                    operand_subword_force (x, i, mode));
}

/* Copy all or part of a BLKmode value X out of registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_from_reg (int regno, rtx x, int nregs)
{
  int i;

  if (nregs == 0)
    return;

  /* See if the machine can do this with a store multiple insn.  */
#ifdef HAVE_store_multiple
  if (HAVE_store_multiple)
    {
      rtx last = get_last_insn ();
      rtx pat = gen_store_multiple (x, gen_rtx_REG (word_mode, regno),
                                    GEN_INT (nregs));
      if (pat)
        {
          emit_insn (pat);
          return;
        }
      else
        delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    {
      rtx tem = operand_subword (x, i, 1, BLKmode);

      gcc_assert (tem);

      emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
    }
}

/* Generate a PARALLEL rtx for a new non-consecutive group of registers from
   ORIG, where ORIG is a non-consecutive group of registers represented by
   a PARALLEL.  The clone is identical to the original except in that the
   original set of registers is replaced by a new set of pseudo registers.
   The new set has the same modes as the original set.  */

rtx
gen_group_rtx (rtx orig)
{
  int i, length;
  rtx *tmps;

  gcc_assert (GET_CODE (orig) == PARALLEL);

  length = XVECLEN (orig, 0);
  tmps = alloca (sizeof (rtx) * length);

  /* Skip a NULL entry in first slot.  */
  i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;

  if (i)
    tmps[0] = 0;

  for (; i < length; i++)
    {
      enum machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
      rtx offset = XEXP (XVECEXP (orig, 0, i), 1);

      tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
    }

  return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
}

/* A subroutine of emit_group_load.  Arguments as for emit_group_load,
   except that values are placed in TMPS[i], and must later be moved
   into corresponding XEXP (XVECEXP (DST, 0, i), 0) element.  */

static void
emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type, int ssize)
{
  rtx src;
  int start, i;
  enum machine_mode m = GET_MODE (orig_src);

  gcc_assert (GET_CODE (dst) == PARALLEL);

  if (m != VOIDmode
      && !SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_src)
      && GET_CODE (orig_src) != CONCAT)
    {
      enum machine_mode imode = int_mode_for_mode (GET_MODE (orig_src));
      if (imode == BLKmode)
        src = assign_stack_temp (GET_MODE (orig_src), ssize, 0);
      else
        src = gen_reg_rtx (imode);
      if (imode != BLKmode)
        src = gen_lowpart (GET_MODE (orig_src), src);
      emit_move_insn (src, orig_src);
      /* ...and back again.  */
      if (imode != BLKmode)
        src = gen_lowpart (imode, src);
      emit_group_load_1 (tmps, dst, src, type, ssize);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (dst, 0, 0), 0))
    start = 0;
  else
    start = 1;

  /* Process the pieces.  */
  for (i = start; i < XVECLEN (dst, 0); i++)
    {
      enum machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (dst, 0, i), 1));
      unsigned int bytelen = GET_MODE_SIZE (mode);
      int shift = 0;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
        {
          /* Arrange to shift the fragment to where it belongs.
             extract_bit_field loads to the lsb of the reg.  */
          if (
#ifdef BLOCK_REG_PADDING
              BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
              == (BYTES_BIG_ENDIAN ? upward : downward)
#else
              BYTES_BIG_ENDIAN
#endif
              )
            shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
          bytelen = ssize - bytepos;
          gcc_assert (bytelen > 0);
        }

      /* If we won't be loading directly from memory, protect the real source
         from strange tricks we might play; but make sure that the source can
         be loaded directly into the destination.  */
      src = orig_src;
      if (!MEM_P (orig_src)
          && (!CONSTANT_P (orig_src)
              || (GET_MODE (orig_src) != mode
                  && GET_MODE (orig_src) != VOIDmode)))
        {
          if (GET_MODE (orig_src) == VOIDmode)
            src = gen_reg_rtx (mode);
          else
            src = gen_reg_rtx (GET_MODE (orig_src));

          emit_move_insn (src, orig_src);
        }

      /* Optimize the access just a bit.  */
      if (MEM_P (src)
          && (! SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (src))
              || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
          && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
          && bytelen == GET_MODE_SIZE (mode))
        {
          tmps[i] = gen_reg_rtx (mode);
          emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
        }
      else if (COMPLEX_MODE_P (mode)
               && GET_MODE (src) == mode
               && bytelen == GET_MODE_SIZE (mode))
        /* Let emit_move_complex do the bulk of the work.  */
        tmps[i] = src;
      else if (GET_CODE (src) == CONCAT)
        {
          unsigned int slen = GET_MODE_SIZE (GET_MODE (src));
          unsigned int slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));

          if ((bytepos == 0 && bytelen == slen0)
              || (bytepos != 0 && bytepos + bytelen <= slen))
            {
              /* The following assumes that the concatenated objects all
                 have the same size.  In this case, a simple calculation
                 can be used to determine the object and the bit field
                 to be extracted.  */
              tmps[i] = XEXP (src, bytepos / slen0);
              if (! CONSTANT_P (tmps[i])
                  && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode))
                tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
                                             (bytepos % slen0) * BITS_PER_UNIT,
                                             1, NULL_RTX, mode, mode);
            }
          else
            {
              rtx mem;

              gcc_assert (!bytepos);
              mem = assign_stack_temp (GET_MODE (src), slen, 0);
              emit_move_insn (mem, src);
              tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
                                           0, 1, NULL_RTX, mode, mode);
            }
        }
      /* FIXME: A SIMD parallel will eventually lead to a subreg of a
         SIMD register, which is currently broken.  While we get GCC
         to emit proper RTL for these cases, let's dump to memory.  */
      else if (VECTOR_MODE_P (GET_MODE (dst))
               && REG_P (src))
        {
          int slen = GET_MODE_SIZE (GET_MODE (src));
          rtx mem;

          mem = assign_stack_temp (GET_MODE (src), slen, 0);
          emit_move_insn (mem, src);
          tmps[i] = adjust_address (mem, mode, (int) bytepos);
        }
      else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
               && XVECLEN (dst, 0) > 1)
        tmps[i] = simplify_gen_subreg (mode, src, GET_MODE(dst), bytepos);
      else if (CONSTANT_P (src)
               || (REG_P (src) && GET_MODE (src) == mode))
        tmps[i] = src;
      else
        tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
                                     bytepos * BITS_PER_UNIT, 1, NULL_RTX,
                                     mode, mode);

      if (shift)
        tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
                                build_int_cst (NULL_TREE, shift), tmps[i], 0);
    }
}

/* Emit code to move a block SRC of type TYPE to a block DST,
   where DST is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_SRC in bytes, or -1
   if not known.  */

void
emit_group_load (rtx dst, rtx src, tree type, int ssize)
{
  rtx *tmps;
  int i;

  tmps = alloca (sizeof (rtx) * XVECLEN (dst, 0));
  emit_group_load_1 (tmps, dst, src, type, ssize);

  /* Copy the extracted pieces into the proper (probable) hard regs.  */
  for (i = 0; i < XVECLEN (dst, 0); i++)
    {
      rtx d = XEXP (XVECEXP (dst, 0, i), 0);
      if (d == NULL)
        continue;
      emit_move_insn (d, tmps[i]);
    }
}

/* Similar, but load SRC into new pseudos in a format that looks like
   PARALLEL.  This can later be fed to emit_group_move to get things
   in the right place.  */

rtx
emit_group_load_into_temps (rtx parallel, rtx src, tree type, int ssize)
{
  rtvec vec;
  int i;

  vec = rtvec_alloc (XVECLEN (parallel, 0));
  emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);

  /* Convert the vector to look just like the original PARALLEL, except
     with the computed values.  */
  for (i = 0; i < XVECLEN (parallel, 0); i++)
    {
      rtx e = XVECEXP (parallel, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
        {
          d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
          e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
        }
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
}

/* Emit code to move a block SRC to block DST, where SRC and DST are
   non-consecutive groups of registers, each represented by a PARALLEL.  */

void
emit_group_move (rtx dst, rtx src)
{
  int i;

  gcc_assert (GET_CODE (src) == PARALLEL
              && GET_CODE (dst) == PARALLEL
              && XVECLEN (src, 0) == XVECLEN (dst, 0));

  /* Skip first entry if NULL.  */
  for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
    emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
                    XEXP (XVECEXP (src, 0, i), 0));
}

/* Move a group of registers represented by a PARALLEL into pseudos.  */

rtx
emit_group_move_into_temps (rtx src)
{
  rtvec vec = rtvec_alloc (XVECLEN (src, 0));
  int i;

  for (i = 0; i < XVECLEN (src, 0); i++)
    {
      rtx e = XVECEXP (src, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
        e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (src), vec);
}

/* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
   where SRC is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_DST, or -1 if not
   known.  */

void
emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED, int ssize)
{
  rtx *tmps, dst;
  int start, finish, i;
  enum machine_mode m = GET_MODE (orig_dst);

  gcc_assert (GET_CODE (src) == PARALLEL);

  if (!SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
    {
      enum machine_mode imode = int_mode_for_mode (GET_MODE (orig_dst));
      if (imode == BLKmode)
        dst = assign_stack_temp (GET_MODE (orig_dst), ssize, 0);
      else
        dst = gen_reg_rtx (imode);
      emit_group_store (dst, src, type, ssize);
      if (imode != BLKmode)
        dst = gen_lowpart (GET_MODE (orig_dst), dst);
      emit_move_insn (orig_dst, dst);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (src, 0, 0), 0))
    start = 0;
  else
    start = 1;
  finish = XVECLEN (src, 0);

  tmps = alloca (sizeof (rtx) * finish);

  /* Copy the (probable) hard regs into pseudos.  */
  for (i = start; i < finish; i++)
    {
      rtx reg = XEXP (XVECEXP (src, 0, i), 0);
      if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
        {
          tmps[i] = gen_reg_rtx (GET_MODE (reg));
          emit_move_insn (tmps[i], reg);
        }
      else
        tmps[i] = reg;
    }

  /* If we won't be storing directly into memory, protect the real destination
     from strange tricks we might play.  */
  dst = orig_dst;
  if (GET_CODE (dst) == PARALLEL)
    {
      rtx temp;

      /* We can get a PARALLEL dst if there is a conditional expression in
         a return statement.  In that case, the dst and src are the same,
         so no action is necessary.  */
      if (rtx_equal_p (dst, src))
        return;

      /* It is unclear if we can ever reach here, but we may as well handle
         it.  Allocate a temporary, and split this into a store/load to/from
         the temporary.  */

      temp = assign_stack_temp (GET_MODE (dst), ssize, 0);
      emit_group_store (temp, src, type, ssize);
      emit_group_load (dst, temp, type, ssize);
      return;
    }
  else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
    {
      enum machine_mode outer = GET_MODE (dst);
      enum machine_mode inner;
      HOST_WIDE_INT bytepos;
      bool done = false;
      rtx temp;

      if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
        dst = gen_reg_rtx (outer);

      /* Make life a bit easier for combine.  */
      /* If the first element of the vector is the low part
         of the destination mode, use a paradoxical subreg to
         initialize the destination.  */
      if (start < finish)
        {
          inner = GET_MODE (tmps[start]);
          bytepos = subreg_lowpart_offset (inner, outer);
          if (INTVAL (XEXP (XVECEXP (src, 0, start), 1)) == bytepos)
            {
              temp = simplify_gen_subreg (outer, tmps[start],
                                          inner, 0);
              if (temp)
                {
                  emit_move_insn (dst, temp);
                  done = true;
                  start++;
                }
            }
        }

      /* If the first element wasn't the low part, try the last.  */
      if (!done
          && start < finish - 1)
        {
          inner = GET_MODE (tmps[finish - 1]);
          bytepos = subreg_lowpart_offset (inner, outer);
          if (INTVAL (XEXP (XVECEXP (src, 0, finish - 1), 1)) == bytepos)
            {
              temp = simplify_gen_subreg (outer, tmps[finish - 1],
                                          inner, 0);
              if (temp)
                {
                  emit_move_insn (dst, temp);
                  done = true;
                  finish--;
                }
            }
        }

      /* Otherwise, simply initialize the result to zero.  */
      if (!done)
        emit_move_insn (dst, CONST0_RTX (outer));
    }

  /* Process the pieces.  */
  for (i = start; i < finish; i++)
    {
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (src, 0, i), 1));
      enum machine_mode mode = GET_MODE (tmps[i]);
      unsigned int bytelen = GET_MODE_SIZE (mode);
      rtx dest = dst;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
        {
          /* store_bit_field always takes its value from the lsb.
             Move the fragment to the lsb if it's not already there.  */
          if (
#ifdef BLOCK_REG_PADDING
              BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
              == (BYTES_BIG_ENDIAN ? upward : downward)
#else
              BYTES_BIG_ENDIAN
#endif
              )
            {
              int shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
              tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
                                      build_int_cst (NULL_TREE, shift),
                                      tmps[i], 0);
            }
          bytelen = ssize - bytepos;
        }

      if (GET_CODE (dst) == CONCAT)
        {
          if (bytepos + bytelen <= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
            dest = XEXP (dst, 0);
          else if (bytepos >= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
            {
              bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
              dest = XEXP (dst, 1);
            }
          else
            {
              gcc_assert (bytepos == 0 && XVECLEN (src, 0));
              dest = assign_stack_temp (GET_MODE (dest),
                                        GET_MODE_SIZE (GET_MODE (dest)), 0);
              emit_move_insn (adjust_address (dest, GET_MODE (tmps[i]), bytepos),
                              tmps[i]);
              dst = dest;
              break;
            }
        }

      /* Optimize the access just a bit.  */
      if (MEM_P (dest)
          && (! SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (dest))
              || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
          && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
          && bytelen == GET_MODE_SIZE (mode))
        emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
      else
        store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
                         mode, tmps[i]);
    }

  /* Copy from the pseudo into the (probable) hard reg.  */
  if (orig_dst != dst)
    emit_move_insn (orig_dst, dst);
}

/* Generate code to copy a BLKmode object of TYPE out of a
   set of registers starting with SRCREG into TGTBLK.  If TGTBLK
   is null, a stack temporary is created.  TGTBLK is returned.

   The purpose of this routine is to handle functions that return
   BLKmode structures in registers.  Some machines (the PA for example)
   want to return all small structures in registers regardless of the
   structure's alignment.  */

rtx
copy_blkmode_from_reg (rtx tgtblk, rtx srcreg, tree type)
{
  unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
  rtx src = NULL, dst = NULL;
  unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
  unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;

  if (tgtblk == 0)
    {
      tgtblk = assign_temp (build_qualified_type (type,
                                                  (TYPE_QUALS (type)
                                                   | TYPE_QUAL_CONST)),
                            0, 1, 1);
      preserve_temp_slots (tgtblk);
    }

  /* This code assumes srcreg is at least a full word.  If it isn't, copy it
     into a new pseudo which is a full word.  */

  if (GET_MODE (srcreg) != BLKmode
      && GET_MODE_SIZE (GET_MODE (srcreg)) < UNITS_PER_WORD)
    srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));

  /* If the structure doesn't take up a whole number of words, see whether
     SRCREG is padded on the left or on the right.  If it's on the left,
     set PADDING_CORRECTION to the number of bits to skip.

     In most ABIs, the structure will be returned at the least end of
     the register, which translates to right padding on little-endian
     targets and left padding on big-endian targets.  The opposite
     holds if the structure is returned at the most significant
     end of the register.  */
  if (bytes % UNITS_PER_WORD != 0
      && (targetm.calls.return_in_msb (type)
          ? !BYTES_BIG_ENDIAN
          : BYTES_BIG_ENDIAN))
    padding_correction
      = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));

  /* Copy the structure BITSIZE bites at a time.

     We could probably emit more efficient code for machines which do not use
     strict alignment, but it doesn't seem worth the effort at the current
     time.  */
  for (bitpos = 0, xbitpos = padding_correction;
       bitpos < bytes * BITS_PER_UNIT;
       bitpos += bitsize, xbitpos += bitsize)
    {
      /* We need a new source operand each time xbitpos is on a
         word boundary and when xbitpos == padding_correction
         (the first time through).  */
      if (xbitpos % BITS_PER_WORD == 0
          || xbitpos == padding_correction)
        src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD,
                                     GET_MODE (srcreg));

      /* We need a new destination operand each time bitpos is on
         a word boundary.  */
      if (bitpos % BITS_PER_WORD == 0)
        dst = operand_subword (tgtblk, bitpos / BITS_PER_WORD, 1, BLKmode);

      /* Use xbitpos for the source extraction (right justified) and
         xbitpos for the destination store (left justified).  */
      store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, word_mode,
                       extract_bit_field (src, bitsize,
                                          xbitpos % BITS_PER_WORD, 1,
                                          NULL_RTX, word_mode, word_mode));
    }

  return tgtblk;
}

/* Add a USE expression for REG to the (possibly empty) list pointed
   to by CALL_FUSAGE.  REG must denote a hard register.  */

void
use_reg (rtx *call_fusage, rtx reg)
{
  gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);

  *call_fusage
    = gen_rtx_EXPR_LIST (VOIDmode,
                         gen_rtx_USE (VOIDmode, reg), *call_fusage);
}

/* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
   starting at REGNO.  All of these registers must be hard registers.  */

void
use_regs (rtx *call_fusage, int regno, int nregs)
{
  int i;

  gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);

  for (i = 0; i < nregs; i++)
    use_reg (call_fusage, regno_reg_rtx[regno + i]);
}

/* Add USE expressions to *CALL_FUSAGE for each REG contained in the
   PARALLEL REGS.  This is for calls that pass values in multiple
   non-contiguous locations.  The Irix 6 ABI has examples of this.  */

void
use_group_regs (rtx *call_fusage, rtx regs)
{
  int i;

  for (i = 0; i < XVECLEN (regs, 0); i++)
    {
      rtx reg = XEXP (XVECEXP (regs, 0, i), 0);

      /* A NULL entry means the parameter goes both on the stack and in
         registers.  This can also be a MEM for targets that pass values
         partially on the stack and partially in registers.  */
      if (reg != 0 && REG_P (reg))
        use_reg (call_fusage, reg);
    }
}
\f

/* Determine whether the LEN bytes generated by CONSTFUN can be
   stored to memory using several move instructions.  CONSTFUNDATA is
   a pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.  Return nonzero if a
   call to store_by_pieces should succeed.  */

int
can_store_by_pieces (unsigned HOST_WIDE_INT len,
                     rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode),
                     void *constfundata, unsigned int align)
{
  unsigned HOST_WIDE_INT l;
  unsigned int max_size;
  HOST_WIDE_INT offset = 0;
  enum machine_mode mode, tmode;
  enum insn_code icode;
  int reverse;
  rtx cst;

  if (len == 0)
    return 1;

  if (! STORE_BY_PIECES_P (len, align))
    return 0;

  tmode = mode_for_size (STORE_MAX_PIECES * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      enum machine_mode xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
           tmode != VOIDmode;
           xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) > STORE_MAX_PIECES
            || SLOW_UNALIGNED_ACCESS (tmode, align))
          break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  /* We would first store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  for (reverse = 0;
       reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
       reverse++)
    {
      l = len;
      mode = VOIDmode;
      max_size = STORE_MAX_PIECES + 1;
      while (max_size > 1)
        {
          for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
               tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
            if (GET_MODE_SIZE (tmode) < max_size)
              mode = tmode;

          if (mode == VOIDmode)
            break;

          icode = mov_optab->handlers[(int) mode].insn_code;
          if (icode != CODE_FOR_nothing
              && align >= GET_MODE_ALIGNMENT (mode))
            {
              unsigned int size = GET_MODE_SIZE (mode);

              while (l >= size)
                {
                  if (reverse)
                    offset -= size;

                  cst = (*constfun) (constfundata, offset, mode);
                  if (!LEGITIMATE_CONSTANT_P (cst))
                    return 0;

                  if (!reverse)
                    offset += size;

                  l -= size;
                }
            }

          max_size = GET_MODE_SIZE (mode);
        }

      /* The code above should have handled everything.  */
      gcc_assert (!l);
    }

  return 1;
}

/* Generate several move instructions to store LEN bytes generated by
   CONSTFUN to block TO.  (A MEM rtx with BLKmode).  CONSTFUNDATA is a
   pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.
   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
                 rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode),
                 void *constfundata, unsigned int align, int endp)
{
  struct store_by_pieces data;

  if (len == 0)
    {
      gcc_assert (endp != 2);
      return to;
    }

  gcc_assert (STORE_BY_PIECES_P (len, align));
  data.constfun = constfun;
  data.constfundata = constfundata;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
        {
          if (endp == 2)
            {
              if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
                emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
              else
                data.to_addr = copy_addr_to_reg (plus_constant (data.to_addr,
                                                                -1));
            }
          to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
                                           data.offset);
        }
      else
        {
          if (endp == 2)
            --data.offset;
          to1 = adjust_address (data.to, QImode, data.offset);
        }
      return to1;
    }
  else
    return data.to;
}

/* Generate several move instructions to clear LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
{
  struct store_by_pieces data;

  if (len == 0)
    return;

  data.constfun = clear_by_pieces_1;
  data.constfundata = NULL;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
}

/* Callback routine for clear_by_pieces.
   Return const0_rtx unconditionally.  */

static rtx
clear_by_pieces_1 (void *data ATTRIBUTE_UNUSED,
                   HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
                   enum machine_mode mode ATTRIBUTE_UNUSED)
{
  return const0_rtx;
}

/* Subroutine of clear_by_pieces and store_by_pieces.
   Generate several move instructions to store LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
store_by_pieces_1 (struct store_by_pieces *data ATTRIBUTE_UNUSED,
                   unsigned int align ATTRIBUTE_UNUSED)
{
  rtx to_addr = XEXP (data->to, 0);
  unsigned int max_size = STORE_MAX_PIECES + 1;
  enum machine_mode mode = VOIDmode, tmode;
  enum insn_code icode;

  data->offset = 0;
  data->to_addr = to_addr;
  data->autinc_to
    = (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
       || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);

  data->explicit_inc_to = 0;
  data->reverse
    = (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
  if (data->reverse)
    data->offset = data->len;

  /* If storing requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!data->autinc_to
      && move_by_pieces_ninsns (data->len, align, max_size) > 2)
    {
      /* Determine the main mode we'll be using.  */
      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) < max_size)
          mode = tmode;

      if (USE_STORE_PRE_DECREMENT (mode) && data->reverse && ! data->autinc_to)
        {
          data->to_addr = copy_addr_to_reg (plus_constant (to_addr, data->len));
          data->autinc_to = 1;
          data->explicit_inc_to = -1;
        }

      if (USE_STORE_POST_INCREMENT (mode) && ! data->reverse
          && ! data->autinc_to)
        {
          data->to_addr = copy_addr_to_reg (to_addr);
          data->autinc_to = 1;
          data->explicit_inc_to = 1;
        }

      if ( !data->autinc_to && CONSTANT_P (to_addr))
        data->to_addr = copy_addr_to_reg (to_addr);
    }

  tmode = mode_for_size (STORE_MAX_PIECES * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      enum machine_mode xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
           tmode != VOIDmode;
           xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) > STORE_MAX_PIECES
            || SLOW_UNALIGNED_ACCESS (tmode, align))
          break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  /* First store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1)
    {
      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) < max_size)
          mode = tmode;

      if (mode == VOIDmode)
        break;

      icode = mov_optab->handlers[(int) mode].insn_code;
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        store_by_pieces_2 (GEN_FCN (icode), mode, data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data->len);
}

/* Subroutine of store_by_pieces_1.  Store as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
store_by_pieces_2 (rtx (*genfun) (rtx, ...), enum machine_mode mode,
                   struct store_by_pieces *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1, cst;

  while (data->len >= size)
    {
      if (data->reverse)
        data->offset -= size;

      if (data->autinc_to)
        to1 = adjust_automodify_address (data->to, mode, data->to_addr,
                                         data->offset);
      else
        to1 = adjust_address (data->to, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
        emit_insn (gen_add2_insn (data->to_addr,
                                  GEN_INT (-(HOST_WIDE_INT) size)));

      cst = (*data->constfun) (data->constfundata, data->offset, mode);
      emit_insn ((*genfun) (to1, cst));

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
        emit_insn (gen_add2_insn (data->to_addr, GEN_INT (size)));

      if (! data->reverse)
        data->offset += size;

      data->len -= size;
    }
}
\f
/* Write zeros through the storage of OBJECT.  If OBJECT has BLKmode, SIZE is
   its length in bytes.  */

rtx
clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
                     unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  enum machine_mode mode = GET_MODE (object);
  unsigned int align;

  gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);

  /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
     just move a zero.  Otherwise, do this a piece at a time.  */
  if (mode != BLKmode
      && GET_CODE (size) == CONST_INT
      && INTVAL (size) == (HOST_WIDE_INT) GET_MODE_SIZE (mode))
    {
      rtx zero = CONST0_RTX (mode);
      if (zero != NULL)
        {
          emit_move_insn (object, zero);
          return NULL;
        }

      if (COMPLEX_MODE_P (mode))
        {
          zero = CONST0_RTX (GET_MODE_INNER (mode));
          if (zero != NULL)
            {
              write_complex_part (object, zero, 0);
              write_complex_part (object, zero, 1);
              return NULL;
            }
        }
    }

  if (size == const0_rtx)
    return NULL;

  align = MEM_ALIGN (object);

  if (GET_CODE (size) == CONST_INT
      && CLEAR_BY_PIECES_P (INTVAL (size), align))
    clear_by_pieces (object, INTVAL (size), align);
  else if (set_storage_via_setmem (object, size, const0_rtx, align,
                                   expected_align, expected_size))
    ;
  else
    return set_storage_via_libcall (object, size, const0_rtx,
                                    method == BLOCK_OP_TAILCALL);

  return NULL;
}

rtx
clear_storage (rtx object, rtx size, enum block_op_methods method)
{
  return clear_storage_hints (object, size, method, 0, -1);
}


/* A subroutine of clear_storage.  Expand a call to memset.
   Return the return value of memset, 0 otherwise.  */

rtx
set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
{
  tree call_expr, fn, object_tree, size_tree, val_tree;
  enum machine_mode size_mode;
  rtx retval;

  /* Emit code to copy OBJECT and SIZE into new pseudos.  We can then
     place those into new pseudos into a VAR_DECL and use them later.  */

  object = copy_to_mode_reg (Pmode, XEXP (object, 0));

  size_mode = TYPE_MODE (sizetype);
  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memset in this context.  This could be a user call to memset and
     the user may wish to examine the return value from memset.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  object_tree = make_tree (ptr_type_node, object);
  if (GET_CODE (val) != CONST_INT)
    val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
  size_tree = make_tree (sizetype, size);
  val_tree = make_tree (integer_type_node, val);

  fn = clear_storage_libcall_fn (true);
  call_expr = build_call_expr (fn, 3,
                               object_tree, integer_zero_node, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of set_storage_via_libcall.  Create the tree node
   for the function we use for block clears.  The first time FOR_CALL
   is true, we call assemble_external.  */

static GTY(()) tree block_clear_fn;

void
init_block_clear_fn (const char *asmspec)
{
  if (!block_clear_fn)
    {
      tree fn, args;

      fn = get_identifier ("memset");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
                                       integer_type_node, sizetype,
                                       NULL_TREE);

      fn = build_decl (FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      block_clear_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_clear_fn, asmspec);
}

static tree
clear_storage_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_clear_fn)
    init_block_clear_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_clear_fn);
      assemble_external (block_clear_fn);
    }

  return block_clear_fn;
}
\f
/* Expand a setmem pattern; return true if successful.  */

bool
set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
                        unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  rtx opalign = GEN_INT (align / BITS_PER_UNIT);
  enum machine_mode mode;

  if (expected_align < align)
    expected_align = align;

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = setmem_optab[(int) mode];
      insn_operand_predicate_fn pred;

      if (code != CODE_FOR_nothing
          /* We don't need MODE to be narrower than
             BITS_PER_HOST_WIDE_INT here because if SIZE is less than
             the mode mask, as it is returned by the macro, it will
             definitely be less than the actual mode mask.  */
          && ((GET_CODE (size) == CONST_INT
               && ((unsigned HOST_WIDE_INT) INTVAL (size)
                   <= (GET_MODE_MASK (mode) >> 1)))
              || GET_MODE_BITSIZE (mode) >= BITS_PER_WORD)
          && ((pred = insn_data[(int) code].operand[0].predicate) == 0
              || (*pred) (object, BLKmode))
          && ((pred = insn_data[(int) code].operand[3].predicate) == 0
              || (*pred) (opalign, VOIDmode)))
        {
          rtx opsize, opchar;
          enum machine_mode char_mode;
          rtx last = get_last_insn ();
          rtx pat;

          opsize = convert_to_mode (mode, size, 1);
          pred = insn_data[(int) code].operand[1].predicate;
          if (pred != 0 && ! (*pred) (opsize, mode))
            opsize = copy_to_mode_reg (mode, opsize);

          opchar = val;
          char_mode = insn_data[(int) code].operand[2].mode;
          if (char_mode != VOIDmode)
            {
              opchar = convert_to_mode (char_mode, opchar, 1);
              pred = insn_data[(int) code].operand[2].predicate;
              if (pred != 0 && ! (*pred) (opchar, char_mode))
                opchar = copy_to_mode_reg (char_mode, opchar);
            }

          if (insn_data[(int) code].n_operands == 4)
            pat = GEN_FCN ((int) code) (object, opsize, opchar, opalign);
          else
            pat = GEN_FCN ((int) code) (object, opsize, opchar, opalign,
                                        GEN_INT (expected_align),
                                        GEN_INT (expected_size));
          if (pat)
            {
              emit_insn (pat);
              return true;
            }
          else
            delete_insns_since (last);
        }
    }

  return false;
}

\f
/* Write to one of the components of the complex value CPLX.  Write VAL to
   the real part if IMAG_P is false, and the imaginary part if its true.  */

static void
write_complex_part (rtx cplx, rtx val, bool imag_p)
{
  enum machine_mode cmode;
  enum machine_mode imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    {
      emit_move_insn (XEXP (cplx, imag_p), val);
      return;
    }

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    {
      emit_move_insn (adjust_address_nv (cplx, imode,
                                         imag_p ? GET_MODE_SIZE (imode) : 0),
                      val);
      return;
    }

  /* If the sub-object is at least word sized, then we know that subregging
     will work.  This special case is important, since store_bit_field
     wants to operate on integer modes, and there's rarely an OImode to
     correspond to TCmode.  */
  if (ibitsize >= BITS_PER_WORD
      /* For hard regs we have exact predicates.  Assume we can split
         the original object if it spans an even number of hard regs.
         This special case is important for SCmode on 64-bit platforms
         where the natural size of floating-point regs is 32-bit.  */
      || (REG_P (cplx)
          && REGNO (cplx) < FIRST_PSEUDO_REGISTER
          && hard_regno_nregs[REGNO (cplx)][cmode] % 2 == 0))
    {
      rtx part = simplify_gen_subreg (imode, cplx, cmode,
                                      imag_p ? GET_MODE_SIZE (imode) : 0);
      if (part)
        {
          emit_move_insn (part, val);
          return;
        }
      else
        /* simplify_gen_subreg may fail for sub-word MEMs.  */
        gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
    }

  store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, imode, val);
}

/* Extract one of the components of the complex value CPLX.  Extract the
   real part if IMAG_P is false, and the imaginary part if it's true.  */

static rtx
read_complex_part (rtx cplx, bool imag_p)
{
  enum machine_mode cmode, imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    return XEXP (cplx, imag_p);

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* Special case reads from complex constants that got spilled to memory.  */
  if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
    {
      tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
      if (decl && TREE_CODE (decl) == COMPLEX_CST)
        {
          tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
          if (CONSTANT_CLASS_P (part))
            return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
        }
    }

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    return adjust_address_nv (cplx, imode,
                              imag_p ? GET_MODE_SIZE (imode) : 0);

  /* If the sub-object is at least word sized, then we know that subregging
     will work.  This special case is important, since extract_bit_field
     wants to operate on integer modes, and there's rarely an OImode to
     correspond to TCmode.  */
  if (ibitsize >= BITS_PER_WORD
      /* For hard regs we have exact predicates.  Assume we can split
         the original object if it spans an even number of hard regs.
         This special case is important for SCmode on 64-bit platforms
         where the natural size of floating-point regs is 32-bit.  */
      || (REG_P (cplx)
          && REGNO (cplx) < FIRST_PSEUDO_REGISTER
          && hard_regno_nregs[REGNO (cplx)][cmode] % 2 == 0))
    {
      rtx ret = simplify_gen_subreg (imode, cplx, cmode,
                                     imag_p ? GET_MODE_SIZE (imode) : 0);
      if (ret)
        return ret;
      else
        /* simplify_gen_subreg may fail for sub-word MEMs.  */
        gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
    }

  return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
                            true, NULL_RTX, imode, imode);
}
\f
/* A subroutine of emit_move_insn_1.  Yet another lowpart generator.
   NEW_MODE and OLD_MODE are the same size.  Return NULL if X cannot be
   represented in NEW_MODE.  If FORCE is true, this will never happen, as
   we'll force-create a SUBREG if needed.  */

static rtx
emit_move_change_mode (enum machine_mode new_mode,
                       enum machine_mode old_mode, rtx x, bool force)
{
  rtx ret;

  if (push_operand (x, GET_MODE (x)))
    {
      ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
      MEM_COPY_ATTRIBUTES (ret, x);
    }
  else if (MEM_P (x))
    {
      /* We don't have to worry about changing the address since the
         size in bytes is supposed to be the same.  */
      if (reload_in_progress)
        {
          /* Copy the MEM to change the mode and move any
             substitutions from the old MEM to the new one.  */
          ret = adjust_address_nv (x, new_mode, 0);
          copy_replacements (x, ret);
        }
      else
        ret = adjust_address (x, new_mode, 0);
    }
  else
    {
      /* Note that we do want simplify_subreg's behavior of validating
         that the new mode is ok for a hard register.  If we were to use
         simplify_gen_subreg, we would create the subreg, but would
         probably run into the target not being able to implement it.  */
      /* Except, of course, when FORCE is true, when this is exactly what
         we want.  Which is needed for CCmodes on some targets.  */
      if (force)
        ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
      else
        ret = simplify_subreg (new_mode, x, old_mode, 0);
    }

  return ret;
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X using
   an integer mode of the same size as MODE.  Returns the instruction
   emitted, or NULL if such a move could not be generated.  */

static rtx
emit_move_via_integer (enum machine_mode mode, rtx x, rtx y, bool force)
{
  enum machine_mode imode;
  enum insn_code code;

  /* There must exist a mode of the exact size we require.  */
  imode = int_mode_for_mode (mode);
  if (imode == BLKmode)
    return NULL_RTX;

  /* The target must support moves in this mode.  */
  code = mov_optab->handlers[imode].insn_code;
  if (code == CODE_FOR_nothing)
    return NULL_RTX;

  x = emit_move_change_mode (imode, mode, x, force);
  if (x == NULL_RTX)
    return NULL_RTX;
  y = emit_move_change_mode (imode, mode, y, force);
  if (y == NULL_RTX)
    return NULL_RTX;
  return emit_insn (GEN_FCN (code) (x, y));
}

/* A subroutine of emit_move_insn_1.  X is a push_operand in MODE.
   Return an equivalent MEM that does not use an auto-increment.  */

static rtx
emit_move_resolve_push (enum machine_mode mode, rtx x)
{
  enum rtx_code code = GET_CODE (XEXP (x, 0));
  HOST_WIDE_INT adjust;
  rtx temp;

  adjust = GET_MODE_SIZE (mode);
#ifdef PUSH_ROUNDING
  adjust = PUSH_ROUNDING (adjust);
#endif
  if (code == PRE_DEC || code == POST_DEC)
    adjust = -adjust;
  else if (code == PRE_MODIFY || code == POST_MODIFY)
    {
      rtx expr = XEXP (XEXP (x, 0), 1);
      HOST_WIDE_INT val;

      gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
      gcc_assert (GET_CODE (XEXP (expr, 1)) == CONST_INT);
      val = INTVAL (XEXP (expr, 1));
      if (GET_CODE (expr) == MINUS)
        val = -val;
      gcc_assert (adjust == val || adjust == -val);
      adjust = val;
    }

  /* Do not use anti_adjust_stack, since we don't want to update
     stack_pointer_delta.  */
  temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
                              GEN_INT (adjust), stack_pointer_rtx,
                              0, OPTAB_LIB_WIDEN);
  if (temp != stack_pointer_rtx)
    emit_move_insn (stack_pointer_rtx, temp);

  switch (code)
    {
    case PRE_INC:
    case PRE_DEC:
    case PRE_MODIFY:
      temp = stack_pointer_rtx;
      break;
    case POST_INC:
    case POST_DEC:
    case POST_MODIFY:
      temp = plus_constant (stack_pointer_rtx, -adjust);
      break;
    default:
      gcc_unreachable ();
    }

  return replace_equiv_address (x, temp);
}

/* A subroutine of emit_move_complex.  Generate a move from Y into X.
   X is known to satisfy push_operand, and MODE is known to be complex.
   Returns the last instruction emitted.  */

rtx
emit_move_complex_push (enum machine_mode mode, rtx x, rtx y)
{
  enum machine_mode submode = GET_MODE_INNER (mode);
  bool imag_first;

#ifdef PUSH_ROUNDING
  unsigned int submodesize = GET_MODE_SIZE (submode);

  /* In case we output to the stack, but the size is smaller than the
     machine can push exactly, we need to use move instructions.  */
  if (PUSH_ROUNDING (submodesize) != submodesize)
    {
      x = emit_move_resolve_push (mode, x);
      return emit_move_insn (x, y);
    }
#endif

  /* Note that the real part always precedes the imag part in memory
     regardless of machine's endianness.  */
  switch (GET_CODE (XEXP (x, 0)))
    {
    case PRE_DEC:
    case POST_DEC:
      imag_first = true;
      break;
    case PRE_INC:
    case POST_INC:
      imag_first = false;
      break;
    default:
      gcc_unreachable ();
    }

  emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
                  read_complex_part (y, imag_first));
  return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
                         read_complex_part (y, !imag_first));
}

/* A subroutine of emit_move_complex.  Perform the move from Y to X
   via two moves of the parts.  Returns the last instruction emitted.  */

rtx
emit_move_complex_parts (rtx x, rtx y)
{
  /* Show the output dies here.  This is necessary for SUBREGs
     of pseudos since we cannot track their lifetimes correctly;
     hard regs shouldn't appear here except as return values.  */
  if (!reload_completed && !reload_in_progress
      && REG_P (x) && !reg_overlap_mentioned_p (x, y))
    emit_insn (gen_rtx_CLOBBER (VOIDmode, x));

  write_complex_part (x, read_complex_part (y, false), false);
  write_complex_part (x, read_complex_part (y, true), true);

  return get_last_insn ();
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is known to be complex.  Returns the last instruction emitted.  */

static rtx
emit_move_complex (enum machine_mode mode, rtx x, rtx y)
{
  bool try_int;

  /* Need to take special care for pushes, to maintain proper ordering
     of the data, and possibly extra padding.  */
  if (push_operand (x, mode))
    return emit_move_complex_push (mode, x, y);

  /* See if we can coerce the target into moving both values at once.  */

  /* Move floating point as parts.  */
  if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
      && mov_optab->handlers[GET_MODE_INNER (mode)].insn_code != CODE_FOR_nothing)
    try_int = false;
  /* Not possible if the values are inherently not adjacent.  */
  else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
    try_int = false;
  /* Is possible if both are registers (or subregs of registers).  */
  else if (register_operand (x, mode) && register_operand (y, mode))
    try_int = true;
  /* If one of the operands is a memory, and alignment constraints
     are friendly enough, we may be able to do combined memory operations.
     We do not attempt this if Y is a constant because that combination is
     usually better with the by-parts thing below.  */
  else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
           && (!STRICT_ALIGNMENT
               || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
    try_int = true;
  else
    try_int = false;

  if (try_int)
    {
      rtx ret;

      /* For memory to memory moves, optimal behavior can be had with the
         existing block move logic.  */
      if (MEM_P (x) && MEM_P (y))
        {
          emit_block_move (x, y, GEN_INT (GET_MODE_SIZE (mode)),
                           BLOCK_OP_NO_LIBCALL);
          return get_last_insn ();
        }

      ret = emit_move_via_integer (mode, x, y, true);
      if (ret)
        return ret;
    }

  return emit_move_complex_parts (x, y);
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is known to be MODE_CC.  Returns the last instruction emitted.  */

static rtx
emit_move_ccmode (enum machine_mode mode, rtx x, rtx y)
{
  rtx ret;

  /* Assume all MODE_CC modes are equivalent; if we have movcc, use it.  */
  if (mode != CCmode)
    {
      enum insn_code code = mov_optab->handlers[CCmode].insn_code;
      if (code != CODE_FOR_nothing)
        {
          x = emit_move_change_mode (CCmode, mode, x, true);
          y = emit_move_change_mode (CCmode, mode, y, true);
          return emit_insn (GEN_FCN (code) (x, y));
        }
    }

  /* Otherwise, find the MODE_INT mode of the same width.  */
  ret = emit_move_via_integer (mode, x, y, false);
  gcc_assert (ret != NULL);
  return ret;
}

/* Return true if word I of OP lies entirely in the
   undefined bits of a paradoxical subreg.  */

static bool
undefined_operand_subword_p (rtx op, int i)
{
  enum machine_mode innermode, innermostmode;
  int offset;
  if (GET_CODE (op) != SUBREG)
    return false;
  innermode = GET_MODE (op);
  innermostmode = GET_MODE (SUBREG_REG (op));
  offset = i * UNITS_PER_WORD + SUBREG_BYTE (op);
  /* The SUBREG_BYTE represents offset, as if the value were stored in
     memory, except for a paradoxical subreg where we define
     SUBREG_BYTE to be 0; undo this exception as in
     simplify_subreg.  */
  if (SUBREG_BYTE (op) == 0
      && GET_MODE_SIZE (innermostmode) < GET_MODE_SIZE (innermode))
    {
      int difference = (GET_MODE_SIZE (innermostmode) - GET_MODE_SIZE (innermode));
      if (WORDS_BIG_ENDIAN)
        offset += (difference / UNITS_PER_WORD) * UNITS_PER_WORD;
      if (BYTES_BIG_ENDIAN)
        offset += difference % UNITS_PER_WORD;
    }
  if (offset >= GET_MODE_SIZE (innermostmode)
      || offset <= -GET_MODE_SIZE (word_mode))
    return true;
  return false;
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is any multi-word or full-word mode that lacks a move_insn
   pattern.  Note that you will get better code if you define such
   patterns, even if they must turn into multiple assembler instructions.  */

static rtx
emit_move_multi_word (enum machine_mode mode, rtx x, rtx y)
{
  rtx last_insn = 0;
  rtx seq, inner;
  bool need_clobber;
  int i;

  gcc_assert (GET_MODE_SIZE (mode) >= UNITS_PER_WORD);

  /* If X is a push on the stack, do the push now and replace
     X with a reference to the stack pointer.  */
  if (push_operand (x, mode))
    x = emit_move_resolve_push (mode, x);

  /* If we are in reload, see if either operand is a MEM whose address
     is scheduled for replacement.  */
  if (reload_in_progress && MEM_P (x)
      && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
    x = replace_equiv_address_nv (x, inner);
  if (reload_in_progress && MEM_P (y)
      && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
    y = replace_equiv_address_nv (y, inner);

  start_sequence ();

  need_clobber = false;
  for (i = 0;
       i < (GET_MODE_SIZE (mode) + (UNITS_PER_WORD - 1)) / UNITS_PER_WORD;
       i++)
    {
      rtx xpart = operand_subword (x, i, 1, mode);
      rtx ypart;

      /* Do not generate code for a move if it would come entirely
         from the undefined bits of a paradoxical subreg.  */
      if (undefined_operand_subword_p (y, i))
        continue;

      ypart = operand_subword (y, i, 1, mode);

      /* If we can't get a part of Y, put Y into memory if it is a
         constant.  Otherwise, force it into a register.  Then we must
         be able to get a part of Y.  */
      if (ypart == 0 && CONSTANT_P (y))
        {
          y = use_anchored_address (force_const_mem (mode, y));
          ypart = operand_subword (y, i, 1, mode);
        }
      else if (ypart == 0)
        ypart = operand_subword_force (y, i, mode);

      gcc_assert (xpart && ypart);

      need_clobber |= (GET_CODE (xpart) == SUBREG);

      last_insn = emit_move_insn (xpart, ypart);
    }

  seq = get_insns ();
  end_sequence ();

  /* Show the output dies here.  This is necessary for SUBREGs
     of pseudos since we cannot track their lifetimes correctly;
     hard regs shouldn't appear here except as return values.
     We never want to emit such a clobber after reload.  */
  if (x != y
      && ! (reload_in_progress || reload_completed)
      && need_clobber != 0)
    emit_insn (gen_rtx_CLOBBER (VOIDmode, x));

  emit_insn (seq);

  return last_insn;
}

/* Low level part of emit_move_insn.
   Called just like emit_move_insn, but assumes X and Y
   are basically valid.  */

rtx
emit_move_insn_1 (rtx x, rtx y)
{
  enum machine_mode mode = GET_MODE (x);
  enum insn_code code;

  gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);

  code = mov_optab->handlers[mode].insn_code;
  if (code != CODE_FOR_nothing)
    return emit_insn (GEN_FCN (code) (x, y));

  /* Expand complex moves by moving real part and imag part.  */
  if (COMPLEX_MODE_P (mode))
    return emit_move_complex (mode, x, y);

  if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT)
    {
      rtx result = emit_move_via_integer (mode, x, y, true);

      /* If we can't find an integer mode, use multi words.  */
      if (result)
        return result;
      else
        return emit_move_multi_word (mode, x, y);
    }

  if (GET_MODE_CLASS (mode) == MODE_CC)
    return emit_move_ccmode (mode, x, y);

  /* Try using a move pattern for the corresponding integer mode.  This is
     only safe when simplify_subreg can convert MODE constants into integer
     constants.  At present, it can only do this reliably if the value
     fits within a HOST_WIDE_INT.  */
  if (!CONSTANT_P (y) || GET_MODE_BITSIZE (mode) <= HOST_BITS_PER_WIDE_INT)
    {
      rtx ret = emit_move_via_integer (mode, x, y, false);
      if (ret)
        return ret;
    }

  return emit_move_multi_word (mode, x, y);
}

/* Generate code to copy Y into X.
   Both Y and X must have the same mode, except that
   Y can be a constant with VOIDmode.
   This mode cannot be BLKmode; use emit_block_move for that.

   Return the last instruction emitted.  */

rtx
emit_move_insn (rtx x, rtx y)
{
  enum machine_mode mode = GET_MODE (x);
  rtx y_cst = NULL_RTX;
  rtx last_insn, set;

  gcc_assert (mode != BLKmode
              && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));

  if (CONSTANT_P (y))
    {
      if (optimize
          && SCALAR_FLOAT_MODE_P (GET_MODE (x))
          && (last_insn = compress_float_constant (x, y)))
        return last_insn;

      y_cst = y;

      if (!LEGITIMATE_CONSTANT_P (y))
        {
          y = force_const_mem (mode, y);

          /* If the target's cannot_force_const_mem prevented the spill,
             assume that the target's move expanders will also take care
             of the non-legitimate constant.  */
          if (!y)
            y = y_cst;
          else
            y = use_anchored_address (y);
        }
    }

  /* If X or Y are memory references, verify that their addresses are valid
     for the machine.  */
  if (MEM_P (x)
      && ((! memory_address_p (GET_MODE (x), XEXP (x, 0))
           && ! push_operand (x, GET_MODE (x)))
          || (flag_force_addr
              && CONSTANT_ADDRESS_P (XEXP (x, 0)))))
    x = validize_mem (x);

  if (MEM_P (y)
      && (! memory_address_p (GET_MODE (y), XEXP (y, 0))
          || (flag_force_addr
              && CONSTANT_ADDRESS_P (XEXP (y, 0)))))
    y = validize_mem (y);

  gcc_assert (mode != BLKmode);

  last_insn = emit_move_insn_1 (x, y);

  if (y_cst && REG_P (x)
      && (set = single_set (last_insn)) != NULL_RTX
      && SET_DEST (set) == x
      && ! rtx_equal_p (y_cst, SET_SRC (set)))
    set_unique_reg_note (last_insn, REG_EQUAL, y_cst);

  return last_insn;
}

/* If Y is representable exactly in a narrower mode, and the target can
   perform the extension directly from constant or memory, then emit the
   move as an extension.  */

static rtx
compress_float_constant (rtx x, rtx y)
{
  enum machine_mode dstmode = GET_MODE (x);
  enum machine_mode orig_srcmode = GET_MODE (y);
  enum machine_mode srcmode;
  REAL_VALUE_TYPE r;
  int oldcost, newcost;

  REAL_VALUE_FROM_CONST_DOUBLE (r, y);

  if (LEGITIMATE_CONSTANT_P (y))
    oldcost = rtx_cost (y, SET);
  else
    oldcost = rtx_cost (force_const_mem (dstmode, y), SET);

  for (srcmode = GET_CLASS_NARROWEST_MODE (GET_MODE_CLASS (orig_srcmode));
       srcmode != orig_srcmode;
       srcmode = GET_MODE_WIDER_MODE (srcmode))
    {
      enum insn_code ic;
      rtx trunc_y, last_insn;

      /* Skip if the target can't extend this way.  */
      ic = can_extend_p (dstmode, srcmode, 0);
      if (ic == CODE_FOR_nothing)
        continue;

      /* Skip if the narrowed value isn't exact.  */
      if (! exact_real_truncate (srcmode, &r))
        continue;

      trunc_y = CONST_DOUBLE_FROM_REAL_VALUE (r, srcmode);

      if (LEGITIMATE_CONSTANT_P (trunc_y))
        {
          /* Skip if the target needs extra instructions to perform
             the extension.  */
          if (! (*insn_data[ic].operand[1].predicate) (trunc_y, srcmode))
            continue;
          /* This is valid, but may not be cheaper than the original. */
          newcost = rtx_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y), SET);
          if (oldcost < newcost)
            continue;
        }
      else if (float_extend_from_mem[dstmode][srcmode])
        {
          trunc_y = force_const_mem (srcmode, trunc_y);
          /* This is valid, but may not be cheaper than the original. */
          newcost = rtx_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y), SET);
          if (oldcost < newcost)
            continue;
          trunc_y = validize_mem (trunc_y);
        }
      else
        continue;

      /* For CSE's benefit, force the compressed constant pool entry
         into a new pseudo.  This constant may be used in different modes,
         and if not, combine will put things back together for us.  */
      trunc_y = force_reg (srcmode, trunc_y);
      emit_unop_insn (ic, x, trunc_y, UNKNOWN);
      last_insn = get_last_insn ();

      if (REG_P (x))
        set_unique_reg_note (last_insn, REG_EQUAL, y);

      return last_insn;
    }

  return NULL_RTX;
}
\f
/* Pushing data onto the stack.  */

/* Push a block of length SIZE (perhaps variable)
   and return an rtx to address the beginning of the block.
   The value may be virtual_outgoing_args_rtx.

   EXTRA is the number of bytes of padding to push in addition to SIZE.
   BELOW nonzero means this padding comes at low addresses;
   otherwise, the padding comes at high addresses.  */

rtx
push_block (rtx size, int extra, int below)
{
  rtx temp;

  size = convert_modes (Pmode, ptr_mode, size, 1);
  if (CONSTANT_P (size))
    anti_adjust_stack (plus_constant (size, extra));
  else if (REG_P (size) && extra == 0)
    anti_adjust_stack (size);
  else
    {
      temp = copy_to_mode_reg (Pmode, size);
      if (extra != 0)
        temp = expand_binop (Pmode, add_optab, temp, GEN_INT (extra),
                             temp, 0, OPTAB_LIB_WIDEN);
      anti_adjust_stack (temp);
    }

#ifndef STACK_GROWS_DOWNWARD
  if (0)
#else
  if (1)
#endif
    {
      temp = virtual_outgoing_args_rtx;
      if (extra != 0 && below)
        temp = plus_constant (temp, extra);
    }
  else
    {
      if (GET_CODE (size) == CONST_INT)
        temp = plus_constant (virtual_outgoing_args_rtx,
                              -INTVAL (size) - (below ? 0 : extra));
      else if (extra != 0 && !below)
        temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
                             negate_rtx (Pmode, plus_constant (size, extra)));
      else
        temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
                             negate_rtx (Pmode, size));
    }

  return memory_address (GET_CLASS_NARROWEST_MODE (MODE_INT), temp);
}

#ifdef PUSH_ROUNDING

/* Emit single push insn.  */

static void
emit_single_push_insn (enum machine_mode mode, rtx x, tree type)
{
  rtx dest_addr;
  unsigned rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
  rtx dest;
  enum insn_code icode;
  insn_operand_predicate_fn pred;

  stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
  /* If there is push pattern, use it.  Otherwise try old way of throwing
     MEM representing push operation to move expander.  */
  icode = push_optab->handlers[(int) mode].insn_code;
  if (icode != CODE_FOR_nothing)
    {
      if (((pred = insn_data[(int) icode].operand[0].predicate)
           && !((*pred) (x, mode))))
        x = force_reg (mode, x);
      emit_insn (GEN_FCN (icode) (x));
      return;
    }
  if (GET_MODE_SIZE (mode) == rounded_size)
    dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
  /* If we are to pad downward, adjust the stack pointer first and
     then store X into the stack location using an offset.  This is
     because emit_move_insn does not know how to pad; it does not have
     access to type.  */
  else if (FUNCTION_ARG_PADDING (mode, type) == downward)
    {
      unsigned padding_size = rounded_size - GET_MODE_SIZE (mode);
      HOST_WIDE_INT offset;

      emit_move_insn (stack_pointer_rtx,
                      expand_binop (Pmode,
#ifdef STACK_GROWS_DOWNWARD
                                    sub_optab,
#else
                                    add_optab,
#endif
                                    stack_pointer_rtx,
                                    GEN_INT (rounded_size),
                                    NULL_RTX, 0, OPTAB_LIB_WIDEN));

      offset = (HOST_WIDE_INT) padding_size;
#ifdef STACK_GROWS_DOWNWARD
      if (STACK_PUSH_CODE == POST_DEC)
        /* We have already decremented the stack pointer, so get the
           previous value.  */
        offset += (HOST_WIDE_INT) rounded_size;
#else
      if (STACK_PUSH_CODE == POST_INC)
        /* We have already incremented the stack pointer, so get the
           previous value.  */
        offset -= (HOST_WIDE_INT) rounded_size;
#endif
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (offset));
    }
  else
    {
#ifdef STACK_GROWS_DOWNWARD
      /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC.  */
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
                                GEN_INT (-(HOST_WIDE_INT) rounded_size));
#else
      /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC.  */
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
                                GEN_INT (rounded_size));
#endif
      dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
    }

  dest = gen_rtx_MEM (mode, dest_addr);

  if (type != 0)
    {
      set_mem_attributes (dest, type, 1);

      if (flag_optimize_sibling_calls)
        /* Function incoming arguments may overlap with sibling call
           outgoing arguments and we cannot allow reordering of reads
           from function arguments with stores to outgoing arguments
           of sibling calls.  */
        set_mem_alias_set (dest, 0);
    }
  emit_move_insn (dest, x);
}
#endif

/* Generate code to push X onto the stack, assuming it has mode MODE and
   type TYPE.
   MODE is redundant except when