[pf3gnuchains/gcc-fork.git] / gcc / expr.c

/* Convert tree expression to rtl instructions, for GNU compiler.
   Copyright (C) 1988, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
   2012 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "machmode.h"
#include "rtl.h"
#include "tree.h"
#include "flags.h"
#include "regs.h"
#include "hard-reg-set.h"
#include "except.h"
#include "function.h"
#include "insn-config.h"
#include "insn-attr.h"
/* Include expr.h after insn-config.h so we get HAVE_conditional_move.  */
#include "expr.h"
#include "optabs.h"
#include "libfuncs.h"
#include "recog.h"
#include "reload.h"
#include "output.h"
#include "typeclass.h"
#include "toplev.h"
#include "langhooks.h"
#include "intl.h"
#include "tm_p.h"
#include "tree-iterator.h"
#include "tree-pass.h"
#include "tree-flow.h"
#include "target.h"
#include "common/common-target.h"
#include "timevar.h"
#include "df.h"
#include "diagnostic.h"
#include "ssaexpand.h"
#include "target-globals.h"
#include "params.h"

/* Decide whether a function's arguments should be processed
   from first to last or from last to first.

   They should if the stack and args grow in opposite directions, but
   only if we have push insns.  */

#ifdef PUSH_ROUNDING

#ifndef PUSH_ARGS_REVERSED
#if defined (STACK_GROWS_DOWNWARD) != defined (ARGS_GROW_DOWNWARD)
#define PUSH_ARGS_REVERSED      /* If it's last to first.  */
#endif
#endif

#endif

#ifndef STACK_PUSH_CODE
#ifdef STACK_GROWS_DOWNWARD
#define STACK_PUSH_CODE PRE_DEC
#else
#define STACK_PUSH_CODE PRE_INC
#endif
#endif


/* If this is nonzero, we do not bother generating VOLATILE
   around volatile memory references, and we are willing to
   output indirect addresses.  If cse is to follow, we reject
   indirect addresses so a useful potential cse is generated;
   if it is used only once, instruction combination will produce
   the same indirect address eventually.  */
int cse_not_expected;

/* This structure is used by move_by_pieces to describe the move to
   be performed.  */
struct move_by_pieces_d
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  rtx from;
  rtx from_addr;
  int autinc_from;
  int explicit_inc_from;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  int reverse;
};

/* This structure is used by store_by_pieces to describe the clear to
   be performed.  */

struct store_by_pieces_d
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode);
  void *constfundata;
  int reverse;
};

static void move_by_pieces_1 (rtx (*) (rtx, ...), enum machine_mode,
                              struct move_by_pieces_d *);
static bool block_move_libcall_safe_for_call_parm (void);
static bool emit_block_move_via_movmem (rtx, rtx, rtx, unsigned, unsigned, HOST_WIDE_INT);
static tree emit_block_move_libcall_fn (int);
static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned);
static rtx clear_by_pieces_1 (void *, HOST_WIDE_INT, enum machine_mode);
static void clear_by_pieces (rtx, unsigned HOST_WIDE_INT, unsigned int);
static void store_by_pieces_1 (struct store_by_pieces_d *, unsigned int);
static void store_by_pieces_2 (rtx (*) (rtx, ...), enum machine_mode,
                               struct store_by_pieces_d *);
static tree clear_storage_libcall_fn (int);
static rtx compress_float_constant (rtx, rtx);
static rtx get_subtarget (rtx);
static void store_constructor_field (rtx, unsigned HOST_WIDE_INT,
                                     HOST_WIDE_INT, enum machine_mode,
                                     tree, tree, int, alias_set_type);
static void store_constructor (tree, rtx, int, HOST_WIDE_INT);
static rtx store_field (rtx, HOST_WIDE_INT, HOST_WIDE_INT,
                        unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
                        enum machine_mode,
                        tree, tree, alias_set_type, bool);

static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);

static int is_aligning_offset (const_tree, const_tree);
static void expand_operands (tree, tree, rtx, rtx*, rtx*,
                             enum expand_modifier);
static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
static rtx do_store_flag (sepops, rtx, enum machine_mode);
#ifdef PUSH_ROUNDING
static void emit_single_push_insn (enum machine_mode, rtx, tree);
#endif
static void do_tablejump (rtx, enum machine_mode, rtx, rtx, rtx);
static rtx const_vector_from_tree (tree);
static void write_complex_part (rtx, rtx, bool);

/* This macro is used to determine whether move_by_pieces should be called
   to perform a structure copy.  */
#ifndef MOVE_BY_PIECES_P
#define MOVE_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, MOVE_MAX_PIECES + 1) \
   < (unsigned int) MOVE_RATIO (optimize_insn_for_speed_p ()))
#endif

/* This macro is used to determine whether clear_by_pieces should be
   called to clear storage.  */
#ifndef CLEAR_BY_PIECES_P
#define CLEAR_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, STORE_MAX_PIECES + 1) \
   < (unsigned int) CLEAR_RATIO (optimize_insn_for_speed_p ()))
#endif

/* This macro is used to determine whether store_by_pieces should be
   called to "memset" storage with byte values other than zero.  */
#ifndef SET_BY_PIECES_P
#define SET_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, STORE_MAX_PIECES + 1) \
   < (unsigned int) SET_RATIO (optimize_insn_for_speed_p ()))
#endif

/* This macro is used to determine whether store_by_pieces should be
   called to "memcpy" storage when the source is a constant string.  */
#ifndef STORE_BY_PIECES_P
#define STORE_BY_PIECES_P(SIZE, ALIGN) \
  (move_by_pieces_ninsns (SIZE, ALIGN, STORE_MAX_PIECES + 1) \
   < (unsigned int) MOVE_RATIO (optimize_insn_for_speed_p ()))
#endif

/* SLOW_UNALIGNED_ACCESS is nonzero if unaligned accesses are very slow.  */

#ifndef SLOW_UNALIGNED_ACCESS
#define SLOW_UNALIGNED_ACCESS(MODE, ALIGN) STRICT_ALIGNMENT
#endif
\f
/* This is run to set up which modes can be used
   directly in memory and to initialize the block move optab.  It is run
   at the beginning of compilation and when the target is reinitialized.  */

void
init_expr_target (void)
{
  rtx insn, pat;
  enum machine_mode mode;
  int num_clobbers;
  rtx mem, mem1;
  rtx reg;

  /* Try indexing by frame ptr and try by stack ptr.
     It is known that on the Convex the stack ptr isn't a valid index.
     With luck, one or the other is valid on any machine.  */
  mem = gen_rtx_MEM (VOIDmode, stack_pointer_rtx);
  mem1 = gen_rtx_MEM (VOIDmode, frame_pointer_rtx);

  /* A scratch register we can modify in-place below to avoid
     useless RTL allocations.  */
  reg = gen_rtx_REG (VOIDmode, -1);

  insn = rtx_alloc (INSN);
  pat = gen_rtx_SET (VOIDmode, NULL_RTX, NULL_RTX);
  PATTERN (insn) = pat;

  for (mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
       mode = (enum machine_mode) ((int) mode + 1))
    {
      int regno;

      direct_load[(int) mode] = direct_store[(int) mode] = 0;
      PUT_MODE (mem, mode);
      PUT_MODE (mem1, mode);
      PUT_MODE (reg, mode);

      /* See if there is some register that can be used in this mode and
         directly loaded or stored from memory.  */

      if (mode != VOIDmode && mode != BLKmode)
        for (regno = 0; regno < FIRST_PSEUDO_REGISTER
             && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
             regno++)
          {
            if (! HARD_REGNO_MODE_OK (regno, mode))
              continue;

            SET_REGNO (reg, regno);

            SET_SRC (pat) = mem;
            SET_DEST (pat) = reg;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_load[(int) mode] = 1;

            SET_SRC (pat) = mem1;
            SET_DEST (pat) = reg;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_load[(int) mode] = 1;

            SET_SRC (pat) = reg;
            SET_DEST (pat) = mem;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_store[(int) mode] = 1;

            SET_SRC (pat) = reg;
            SET_DEST (pat) = mem1;
            if (recog (pat, insn, &num_clobbers) >= 0)
              direct_store[(int) mode] = 1;
          }
    }

  mem = gen_rtx_MEM (VOIDmode, gen_rtx_raw_REG (Pmode, 10000));

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum machine_mode srcmode;
      for (srcmode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); srcmode != mode;
           srcmode = GET_MODE_WIDER_MODE (srcmode))
        {
          enum insn_code ic;

          ic = can_extend_p (mode, srcmode, 0);
          if (ic == CODE_FOR_nothing)
            continue;

          PUT_MODE (mem, srcmode);

          if (insn_operand_matches (ic, 1, mem))
            float_extend_from_mem[mode][srcmode] = true;
        }
    }
}

/* This is run at the start of compiling a function.  */

void
init_expr (void)
{
  memset (&crtl->expr, 0, sizeof (crtl->expr));
}
\f
/* Copy data from FROM to TO, where the machine modes are not the same.
   Both modes may be integer, or both may be floating, or both may be
   fixed-point.
   UNSIGNEDP should be nonzero if FROM is an unsigned type.
   This causes zero-extension instead of sign-extension.  */

void
convert_move (rtx to, rtx from, int unsignedp)
{
  enum machine_mode to_mode = GET_MODE (to);
  enum machine_mode from_mode = GET_MODE (from);
  int to_real = SCALAR_FLOAT_MODE_P (to_mode);
  int from_real = SCALAR_FLOAT_MODE_P (from_mode);
  enum insn_code code;
  rtx libcall;

  /* rtx code for making an equivalent value.  */
  enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
                              : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));


  gcc_assert (to_real == from_real);
  gcc_assert (to_mode != BLKmode);
  gcc_assert (from_mode != BLKmode);

  /* If the source and destination are already the same, then there's
     nothing to do.  */
  if (to == from)
    return;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  We don't handle such SUBREGs as
     TO here.  */

  if (GET_CODE (from) == SUBREG && SUBREG_PROMOTED_VAR_P (from)
      && (GET_MODE_PRECISION (GET_MODE (SUBREG_REG (from)))
          >= GET_MODE_PRECISION (to_mode))
      && SUBREG_PROMOTED_UNSIGNED_P (from) == unsignedp)
    from = gen_lowpart (to_mode, from), from_mode = to_mode;

  gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));

  if (to_mode == from_mode
      || (from_mode == VOIDmode && CONSTANT_P (from)))
    {
      emit_move_insn (to, from);
      return;
    }

  if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
    {
      gcc_assert (GET_MODE_BITSIZE (from_mode) == GET_MODE_BITSIZE (to_mode));

      if (VECTOR_MODE_P (to_mode))
        from = simplify_gen_subreg (to_mode, from, GET_MODE (from), 0);
      else
        to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);

      emit_move_insn (to, from);
      return;
    }

  if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
    {
      convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
      convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
      return;
    }

  if (to_real)
    {
      rtx value, insns;
      convert_optab tab;

      gcc_assert ((GET_MODE_PRECISION (from_mode)
                   != GET_MODE_PRECISION (to_mode))
                  || (DECIMAL_FLOAT_MODE_P (from_mode)
                      != DECIMAL_FLOAT_MODE_P (to_mode)));

      if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
        /* Conversion between decimal float and binary float, same size.  */
        tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
      else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
        tab = sext_optab;
      else
        tab = trunc_optab;

      /* Try converting directly if the insn is supported.  */

      code = convert_optab_handler (tab, to_mode, from_mode);
      if (code != CODE_FOR_nothing)
        {
          emit_unop_insn (code, to, from,
                          tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
          return;
        }

      /* Otherwise use a libcall.  */
      libcall = convert_optab_libfunc (tab, to_mode, from_mode);

      /* Is this conversion implemented yet?  */
      gcc_assert (libcall);

      start_sequence ();
      value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
                                       1, from, from_mode);
      insns = get_insns ();
      end_sequence ();
      emit_libcall_block (insns, to, value,
                          tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
                                                                       from)
                          : gen_rtx_FLOAT_EXTEND (to_mode, from));
      return;
    }

  /* Handle pointer conversion.  */                     /* SPEE 900220.  */
  /* Targets are expected to provide conversion insns between PxImode and
     xImode for all MODE_PARTIAL_INT modes they use, but no others.  */
  if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
    {
      enum machine_mode full_mode
        = smallest_mode_for_size (GET_MODE_BITSIZE (to_mode), MODE_INT);

      gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
                  != CODE_FOR_nothing);

      if (full_mode != from_mode)
        from = convert_to_mode (full_mode, from, unsignedp);
      emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
                      to, from, UNKNOWN);
      return;
    }
  if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
    {
      rtx new_from;
      enum machine_mode full_mode
        = smallest_mode_for_size (GET_MODE_BITSIZE (from_mode), MODE_INT);

      gcc_assert (convert_optab_handler (sext_optab, full_mode, from_mode)
                  != CODE_FOR_nothing);

      if (to_mode == full_mode)
        {
          emit_unop_insn (convert_optab_handler (sext_optab, full_mode,
                                                 from_mode),
                          to, from, UNKNOWN);
          return;
        }

      new_from = gen_reg_rtx (full_mode);
      emit_unop_insn (convert_optab_handler (sext_optab, full_mode, from_mode),
                      new_from, from, UNKNOWN);

      /* else proceed to integer conversions below.  */
      from_mode = full_mode;
      from = new_from;
    }

   /* Make sure both are fixed-point modes or both are not.  */
   gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
               ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
   if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
    {
      /* If we widen from_mode to to_mode and they are in the same class,
         we won't saturate the result.
         Otherwise, always saturate the result to play safe.  */
      if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
          && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
        expand_fixed_convert (to, from, 0, 0);
      else
        expand_fixed_convert (to, from, 0, 1);
      return;
    }

  /* Now both modes are integers.  */

  /* Handle expanding beyond a word.  */
  if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
      && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
    {
      rtx insns;
      rtx lowpart;
      rtx fill_value;
      rtx lowfrom;
      int i;
      enum machine_mode lowpart_mode;
      int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);

      /* Try converting directly if the insn is supported.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
          != CODE_FOR_nothing)
        {
          /* If FROM is a SUBREG, put it into a register.  Do this
             so that we always generate the same set of insns for
             better cse'ing; if an intermediate assignment occurred,
             we won't be doing the operation directly on the SUBREG.  */
          if (optimize > 0 && GET_CODE (from) == SUBREG)
            from = force_reg (from_mode, from);
          emit_unop_insn (code, to, from, equiv_code);
          return;
        }
      /* Next, try converting via full word.  */
      else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
               && ((code = can_extend_p (to_mode, word_mode, unsignedp))
                   != CODE_FOR_nothing))
        {
          rtx word_to = gen_reg_rtx (word_mode);
          if (REG_P (to))
            {
              if (reg_overlap_mentioned_p (to, from))
                from = force_reg (from_mode, from);
              emit_clobber (to);
            }
          convert_move (word_to, from, unsignedp);
          emit_unop_insn (code, to, word_to, equiv_code);
          return;
        }

      /* No special multiword conversion insn; do it by hand.  */
      start_sequence ();

      /* Since we will turn this into a no conflict block, we must ensure
         that the source does not overlap the target.  */

      if (reg_overlap_mentioned_p (to, from))
        from = force_reg (from_mode, from);

      /* Get a copy of FROM widened to a word, if necessary.  */
      if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
        lowpart_mode = word_mode;
      else
        lowpart_mode = from_mode;

      lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);

      lowpart = gen_lowpart (lowpart_mode, to);
      emit_move_insn (lowpart, lowfrom);

      /* Compute the value to put in each remaining word.  */
      if (unsignedp)
        fill_value = const0_rtx;
      else
        fill_value = emit_store_flag (gen_reg_rtx (word_mode),
                                      LT, lowfrom, const0_rtx,
                                      VOIDmode, 0, -1);

      /* Fill the remaining words.  */
      for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
        {
          int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
          rtx subword = operand_subword (to, index, 1, to_mode);

          gcc_assert (subword);

          if (fill_value != subword)
            emit_move_insn (subword, fill_value);
        }

      insns = get_insns ();
      end_sequence ();

      emit_insn (insns);
      return;
    }

  /* Truncating multi-word to a word or less.  */
  if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
      && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
    {
      if (!((MEM_P (from)
             && ! MEM_VOLATILE_P (from)
             && direct_load[(int) to_mode]
             && ! mode_dependent_address_p (XEXP (from, 0)))
            || REG_P (from)
            || GET_CODE (from) == SUBREG))
        from = force_reg (from_mode, from);
      convert_move (to, gen_lowpart (word_mode, from), 0);
      return;
    }

  /* Now follow all the conversions between integers
     no more than a word long.  */

  /* For truncation, usually we can just refer to FROM in a narrower mode.  */
  if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
      && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
    {
      if (!((MEM_P (from)
             && ! MEM_VOLATILE_P (from)
             && direct_load[(int) to_mode]
             && ! mode_dependent_address_p (XEXP (from, 0)))
            || REG_P (from)
            || GET_CODE (from) == SUBREG))
        from = force_reg (from_mode, from);
      if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
          && ! HARD_REGNO_MODE_OK (REGNO (from), to_mode))
        from = copy_to_reg (from);
      emit_move_insn (to, gen_lowpart (to_mode, from));
      return;
    }

  /* Handle extension.  */
  if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
    {
      /* Convert directly if that works.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
          != CODE_FOR_nothing)
        {
          emit_unop_insn (code, to, from, equiv_code);
          return;
        }
      else
        {
          enum machine_mode intermediate;
          rtx tmp;
          int shift_amount;

          /* Search for a mode to convert via.  */
          for (intermediate = from_mode; intermediate != VOIDmode;
               intermediate = GET_MODE_WIDER_MODE (intermediate))
            if (((can_extend_p (to_mode, intermediate, unsignedp)
                  != CODE_FOR_nothing)
                 || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
                     && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, intermediate)))
                && (can_extend_p (intermediate, from_mode, unsignedp)
                    != CODE_FOR_nothing))
              {
                convert_move (to, convert_to_mode (intermediate, from,
                                                   unsignedp), unsignedp);
                return;
              }

          /* No suitable intermediate mode.
             Generate what we need with shifts.  */
          shift_amount = (GET_MODE_PRECISION (to_mode)
                          - GET_MODE_PRECISION (from_mode));
          from = gen_lowpart (to_mode, force_reg (from_mode, from));
          tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
                              to, unsignedp);
          tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
                              to, unsignedp);
          if (tmp != to)
            emit_move_insn (to, tmp);
          return;
        }
    }

  /* Support special truncate insns for certain modes.  */
  if (convert_optab_handler (trunc_optab, to_mode,
                             from_mode) != CODE_FOR_nothing)
    {
      emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
                      to, from, UNKNOWN);
      return;
    }

  /* Handle truncation of volatile memrefs, and so on;
     the things that couldn't be truncated directly,
     and for which there was no special instruction.

     ??? Code above formerly short-circuited this, for most integer
     mode pairs, with a force_reg in from_mode followed by a recursive
     call to this routine.  Appears always to have been wrong.  */
  if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
    {
      rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
      emit_move_insn (to, temp);
      return;
    }

  /* Mode combination is not recognized.  */
  gcc_unreachable ();
}

/* Return an rtx for a value that would result
   from converting X to mode MODE.
   Both X and MODE may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.
   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.  */

rtx
convert_to_mode (enum machine_mode mode, rtx x, int unsignedp)
{
  return convert_modes (mode, VOIDmode, x, unsignedp);
}

/* Return an rtx for a value that would result
   from converting X from mode OLDMODE to mode MODE.
   Both modes may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.

   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.

   You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode.  */

rtx
convert_modes (enum machine_mode mode, enum machine_mode oldmode, rtx x, int unsignedp)
{
  rtx temp;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  */

  if (GET_CODE (x) == SUBREG && SUBREG_PROMOTED_VAR_P (x)
      && GET_MODE_SIZE (GET_MODE (SUBREG_REG (x))) >= GET_MODE_SIZE (mode)
      && SUBREG_PROMOTED_UNSIGNED_P (x) == unsignedp)
    x = gen_lowpart (mode, x);

  if (GET_MODE (x) != VOIDmode)
    oldmode = GET_MODE (x);

  if (mode == oldmode)
    return x;

  /* There is one case that we must handle specially: If we are converting
     a CONST_INT into a mode whose size is twice HOST_BITS_PER_WIDE_INT and
     we are to interpret the constant as unsigned, gen_lowpart will do
     the wrong if the constant appears negative.  What we want to do is
     make the high-order word of the constant zero, not all ones.  */

  if (unsignedp && GET_MODE_CLASS (mode) == MODE_INT
      && GET_MODE_BITSIZE (mode) == 2 * HOST_BITS_PER_WIDE_INT
      && CONST_INT_P (x) && INTVAL (x) < 0)
    {
      double_int val = uhwi_to_double_int (INTVAL (x));

      /* We need to zero extend VAL.  */
      if (oldmode != VOIDmode)
        val = double_int_zext (val, GET_MODE_BITSIZE (oldmode));

      return immed_double_int_const (val, mode);
    }

  /* We can do this with a gen_lowpart if both desired and current modes
     are integer, and this is either a constant integer, a register, or a
     non-volatile MEM.  Except for the constant case where MODE is no
     wider than HOST_BITS_PER_WIDE_INT, we must be narrowing the operand.  */

  if ((CONST_INT_P (x)
       && GET_MODE_PRECISION (mode) <= HOST_BITS_PER_WIDE_INT)
      || (GET_MODE_CLASS (mode) == MODE_INT
          && GET_MODE_CLASS (oldmode) == MODE_INT
          && (GET_CODE (x) == CONST_DOUBLE
              || (GET_MODE_PRECISION (mode) <= GET_MODE_PRECISION (oldmode)
                  && ((MEM_P (x) && ! MEM_VOLATILE_P (x)
                       && direct_load[(int) mode])
                      || (REG_P (x)
                          && (! HARD_REGISTER_P (x)
                              || HARD_REGNO_MODE_OK (REGNO (x), mode))
                          && TRULY_NOOP_TRUNCATION_MODES_P (mode,
                                                            GET_MODE (x))))))))
    {
      /* ?? If we don't know OLDMODE, we have to assume here that
         X does not need sign- or zero-extension.   This may not be
         the case, but it's the best we can do.  */
      if (CONST_INT_P (x) && oldmode != VOIDmode
          && GET_MODE_PRECISION (mode) > GET_MODE_PRECISION (oldmode))
        {
          HOST_WIDE_INT val = INTVAL (x);

          /* We must sign or zero-extend in this case.  Start by
             zero-extending, then sign extend if we need to.  */
          val &= GET_MODE_MASK (oldmode);
          if (! unsignedp
              && val_signbit_known_set_p (oldmode, val))
            val |= ~GET_MODE_MASK (oldmode);

          return gen_int_mode (val, mode);
        }

      return gen_lowpart (mode, x);
    }

  /* Converting from integer constant into mode is always equivalent to an
     subreg operation.  */
  if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
    {
      gcc_assert (GET_MODE_BITSIZE (mode) == GET_MODE_BITSIZE (oldmode));
      return simplify_gen_subreg (mode, x, oldmode, 0);
    }

  temp = gen_reg_rtx (mode);
  convert_move (temp, x, unsignedp);
  return temp;
}
\f
/* Return the largest alignment we can use for doing a move (or store)
   of MAX_PIECES.  ALIGN is the largest alignment we could use.  */

static unsigned int
alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
{
  enum machine_mode tmode;

  tmode = mode_for_size (max_pieces * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      enum machine_mode tmode, xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
           tmode != VOIDmode;
           xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
        if (GET_MODE_SIZE (tmode) > max_pieces
            || SLOW_UNALIGNED_ACCESS (tmode, align))
          break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  return align;
}

/* Return the widest integer mode no wider than SIZE.  If no such mode
   can be found, return VOIDmode.  */

static enum machine_mode
widest_int_mode_for_size (unsigned int size)
{
  enum machine_mode tmode, mode = VOIDmode;

  for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
       tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
    if (GET_MODE_SIZE (tmode) < size)
      mode = tmode;

  return mode;
}

/* STORE_MAX_PIECES is the number of bytes at a time that we can
   store efficiently.  Due to internal GCC limitations, this is
   MOVE_MAX_PIECES limited by the number of bytes GCC can represent
   for an immediate constant.  */

#define STORE_MAX_PIECES  MIN (MOVE_MAX_PIECES, 2 * sizeof (HOST_WIDE_INT))

/* Determine whether the LEN bytes can be moved by using several move
   instructions.  Return nonzero if a call to move_by_pieces should
   succeed.  */

int
can_move_by_pieces (unsigned HOST_WIDE_INT len,
                    unsigned int align ATTRIBUTE_UNUSED)
{
  return MOVE_BY_PIECES_P (len, align);
}

/* Generate several move instructions to copy LEN bytes from block FROM to
   block TO.  (These are MEM rtx's with BLKmode).

   If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
   used to push FROM to the stack.

   ALIGN is maximum stack alignment we can assume.

   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
                unsigned int align, int endp)
{
  struct move_by_pieces_d data;
  enum machine_mode to_addr_mode, from_addr_mode
    = targetm.addr_space.address_mode (MEM_ADDR_SPACE (from));
  rtx to_addr, from_addr = XEXP (from, 0);
  unsigned int max_size = MOVE_MAX_PIECES + 1;
  enum insn_code icode;

  align = MIN (to ? MEM_ALIGN (to) : align, MEM_ALIGN (from));

  data.offset = 0;
  data.from_addr = from_addr;
  if (to)
    {
      to_addr_mode = targetm.addr_space.address_mode (MEM_ADDR_SPACE (to));
      to_addr = XEXP (to, 0);
      data.to = to;
      data.autinc_to
        = (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
           || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);
      data.reverse
        = (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
    }
  else
    {
      to_addr_mode = VOIDmode;
      to_addr = NULL_RTX;
      data.to = NULL_RTX;
      data.autinc_to = 1;
#ifdef STACK_GROWS_DOWNWARD
      data.reverse = 1;
#else
      data.reverse = 0;
#endif
    }
  data.to_addr = to_addr;
  data.from = from;
  data.autinc_from
    = (GET_CODE (from_addr) == PRE_INC || GET_CODE (from_addr) == PRE_DEC
       || GET_CODE (from_addr) == POST_INC
       || GET_CODE (from_addr) == POST_DEC);

  data.explicit_inc_from = 0;
  data.explicit_inc_to = 0;
  if (data.reverse) data.offset = len;
  data.len = len;

  /* If copying requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!(data.autinc_from && data.autinc_to)
      && move_by_pieces_ninsns (len, align, max_size) > 2)
    {
      /* Find the mode of the largest move...
         MODE might not be used depending on the definitions of the
         USE_* macros below.  */
      enum machine_mode mode ATTRIBUTE_UNUSED
        = widest_int_mode_for_size (max_size);

      if (USE_LOAD_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_from)
        {
          data.from_addr = copy_to_mode_reg (from_addr_mode,
                                             plus_constant (from_addr, len));
          data.autinc_from = 1;
          data.explicit_inc_from = -1;
        }
      if (USE_LOAD_POST_INCREMENT (mode) && ! data.autinc_from)
        {
          data.from_addr = copy_to_mode_reg (from_addr_mode, from_addr);
          data.autinc_from = 1;
          data.explicit_inc_from = 1;
        }
      if (!data.autinc_from && CONSTANT_P (from_addr))
        data.from_addr = copy_to_mode_reg (from_addr_mode, from_addr);
      if (USE_STORE_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_to)
        {
          data.to_addr = copy_to_mode_reg (to_addr_mode,
                                           plus_constant (to_addr, len));
          data.autinc_to = 1;
          data.explicit_inc_to = -1;
        }
      if (USE_STORE_POST_INCREMENT (mode) && ! data.reverse && ! data.autinc_to)
        {
          data.to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
          data.autinc_to = 1;
          data.explicit_inc_to = 1;
        }
      if (!data.autinc_to && CONSTANT_P (to_addr))
        data.to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
    }

  align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);

  /* First move what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1)
    {
      enum machine_mode mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
        break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        move_by_pieces_1 (GEN_FCN (icode), mode, &data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data.len);

  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
        {
          if (endp == 2)
            {
              if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
                emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
              else
                data.to_addr = copy_to_mode_reg (to_addr_mode,
                                                 plus_constant (data.to_addr,
                                                                -1));
            }
          to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
                                           data.offset);
        }
      else
        {
          if (endp == 2)
            --data.offset;
          to1 = adjust_address (data.to, QImode, data.offset);
        }
      return to1;
    }
  else
    return data.to;
}

/* Return number of insns required to move L bytes by pieces.
   ALIGN (in bits) is maximum alignment we can assume.  */

unsigned HOST_WIDE_INT
move_by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
                       unsigned int max_size)
{
  unsigned HOST_WIDE_INT n_insns = 0;

  align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);

  while (max_size > 1)
    {
      enum machine_mode mode;
      enum insn_code icode;

      mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
        break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        n_insns += l / GET_MODE_SIZE (mode), l %= GET_MODE_SIZE (mode);

      max_size = GET_MODE_SIZE (mode);
    }

  gcc_assert (!l);
  return n_insns;
}

/* Subroutine of move_by_pieces.  Move as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
move_by_pieces_1 (rtx (*genfun) (rtx, ...), enum machine_mode mode,
                  struct move_by_pieces_d *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1 = NULL_RTX, from1;

  while (data->len >= size)
    {
      if (data->reverse)
        data->offset -= size;

      if (data->to)
        {
          if (data->autinc_to)
            to1 = adjust_automodify_address (data->to, mode, data->to_addr,
                                             data->offset);
          else
            to1 = adjust_address (data->to, mode, data->offset);
        }

      if (data->autinc_from)
        from1 = adjust_automodify_address (data->from, mode, data->from_addr,
                                           data->offset);
      else
        from1 = adjust_address (data->from, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
        emit_insn (gen_add2_insn (data->to_addr,
                                  GEN_INT (-(HOST_WIDE_INT)size)));
      if (HAVE_PRE_DECREMENT && data->explicit_inc_from < 0)
        emit_insn (gen_add2_insn (data->from_addr,
                                  GEN_INT (-(HOST_WIDE_INT)size)));

      if (data->to)
        emit_insn ((*genfun) (to1, from1));
      else
        {
#ifdef PUSH_ROUNDING
          emit_single_push_insn (mode, from1, NULL);
#else
          gcc_unreachable ();
#endif
        }

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
        emit_insn (gen_add2_insn (data->to_addr, GEN_INT (size)));
      if (HAVE_POST_INCREMENT && data->explicit_inc_from > 0)
        emit_insn (gen_add2_insn (data->from_addr, GEN_INT (size)));

      if (! data->reverse)
        data->offset += size;

      data->len -= size;
    }
}
\f
/* Emit code to move a block Y to a block X.  This may be done with
   string-move instructions, with multiple scalar move instructions,
   or with a library call.

   Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
   SIZE is an rtx that says how long they are.
   ALIGN is the maximum alignment we can assume they have.
   METHOD describes what kind of copy this is, and what mechanisms may be used.

   Return the address of the new block, if memcpy is called and returns it,
   0 otherwise.  */

rtx
emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
                       unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  bool may_use_call;
  rtx retval = 0;
  unsigned int align;

  gcc_assert (size);
  if (CONST_INT_P (size)
      && INTVAL (size) == 0)
    return 0;

  switch (method)
    {
    case BLOCK_OP_NORMAL:
    case BLOCK_OP_TAILCALL:
      may_use_call = true;
      break;

    case BLOCK_OP_CALL_PARM:
      may_use_call = block_move_libcall_safe_for_call_parm ();

      /* Make inhibit_defer_pop nonzero around the library call
         to force it to pop the arguments right away.  */
      NO_DEFER_POP;
      break;

    case BLOCK_OP_NO_LIBCALL:
      may_use_call = false;
      break;

    default:
      gcc_unreachable ();
    }

  gcc_assert (MEM_P (x) && MEM_P (y));
  align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
  gcc_assert (align >= BITS_PER_UNIT);

  /* Make sure we've got BLKmode addresses; store_one_arg can decide that
     block copy is more efficient for other large modes, e.g. DCmode.  */
  x = adjust_address (x, BLKmode, 0);
  y = adjust_address (y, BLKmode, 0);

  /* Set MEM_SIZE as appropriate for this block copy.  The main place this
     can be incorrect is coming from __builtin_memcpy.  */
  if (CONST_INT_P (size))
    {
      x = shallow_copy_rtx (x);
      y = shallow_copy_rtx (y);
      set_mem_size (x, INTVAL (size));
      set_mem_size (y, INTVAL (size));
    }

  if (CONST_INT_P (size) && MOVE_BY_PIECES_P (INTVAL (size), align))
    move_by_pieces (x, y, INTVAL (size), align, 0);
  else if (emit_block_move_via_movmem (x, y, size, align,
                                       expected_align, expected_size))
    ;
  else if (may_use_call
           && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
           && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
    {
      /* Since x and y are passed to a libcall, mark the corresponding
         tree EXPR as addressable.  */
      tree y_expr = MEM_EXPR (y);
      tree x_expr = MEM_EXPR (x);
      if (y_expr)
        mark_addressable (y_expr);
      if (x_expr)
        mark_addressable (x_expr);
      retval = emit_block_move_via_libcall (x, y, size,
                                            method == BLOCK_OP_TAILCALL);
    }

  else
    emit_block_move_via_loop (x, y, size, align);

  if (method == BLOCK_OP_CALL_PARM)
    OK_DEFER_POP;

  return retval;
}

rtx
emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method)
{
  return emit_block_move_hints (x, y, size, method, 0, -1);
}

/* A subroutine of emit_block_move.  Returns true if calling the
   block move libcall will not clobber any parameters which may have
   already been placed on the stack.  */

static bool
block_move_libcall_safe_for_call_parm (void)
{
#if defined (REG_PARM_STACK_SPACE)
  tree fn;
#endif

  /* If arguments are pushed on the stack, then they're safe.  */
  if (PUSH_ARGS)
    return true;

  /* If registers go on the stack anyway, any argument is sure to clobber
     an outgoing argument.  */
#if defined (REG_PARM_STACK_SPACE)
  fn = emit_block_move_libcall_fn (false);
  /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
     depend on its argument.  */
  (void) fn;
  if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
      && REG_PARM_STACK_SPACE (fn) != 0)
    return false;
#endif

  /* If any argument goes in memory, then it might clobber an outgoing
     argument.  */
  {
    CUMULATIVE_ARGS args_so_far_v;
    cumulative_args_t args_so_far;
    tree fn, arg;

    fn = emit_block_move_libcall_fn (false);
    INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
    args_so_far = pack_cumulative_args (&args_so_far_v);

    arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
    for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
      {
        enum machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
        rtx tmp = targetm.calls.function_arg (args_so_far, mode,
                                              NULL_TREE, true);
        if (!tmp || !REG_P (tmp))
          return false;
        if (targetm.calls.arg_partial_bytes (args_so_far, mode, NULL, 1))
          return false;
        targetm.calls.function_arg_advance (args_so_far, mode,
                                            NULL_TREE, true);
      }
  }
  return true;
}

/* A subroutine of emit_block_move.  Expand a movmem pattern;
   return true if successful.  */

static bool
emit_block_move_via_movmem (rtx x, rtx y, rtx size, unsigned int align,
                            unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  int save_volatile_ok = volatile_ok;
  enum machine_mode mode;

  if (expected_align < align)
    expected_align = align;

  /* Since this is a move insn, we don't care about volatility.  */
  volatile_ok = 1;

  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = direct_optab_handler (movmem_optab, mode);

      if (code != CODE_FOR_nothing
          /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
             here because if SIZE is less than the mode mask, as it is
             returned by the macro, it will definitely be less than the
             actual mode mask.  */
          && ((CONST_INT_P (size)
               && ((unsigned HOST_WIDE_INT) INTVAL (size)
                   <= (GET_MODE_MASK (mode) >> 1)))
              || GET_MODE_BITSIZE (mode) >= BITS_PER_WORD))
        {
          struct expand_operand ops[6];
          unsigned int nops;

          /* ??? When called via emit_block_move_for_call, it'd be
             nice if there were some way to inform the backend, so
             that it doesn't fail the expansion because it thinks
             emitting the libcall would be more efficient.  */
          nops = insn_data[(int) code].n_generator_args;
          gcc_assert (nops == 4 || nops == 6);

          create_fixed_operand (&ops[0], x);
          create_fixed_operand (&ops[1], y);
          /* The check above guarantees that this size conversion is valid.  */
          create_convert_operand_to (&ops[2], size, mode, true);
          create_integer_operand (&ops[3], align / BITS_PER_UNIT);
          if (nops == 6)
            {
              create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
              create_integer_operand (&ops[5], expected_size);
            }
          if (maybe_expand_insn (code, nops, ops))
            {
              volatile_ok = save_volatile_ok;
              return true;
            }
        }
    }

  volatile_ok = save_volatile_ok;
  return false;
}

/* A subroutine of emit_block_move.  Expand a call to memcpy.
   Return the return value from memcpy, 0 otherwise.  */

rtx
emit_block_move_via_libcall (rtx dst, rtx src, rtx size, bool tailcall)
{
  rtx dst_addr, src_addr;
  tree call_expr, fn, src_tree, dst_tree, size_tree;
  enum machine_mode size_mode;
  rtx retval;

  /* Emit code to copy the addresses of DST and SRC and SIZE into new
     pseudos.  We can then place those new pseudos into a VAR_DECL and
     use them later.  */

  dst_addr = copy_to_mode_reg (Pmode, XEXP (dst, 0));
  src_addr = copy_to_mode_reg (Pmode, XEXP (src, 0));

  dst_addr = convert_memory_address (ptr_mode, dst_addr);
  src_addr = convert_memory_address (ptr_mode, src_addr);

  dst_tree = make_tree (ptr_type_node, dst_addr);
  src_tree = make_tree (ptr_type_node, src_addr);

  size_mode = TYPE_MODE (sizetype);

  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memcpy in this context.  This could be a user call to memcpy and
     the user may wish to examine the return value from memcpy.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  size_tree = make_tree (sizetype, size);

  fn = emit_block_move_libcall_fn (true);
  call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of emit_block_move_via_libcall.  Create the tree node
   for the function we use for block copies.  The first time FOR_CALL
   is true, we call assemble_external.  */

static GTY(()) tree block_move_fn;

void
init_block_move_fn (const char *asmspec)
{
  if (!block_move_fn)
    {
      tree args, fn;

      fn = get_identifier ("memcpy");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
                                       const_ptr_type_node, sizetype,
                                       NULL_TREE);

      fn = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      block_move_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_move_fn, asmspec);
}

static tree
emit_block_move_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_move_fn)
    init_block_move_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_move_fn);
      assemble_external (block_move_fn);
    }

  return block_move_fn;
}

/* A subroutine of emit_block_move.  Copy the data via an explicit
   loop.  This is used only when libcalls are forbidden.  */
/* ??? It'd be nice to copy in hunks larger than QImode.  */

static void
emit_block_move_via_loop (rtx x, rtx y, rtx size,
                          unsigned int align ATTRIBUTE_UNUSED)
{
  rtx cmp_label, top_label, iter, x_addr, y_addr, tmp;
  enum machine_mode x_addr_mode
    = targetm.addr_space.address_mode (MEM_ADDR_SPACE (x));
  enum machine_mode y_addr_mode
    = targetm.addr_space.address_mode (MEM_ADDR_SPACE (y));
  enum machine_mode iter_mode;

  iter_mode = GET_MODE (size);
  if (iter_mode == VOIDmode)
    iter_mode = word_mode;

  top_label = gen_label_rtx ();
  cmp_label = gen_label_rtx ();
  iter = gen_reg_rtx (iter_mode);

  emit_move_insn (iter, const0_rtx);

  x_addr = force_operand (XEXP (x, 0), NULL_RTX);
  y_addr = force_operand (XEXP (y, 0), NULL_RTX);
  do_pending_stack_adjust ();

  emit_jump (cmp_label);
  emit_label (top_label);

  tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
  x_addr = gen_rtx_PLUS (x_addr_mode, x_addr, tmp);

  if (x_addr_mode != y_addr_mode)
    tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
  y_addr = gen_rtx_PLUS (y_addr_mode, y_addr, tmp);

  x = change_address (x, QImode, x_addr);
  y = change_address (y, QImode, y_addr);

  emit_move_insn (x, y);

  tmp = expand_simple_binop (iter_mode, PLUS, iter, const1_rtx, iter,
                             true, OPTAB_LIB_WIDEN);
  if (tmp != iter)
    emit_move_insn (iter, tmp);

  emit_label (cmp_label);

  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
                           true, top_label);
}
\f
/* Copy all or part of a value X into registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_to_reg (int regno, rtx x, int nregs, enum machine_mode mode)
{
  int i;
#ifdef HAVE_load_multiple
  rtx pat;
  rtx last;
#endif

  if (nregs == 0)
    return;

  if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
    x = validize_mem (force_const_mem (mode, x));

  /* See if the machine can do this with a load multiple insn.  */
#ifdef HAVE_load_multiple
  if (HAVE_load_multiple)
    {
      last = get_last_insn ();
      pat = gen_load_multiple (gen_rtx_REG (word_mode, regno), x,
                               GEN_INT (nregs));
      if (pat)
        {
          emit_insn (pat);
          return;
        }
      else
        delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    emit_move_insn (gen_rtx_REG (word_mode, regno + i),
                    operand_subword_force (x, i, mode));
}

/* Copy all or part of a BLKmode value X out of registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_from_reg (int regno, rtx x, int nregs)
{
  int i;

  if (nregs == 0)
    return;

  /* See if the machine can do this with a store multiple insn.  */
#ifdef HAVE_store_multiple
  if (HAVE_store_multiple)
    {
      rtx last = get_last_insn ();
      rtx pat = gen_store_multiple (x, gen_rtx_REG (word_mode, regno),
                                    GEN_INT (nregs));
      if (pat)
        {
          emit_insn (pat);
          return;
        }
      else
        delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    {
      rtx tem = operand_subword (x, i, 1, BLKmode);

      gcc_assert (tem);

      emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
    }
}

/* Generate a PARALLEL rtx for a new non-consecutive group of registers from
   ORIG, where ORIG is a non-consecutive group of registers represented by
   a PARALLEL.  The clone is identical to the original except in that the
   original set of registers is replaced by a new set of pseudo registers.
   The new set has the same modes as the original set.  */

rtx
gen_group_rtx (rtx orig)
{
  int i, length;
  rtx *tmps;

  gcc_assert (GET_CODE (orig) == PARALLEL);

  length = XVECLEN (orig, 0);
  tmps = XALLOCAVEC (rtx, length);

  /* Skip a NULL entry in first slot.  */
  i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;

  if (i)
    tmps[0] = 0;

  for (; i < length; i++)
    {
      enum machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
      rtx offset = XEXP (XVECEXP (orig, 0, i), 1);

      tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
    }

  return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
}

/* A subroutine of emit_group_load.  Arguments as for emit_group_load,
   except that values are placed in TMPS[i], and must later be moved
   into corresponding XEXP (XVECEXP (DST, 0, i), 0) element.  */

static void
emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type, int ssize)
{
  rtx src;
  int start, i;
  enum machine_mode m = GET_MODE (orig_src);

  gcc_assert (GET_CODE (dst) == PARALLEL);

  if (m != VOIDmode
      && !SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_src)
      && GET_CODE (orig_src) != CONCAT)
    {
      enum machine_mode imode = int_mode_for_mode (GET_MODE (orig_src));
      if (imode == BLKmode)
        src = assign_stack_temp (GET_MODE (orig_src), ssize, 0);
      else
        src = gen_reg_rtx (imode);
      if (imode != BLKmode)
        src = gen_lowpart (GET_MODE (orig_src), src);
      emit_move_insn (src, orig_src);
      /* ...and back again.  */
      if (imode != BLKmode)
        src = gen_lowpart (imode, src);
      emit_group_load_1 (tmps, dst, src, type, ssize);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (dst, 0, 0), 0))
    start = 0;
  else
    start = 1;

  /* Process the pieces.  */
  for (i = start; i < XVECLEN (dst, 0); i++)
    {
      enum machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (dst, 0, i), 1));
      unsigned int bytelen = GET_MODE_SIZE (mode);
      int shift = 0;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
        {
          /* Arrange to shift the fragment to where it belongs.
             extract_bit_field loads to the lsb of the reg.  */
          if (
#ifdef BLOCK_REG_PADDING
              BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
              == (BYTES_BIG_ENDIAN ? upward : downward)
#else
              BYTES_BIG_ENDIAN
#endif
              )
            shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
          bytelen = ssize - bytepos;
          gcc_assert (bytelen > 0);
        }

      /* If we won't be loading directly from memory, protect the real source
         from strange tricks we might play; but make sure that the source can
         be loaded directly into the destination.  */
      src = orig_src;
      if (!MEM_P (orig_src)
          && (!CONSTANT_P (orig_src)
              || (GET_MODE (orig_src) != mode
                  && GET_MODE (orig_src) != VOIDmode)))
        {
          if (GET_MODE (orig_src) == VOIDmode)
            src = gen_reg_rtx (mode);
          else
            src = gen_reg_rtx (GET_MODE (orig_src));

          emit_move_insn (src, orig_src);
        }

      /* Optimize the access just a bit.  */
      if (MEM_P (src)
          && (! SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (src))
              || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
          && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
          && bytelen == GET_MODE_SIZE (mode))
        {
          tmps[i] = gen_reg_rtx (mode);
          emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
        }
      else if (COMPLEX_MODE_P (mode)
               && GET_MODE (src) == mode
               && bytelen == GET_MODE_SIZE (mode))
        /* Let emit_move_complex do the bulk of the work.  */
        tmps[i] = src;
      else if (GET_CODE (src) == CONCAT)
        {
          unsigned int slen = GET_MODE_SIZE (GET_MODE (src));
          unsigned int slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));

          if ((bytepos == 0 && bytelen == slen0)
              || (bytepos != 0 && bytepos + bytelen <= slen))
            {
              /* The following assumes that the concatenated objects all
                 have the same size.  In this case, a simple calculation
                 can be used to determine the object and the bit field
                 to be extracted.  */
              tmps[i] = XEXP (src, bytepos / slen0);
              if (! CONSTANT_P (tmps[i])
                  && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode))
                tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
                                             (bytepos % slen0) * BITS_PER_UNIT,
                                             1, false, NULL_RTX, mode, mode);
            }
          else
            {
              rtx mem;

              gcc_assert (!bytepos);
              mem = assign_stack_temp (GET_MODE (src), slen, 0);
              emit_move_insn (mem, src);
              tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
                                           0, 1, false, NULL_RTX, mode, mode);
            }
        }
      /* FIXME: A SIMD parallel will eventually lead to a subreg of a
         SIMD register, which is currently broken.  While we get GCC
         to emit proper RTL for these cases, let's dump to memory.  */
      else if (VECTOR_MODE_P (GET_MODE (dst))
               && REG_P (src))
        {
          int slen = GET_MODE_SIZE (GET_MODE (src));
          rtx mem;

          mem = assign_stack_temp (GET_MODE (src), slen, 0);
          emit_move_insn (mem, src);
          tmps[i] = adjust_address (mem, mode, (int) bytepos);
        }
      else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
               && XVECLEN (dst, 0) > 1)
        tmps[i] = simplify_gen_subreg (mode, src, GET_MODE(dst), bytepos);
      else if (CONSTANT_P (src))
        {
          HOST_WIDE_INT len = (HOST_WIDE_INT) bytelen;

          if (len == ssize)
            tmps[i] = src;
          else
            {
              rtx first, second;

              gcc_assert (2 * len == ssize);
              split_double (src, &first, &second);
              if (i)
                tmps[i] = second;
              else
                tmps[i] = first;
            }
        }
      else if (REG_P (src) && GET_MODE (src) == mode)
        tmps[i] = src;
      else
        tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
                                     bytepos * BITS_PER_UNIT, 1, false, NULL_RTX,
                                     mode, mode);

      if (shift)
        tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
                                shift, tmps[i], 0);
    }
}

/* Emit code to move a block SRC of type TYPE to a block DST,
   where DST is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_SRC in bytes, or -1
   if not known.  */

void
emit_group_load (rtx dst, rtx src, tree type, int ssize)
{
  rtx *tmps;
  int i;

  tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
  emit_group_load_1 (tmps, dst, src, type, ssize);

  /* Copy the extracted pieces into the proper (probable) hard regs.  */
  for (i = 0; i < XVECLEN (dst, 0); i++)
    {
      rtx d = XEXP (XVECEXP (dst, 0, i), 0);
      if (d == NULL)
        continue;
      emit_move_insn (d, tmps[i]);
    }
}

/* Similar, but load SRC into new pseudos in a format that looks like
   PARALLEL.  This can later be fed to emit_group_move to get things
   in the right place.  */

rtx
emit_group_load_into_temps (rtx parallel, rtx src, tree type, int ssize)
{
  rtvec vec;
  int i;

  vec = rtvec_alloc (XVECLEN (parallel, 0));
  emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);

  /* Convert the vector to look just like the original PARALLEL, except
     with the computed values.  */
  for (i = 0; i < XVECLEN (parallel, 0); i++)
    {
      rtx e = XVECEXP (parallel, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
        {
          d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
          e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
        }
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
}

/* Emit code to move a block SRC to block DST, where SRC and DST are
   non-consecutive groups of registers, each represented by a PARALLEL.  */

void
emit_group_move (rtx dst, rtx src)
{
  int i;

  gcc_assert (GET_CODE (src) == PARALLEL
              && GET_CODE (dst) == PARALLEL
              && XVECLEN (src, 0) == XVECLEN (dst, 0));

  /* Skip first entry if NULL.  */
  for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
    emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
                    XEXP (XVECEXP (src, 0, i), 0));
}

/* Move a group of registers represented by a PARALLEL into pseudos.  */

rtx
emit_group_move_into_temps (rtx src)
{
  rtvec vec = rtvec_alloc (XVECLEN (src, 0));
  int i;

  for (i = 0; i < XVECLEN (src, 0); i++)
    {
      rtx e = XVECEXP (src, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
        e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (src), vec);
}

/* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
   where SRC is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_DST, or -1 if not
   known.  */

void
emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED, int ssize)
{
  rtx *tmps, dst;
  int start, finish, i;
  enum machine_mode m = GET_MODE (orig_dst);

  gcc_assert (GET_CODE (src) == PARALLEL);

  if (!SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
    {
      enum machine_mode imode = int_mode_for_mode (GET_MODE (orig_dst));
      if (imode == BLKmode)
        dst = assign_stack_temp (GET_MODE (orig_dst), ssize, 0);
      else
        dst = gen_reg_rtx (imode);
      emit_group_store (dst, src, type, ssize);
      if (imode != BLKmode)
        dst = gen_lowpart (GET_MODE (orig_dst), dst);
      emit_move_insn (orig_dst, dst);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (src, 0, 0), 0))
    start = 0;
  else
    start = 1;
  finish = XVECLEN (src, 0);

  tmps = XALLOCAVEC (rtx, finish);

  /* Copy the (probable) hard regs into pseudos.  */
  for (i = start; i < finish; i++)
    {
      rtx reg = XEXP (XVECEXP (src, 0, i), 0);
      if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
        {
          tmps[i] = gen_reg_rtx (GET_MODE (reg));
          emit_move_insn (tmps[i], reg);
        }
      else
        tmps[i] = reg;
    }

  /* If we won't be storing directly into memory, protect the real destination
     from strange tricks we might play.  */
  dst = orig_dst;
  if (GET_CODE (dst) == PARALLEL)
    {
      rtx temp;

      /* We can get a PARALLEL dst if there is a conditional expression in
         a return statement.  In that case, the dst and src are the same,
         so no action is necessary.  */
      if (rtx_equal_p (dst, src))
        return;

      /* It is unclear if we can ever reach here, but we may as well handle
         it.  Allocate a temporary, and split this into a store/load to/from
         the temporary.  */

      temp = assign_stack_temp (GET_MODE (dst), ssize, 0);
      emit_group_store (temp, src, type, ssize);
      emit_group_load (dst, temp, type, ssize);
      return;
    }
  else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
    {
      enum machine_mode outer = GET_MODE (dst);
      enum machine_mode inner;
      HOST_WIDE_INT bytepos;
      bool done = false;
      rtx temp;

      if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
        dst = gen_reg_rtx (outer);

      /* Make life a bit easier for combine.  */
      /* If the first element of the vector is the low part
         of the destination mode, use a paradoxical subreg to
         initialize the destination.  */
      if (start < finish)
        {
          inner = GET_MODE (tmps[start]);
          bytepos = subreg_lowpart_offset (inner, outer);
          if (INTVAL (XEXP (XVECEXP (src, 0, start), 1)) == bytepos)
            {
              temp = simplify_gen_subreg (outer, tmps[start],
                                          inner, 0);
              if (temp)
                {
                  emit_move_insn (dst, temp);
                  done = true;
                  start++;
                }
            }
        }

      /* If the first element wasn't the low part, try the last.  */
      if (!done
          && start < finish - 1)
        {
          inner = GET_MODE (tmps[finish - 1]);
          bytepos = subreg_lowpart_offset (inner, outer);
          if (INTVAL (XEXP (XVECEXP (src, 0, finish - 1), 1)) == bytepos)
            {
              temp = simplify_gen_subreg (outer, tmps[finish - 1],
                                          inner, 0);
              if (temp)
                {
                  emit_move_insn (dst, temp);
                  done = true;
                  finish--;
                }
            }
        }

      /* Otherwise, simply initialize the result to zero.  */
      if (!done)
        emit_move_insn (dst, CONST0_RTX (outer));
    }

  /* Process the pieces.  */
  for (i = start; i < finish; i++)
    {
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (src, 0, i), 1));
      enum machine_mode mode = GET_MODE (tmps[i]);
      unsigned int bytelen = GET_MODE_SIZE (mode);
      unsigned int adj_bytelen = bytelen;
      rtx dest = dst;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
        adj_bytelen = ssize - bytepos;

      if (GET_CODE (dst) == CONCAT)
        {
          if (bytepos + adj_bytelen
              <= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
            dest = XEXP (dst, 0);
          else if (bytepos >= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
            {
              bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
              dest = XEXP (dst, 1);
            }
          else
            {
              enum machine_mode dest_mode = GET_MODE (dest);
              enum machine_mode tmp_mode = GET_MODE (tmps[i]);

              gcc_assert (bytepos == 0 && XVECLEN (src, 0));

              if (GET_MODE_ALIGNMENT (dest_mode)
                  >= GET_MODE_ALIGNMENT (tmp_mode))
                {
                  dest = assign_stack_temp (dest_mode,
                                            GET_MODE_SIZE (dest_mode),
                                            0);
                  emit_move_insn (adjust_address (dest,
                                                  tmp_mode,
                                                  bytepos),
                                  tmps[i]);
                  dst = dest;
                }
              else
                {
                  dest = assign_stack_temp (tmp_mode,
                                            GET_MODE_SIZE (tmp_mode),
                                            0);
                  emit_move_insn (dest, tmps[i]);
                  dst = adjust_address (dest, dest_mode, bytepos);
                }
              break;
            }
        }

      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
        {
          /* store_bit_field always takes its value from the lsb.
             Move the fragment to the lsb if it's not already there.  */
          if (
#ifdef BLOCK_REG_PADDING
              BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
              == (BYTES_BIG_ENDIAN ? upward : downward)
#else
              BYTES_BIG_ENDIAN
#endif
              )
            {
              int shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
              tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
                                      shift, tmps[i], 0);
            }
          bytelen = adj_bytelen;
        }

      /* Optimize the access just a bit.  */
      if (MEM_P (dest)
          && (! SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (dest))
              || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
          && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
          && bytelen == GET_MODE_SIZE (mode))
        emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
      else
        store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
                         0, 0, mode, tmps[i]);
    }

  /* Copy from the pseudo into the (probable) hard reg.  */
  if (orig_dst != dst)
    emit_move_insn (orig_dst, dst);
}

/* Generate code to copy a BLKmode object of TYPE out of a
   set of registers starting with SRCREG into TGTBLK.  If TGTBLK
   is null, a stack temporary is created.  TGTBLK is returned.

   The purpose of this routine is to handle functions that return
   BLKmode structures in registers.  Some machines (the PA for example)
   want to return all small structures in registers regardless of the
   structure's alignment.  */

rtx
copy_blkmode_from_reg (rtx tgtblk, rtx srcreg, tree type)
{
  unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
  rtx src = NULL, dst = NULL;
  unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
  unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
  enum machine_mode copy_mode;

  if (tgtblk == 0)
    {
      tgtblk = assign_temp (build_qualified_type (type,
                                                  (TYPE_QUALS (type)
                                                   | TYPE_QUAL_CONST)),
                            0, 1, 1);
      preserve_temp_slots (tgtblk);
    }

  /* This code assumes srcreg is at least a full word.  If it isn't, copy it
     into a new pseudo which is a full word.  */

  if (GET_MODE (srcreg) != BLKmode
      && GET_MODE_SIZE (GET_MODE (srcreg)) < UNITS_PER_WORD)
    srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));

  /* If the structure doesn't take up a whole number of words, see whether
     SRCREG is padded on the left or on the right.  If it's on the left,
     set PADDING_CORRECTION to the number of bits to skip.

     In most ABIs, the structure will be returned at the least end of
     the register, which translates to right padding on little-endian
     targets and left padding on big-endian targets.  The opposite
     holds if the structure is returned at the most significant
     end of the register.  */
  if (bytes % UNITS_PER_WORD != 0
      && (targetm.calls.return_in_msb (type)
          ? !BYTES_BIG_ENDIAN
          : BYTES_BIG_ENDIAN))
    padding_correction
      = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));

  /* Copy the structure BITSIZE bits at a time.  If the target lives in
     memory, take care of not reading/writing past its end by selecting
     a copy mode suited to BITSIZE.  This should always be possible given
     how it is computed.

     We could probably emit more efficient code for machines which do not use
     strict alignment, but it doesn't seem worth the effort at the current
     time.  */

  copy_mode = word_mode;
  if (MEM_P (tgtblk))
    {
      enum machine_mode mem_mode = mode_for_size (bitsize, MODE_INT, 1);
      if (mem_mode != BLKmode)
        copy_mode = mem_mode;
    }

  for (bitpos = 0, xbitpos = padding_correction;
       bitpos < bytes * BITS_PER_UNIT;
       bitpos += bitsize, xbitpos += bitsize)
    {
      /* We need a new source operand each time xbitpos is on a
         word boundary and when xbitpos == padding_correction
         (the first time through).  */
      if (xbitpos % BITS_PER_WORD == 0
          || xbitpos == padding_correction)
        src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD,
                                     GET_MODE (srcreg));

      /* We need a new destination operand each time bitpos is on
         a word boundary.  */
      if (bitpos % BITS_PER_WORD == 0)
        dst = operand_subword (tgtblk, bitpos / BITS_PER_WORD, 1, BLKmode);

      /* Use xbitpos for the source extraction (right justified) and
         bitpos for the destination store (left justified).  */
      store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
                       extract_bit_field (src, bitsize,
                                          xbitpos % BITS_PER_WORD, 1, false,
                                          NULL_RTX, copy_mode, copy_mode));
    }

  return tgtblk;
}

/* Copy BLKmode value SRC into a register of mode MODE.  Return the
   register if it contains any data, otherwise return null.

   This is used on targets that return BLKmode values in registers.  */

rtx
copy_blkmode_to_reg (enum machine_mode mode, tree src)
{
  int i, n_regs;
  unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
  unsigned int bitsize;
  rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
  enum machine_mode dst_mode;

  gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);

  x = expand_normal (src);

  bytes = int_size_in_bytes (TREE_TYPE (src));
  if (bytes == 0)
    return NULL_RTX;

  /* If the structure doesn't take up a whole number of words, see
     whether the register value should be padded on the left or on
     the right.  Set PADDING_CORRECTION to the number of padding
     bits needed on the left side.

     In most ABIs, the structure will be returned at the least end of
     the register, which translates to right padding on little-endian
     targets and left padding on big-endian targets.  The opposite
     holds if the structure is returned at the most significant
     end of the register.  */
  if (bytes % UNITS_PER_WORD != 0
      && (targetm.calls.return_in_msb (TREE_TYPE (src))
          ? !BYTES_BIG_ENDIAN
          : BYTES_BIG_ENDIAN))
    padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
                                           * BITS_PER_UNIT));

  n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
  dst_words = XALLOCAVEC (rtx, n_regs);
  bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);

  /* Copy the structure BITSIZE bits at a time.  */
  for (bitpos = 0, xbitpos = padding_correction;
       bitpos < bytes * BITS_PER_UNIT;
       bitpos += bitsize, xbitpos += bitsize)
    {
      /* We need a new destination pseudo each time xbitpos is
         on a word boundary and when xbitpos == padding_correction
         (the first time through).  */
      if (xbitpos % BITS_PER_WORD == 0
          || xbitpos == padding_correction)
        {
          /* Generate an appropriate register.  */
          dst_word = gen_reg_rtx (word_mode);
          dst_words[xbitpos / BITS_PER_WORD] = dst_word;

          /* Clear the destination before we move anything into it.  */
          emit_move_insn (dst_word, CONST0_RTX (word_mode));
        }

      /* We need a new source operand each time bitpos is on a word
         boundary.  */
      if (bitpos % BITS_PER_WORD == 0)
        src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);

      /* Use bitpos for the source extraction (left justified) and
         xbitpos for the destination store (right justified).  */
      store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
                       0, 0, word_mode,
                       extract_bit_field (src_word, bitsize,
                                          bitpos % BITS_PER_WORD, 1, false,
                                          NULL_RTX, word_mode, word_mode));
    }

  if (mode == BLKmode)
    {
      /* Find the smallest integer mode large enough to hold the
         entire structure.  */
      for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
           mode != VOIDmode;
           mode = GET_MODE_WIDER_MODE (mode))
        /* Have we found a large enough mode?  */
        if (GET_MODE_SIZE (mode) >= bytes)
          break;

      /* A suitable mode should have been found.  */
      gcc_assert (mode != VOIDmode);
    }

  if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
    dst_mode = word_mode;
  else
    dst_mode = mode;
  dst = gen_reg_rtx (dst_mode);

  for (i = 0; i < n_regs; i++)
    emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);

  if (mode != dst_mode)
    dst = gen_lowpart (mode, dst);

  return dst;
}

/* Add a USE expression for REG to the (possibly empty) list pointed
   to by CALL_FUSAGE.  REG must denote a hard register.  */

void
use_reg_mode (rtx *call_fusage, rtx reg, enum machine_mode mode)
{
  gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);

  *call_fusage
    = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
}

/* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
   starting at REGNO.  All of these registers must be hard registers.  */

void
use_regs (rtx *call_fusage, int regno, int nregs)
{
  int i;

  gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);

  for (i = 0; i < nregs; i++)
    use_reg (call_fusage, regno_reg_rtx[regno + i]);
}

/* Add USE expressions to *CALL_FUSAGE for each REG contained in the
   PARALLEL REGS.  This is for calls that pass values in multiple
   non-contiguous locations.  The Irix 6 ABI has examples of this.  */

void
use_group_regs (rtx *call_fusage, rtx regs)
{
  int i;

  for (i = 0; i < XVECLEN (regs, 0); i++)
    {
      rtx reg = XEXP (XVECEXP (regs, 0, i), 0);

      /* A NULL entry means the parameter goes both on the stack and in
         registers.  This can also be a MEM for targets that pass values
         partially on the stack and partially in registers.  */
      if (reg != 0 && REG_P (reg))
        use_reg (call_fusage, reg);
    }
}

/* Return the defining gimple statement for SSA_NAME NAME if it is an
   assigment and the code of the expresion on the RHS is CODE.  Return
   NULL otherwise.  */

static gimple
get_def_for_expr (tree name, enum tree_code code)
{
  gimple def_stmt;

  if (TREE_CODE (name) != SSA_NAME)
    return NULL;

  def_stmt = get_gimple_for_ssa_name (name);
  if (!def_stmt
      || gimple_assign_rhs_code (def_stmt) != code)
    return NULL;

  return def_stmt;
}
\f

/* Determine whether the LEN bytes generated by CONSTFUN can be
   stored to memory using several move instructions.  CONSTFUNDATA is
   a pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.  MEMSETP is true if this is
   a memset operation and false if it's a copy of a constant string.
   Return nonzero if a call to store_by_pieces should succeed.  */

int
can_store_by_pieces (unsigned HOST_WIDE_INT len,
                     rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode),
                     void *constfundata, unsigned int align, bool memsetp)
{
  unsigned HOST_WIDE_INT l;
  unsigned int max_size;
  HOST_WIDE_INT offset = 0;
  enum machine_mode mode;
  enum insn_code icode;
  int reverse;
  /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it.  */
  rtx cst ATTRIBUTE_UNUSED;

  if (len == 0)
    return 1;

  if (! (memsetp
         ? SET_BY_PIECES_P (len, align)
         : STORE_BY_PIECES_P (len, align)))
    return 0;

  align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);

  /* We would first store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  for (reverse = 0;
       reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
       reverse++)
    {
      l = len;
      max_size = STORE_MAX_PIECES + 1;
      while (max_size > 1)
        {
          mode = widest_int_mode_for_size (max_size);

          if (mode == VOIDmode)
            break;

          icode = optab_handler (mov_optab, mode);
          if (icode != CODE_FOR_nothing
              && align >= GET_MODE_ALIGNMENT (mode))
            {
              unsigned int size = GET_MODE_SIZE (mode);

              while (l >= size)
                {
                  if (reverse)
                    offset -= size;

                  cst = (*constfun) (constfundata, offset, mode);
                  if (!targetm.legitimate_constant_p (mode, cst))
                    return 0;

                  if (!reverse)
                    offset += size;

                  l -= size;
                }
            }

          max_size = GET_MODE_SIZE (mode);
        }

      /* The code above should have handled everything.  */
      gcc_assert (!l);
    }

  return 1;
}

/* Generate several move instructions to store LEN bytes generated by
   CONSTFUN to block TO.  (A MEM rtx with BLKmode).  CONSTFUNDATA is a
   pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.  MEMSETP is true if this is
   a memset operation and false if it's a copy of a constant string.
   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
                 rtx (*constfun) (void *, HOST_WIDE_INT, enum machine_mode),
                 void *constfundata, unsigned int align, bool memsetp, int endp)
{
  enum machine_mode to_addr_mode
    = targetm.addr_space.address_mode (MEM_ADDR_SPACE (to));
  struct store_by_pieces_d data;

  if (len == 0)
    {
      gcc_assert (endp != 2);
      return to;
    }

  gcc_assert (memsetp
              ? SET_BY_PIECES_P (len, align)
              : STORE_BY_PIECES_P (len, align));
  data.constfun = constfun;
  data.constfundata = constfundata;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
        {
          if (endp == 2)
            {
              if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
                emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
              else
                data.to_addr = copy_to_mode_reg (to_addr_mode,
                                                 plus_constant (data.to_addr,
                                                                -1));
            }
          to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
                                           data.offset);
        }
      else
        {
          if (endp == 2)
            --data.offset;
          to1 = adjust_address (data.to, QImode, data.offset);
        }
      return to1;
    }
  else
    return data.to;
}

/* Generate several move instructions to clear LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
{
  struct store_by_pieces_d data;

  if (len == 0)
    return;

  data.constfun = clear_by_pieces_1;
  data.constfundata = NULL;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
}

/* Callback routine for clear_by_pieces.
   Return const0_rtx unconditionally.  */

static rtx
clear_by_pieces_1 (void *data ATTRIBUTE_UNUSED,
                   HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
                   enum machine_mode mode ATTRIBUTE_UNUSED)
{
  return const0_rtx;
}

/* Subroutine of clear_by_pieces and store_by_pieces.
   Generate several move instructions to store LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
store_by_pieces_1 (struct store_by_pieces_d *data ATTRIBUTE_UNUSED,
                   unsigned int align ATTRIBUTE_UNUSED)
{
  enum machine_mode to_addr_mode
    = targetm.addr_space.address_mode (MEM_ADDR_SPACE (data->to));
  rtx to_addr = XEXP (data->to, 0);
  unsigned int max_size = STORE_MAX_PIECES + 1;
  enum insn_code icode;

  data->offset = 0;
  data->to_addr = to_addr;
  data->autinc_to
    = (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
       || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);

  data->explicit_inc_to = 0;
  data->reverse
    = (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
  if (data->reverse)
    data->offset = data->len;

  /* If storing requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!data->autinc_to
      && move_by_pieces_ninsns (data->len, align, max_size) > 2)
    {
      /* Determine the main mode we'll be using.
         MODE might not be used depending on the definitions of the
         USE_* macros below.  */
      enum machine_mode mode ATTRIBUTE_UNUSED
        = widest_int_mode_for_size (max_size);

      if (USE_STORE_PRE_DECREMENT (mode) && data->reverse && ! data->autinc_to)
        {
          data->to_addr = copy_to_mode_reg (to_addr_mode,
                                            plus_constant (to_addr, data->len));
          data->autinc_to = 1;
          data->explicit_inc_to = -1;
        }

      if (USE_STORE_POST_INCREMENT (mode) && ! data->reverse
          && ! data->autinc_to)
        {
          data->to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
          data->autinc_to = 1;
          data->explicit_inc_to = 1;
        }

      if ( !data->autinc_to && CONSTANT_P (to_addr))
        data->to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
    }

  align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);

  /* First store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1)
    {
      enum machine_mode mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
        break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
        store_by_pieces_2 (GEN_FCN (icode), mode, data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data->len);
}

/* Subroutine of store_by_pieces_1.  Store as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
store_by_pieces_2 (rtx (*genfun) (rtx, ...), enum machine_mode mode,
                   struct store_by_pieces_d *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1, cst;

  while (data->len >= size)
    {
      if (data->reverse)
        data->offset -= size;

      if (data->autinc_to)
        to1 = adjust_automodify_address (data->to, mode, data->to_addr,
                                         data->offset);
      else
        to1 = adjust_address (data->to, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
        emit_insn (gen_add2_insn (data->to_addr,
                                  GEN_INT (-(HOST_WIDE_INT) size)));

      cst = (*data->constfun) (data->constfundata, data->offset, mode);
      emit_insn ((*genfun) (to1, cst));

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
        emit_insn (gen_add2_insn (data->to_addr, GEN_INT (size)));

      if (! data->reverse)
        data->offset += size;

      data->len -= size;
    }
}
\f
/* Write zeros through the storage of OBJECT.  If OBJECT has BLKmode, SIZE is
   its length in bytes.  */

rtx
clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
                     unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  enum machine_mode mode = GET_MODE (object);
  unsigned int align;

  gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);

  /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
     just move a zero.  Otherwise, do this a piece at a time.  */
  if (mode != BLKmode
      && CONST_INT_P (size)
      && INTVAL (size) == (HOST_WIDE_INT) GET_MODE_SIZE (mode))
    {
      rtx zero = CONST0_RTX (mode);
      if (zero != NULL)
        {
          emit_move_insn (object, zero);
          return NULL;
        }

      if (COMPLEX_MODE_P (mode))
        {
          zero = CONST0_RTX (GET_MODE_INNER (mode));
          if (zero != NULL)
            {
              write_complex_part (object, zero, 0);
              write_complex_part (object, zero, 1);
              return NULL;
            }
        }
    }

  if (size == const0_rtx)
    return NULL;

  align = MEM_ALIGN (object);

  if (CONST_INT_P (size)
      && CLEAR_BY_PIECES_P (INTVAL (size), align))
    clear_by_pieces (object, INTVAL (size), align);
  else if (set_storage_via_setmem (object, size, const0_rtx, align,
                                   expected_align, expected_size))
    ;
  else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
    return set_storage_via_libcall (object, size, const0_rtx,
                                    method == BLOCK_OP_TAILCALL);
  else
    gcc_unreachable ();

  return NULL;
}

rtx
clear_storage (rtx object, rtx size, enum block_op_methods method)
{
  return clear_storage_hints (object, size, method, 0, -1);
}


/* A subroutine of clear_storage.  Expand a call to memset.
   Return the return value of memset, 0 otherwise.  */

rtx
set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
{
  tree call_expr, fn, object_tree, size_tree, val_tree;
  enum machine_mode size_mode;
  rtx retval;

  /* Emit code to copy OBJECT and SIZE into new pseudos.  We can then
     place those into new pseudos into a VAR_DECL and use them later.  */

  object = copy_to_mode_reg (Pmode, XEXP (object, 0));

  size_mode = TYPE_MODE (sizetype);
  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memset in this context.  This could be a user call to memset and
     the user may wish to examine the return value from memset.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  object_tree = make_tree (ptr_type_node, object);
  if (!CONST_INT_P (val))
    val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
  size_tree = make_tree (sizetype, size);
  val_tree = make_tree (integer_type_node, val);

  fn = clear_storage_libcall_fn (true);
  call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of set_storage_via_libcall.  Create the tree node
   for the function we use for block clears.  The first time FOR_CALL
   is true, we call assemble_external.  */

tree block_clear_fn;

void
init_block_clear_fn (const char *asmspec)
{
  if (!block_clear_fn)
    {
      tree fn, args;

      fn = get_identifier ("memset");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
                                       integer_type_node, sizetype,
                                       NULL_TREE);

      fn = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      block_clear_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_clear_fn, asmspec);
}

static tree
clear_storage_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_clear_fn)
    init_block_clear_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_clear_fn);
      assemble_external (block_clear_fn);
    }

  return block_clear_fn;
}
\f
/* Expand a setmem pattern; return true if successful.  */

bool
set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
                        unsigned int expected_align, HOST_WIDE_INT expected_size)
{
  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  enum machine_mode mode;

  if (expected_align < align)
    expected_align = align;

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = direct_optab_handler (setmem_optab, mode);

      if (code != CODE_FOR_nothing
          /* We don't need MODE to be narrower than
             BITS_PER_HOST_WIDE_INT here because if SIZE is less than
             the mode mask, as it is returned by the macro, it will
             definitely be less than the actual mode mask.  */
          && ((CONST_INT_P (size)
               && ((unsigned HOST_WIDE_INT) INTVAL (size)
                   <= (GET_MODE_MASK (mode) >> 1)))
              || GET_MODE_BITSIZE (mode) >= BITS_PER_WORD))
        {
          struct expand_operand ops[6];
          unsigned int nops;

          nops = insn_data[(int) code].n_generator_args;
          gcc_assert (nops == 4 || nops == 6);

          create_fixed_operand (&ops[0], object);
          /* The check above guarantees that this size conversion is valid.  */
          create_convert_operand_to (&ops[1], size, mode, true);
          create_convert_operand_from (&ops[2], val, byte_mode, true);
          create_integer_operand (&ops[3], align / BITS_PER_UNIT);
          if (nops == 6)
            {
              create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
              create_integer_operand (&ops[5], expected_size);
            }
          if (maybe_expand_insn (code, nops, ops))
            return true;
        }
    }

  return false;
}

\f
/* Write to one of the components of the complex value CPLX.  Write VAL to
   the real part if IMAG_P is false, and the imaginary part if its true.  */

static void
write_complex_part (rtx cplx, rtx val, bool imag_p)
{
  enum machine_mode cmode;
  enum machine_mode imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    {
      emit_move_insn (XEXP (cplx, imag_p), val);
      return;
    }

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    {
      emit_move_insn (adjust_address_nv (cplx, imode,
                                         imag_p ? GET_MODE_SIZE (imode) : 0),
                      val);
      return;
    }

  /* If the sub-object is at least word sized, then we know that subregging
     will work.  This special case is important, since store_bit_field
     wants to operate on integer modes, and there's rarely an OImode to
     correspond to TCmode.  */
  if (ibitsize >= BITS_PER_WORD
      /* For hard regs we have exact predicates.  Assume we can split
         the original object if it spans an even number of hard regs.
         This special case is important for SCmode on 64-bit platforms
         where the natural size of floating-point regs is 32-bit.  */
      || (REG_P (cplx)
          && REGNO (cplx) < FIRST_PSEUDO_REGISTER
          && hard_regno_nregs[REGNO (cplx)][cmode] % 2 == 0))
    {
      rtx part = simplify_gen_subreg (imode, cplx, cmode,
                                      imag_p ? GET_MODE_SIZE (imode) : 0);
      if (part)
        {
          emit_move_insn (part, val);
          return;
        }
      else
        /* simplify_gen_subreg may fail for sub-word MEMs.  */
        gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
    }

  store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val);
}

/* Extract one of the components of the complex value CPLX.  Extract the
   real part if IMAG_P is false, and the imaginary part if it's true.  */

static rtx
read_complex_part (rtx cplx, bool imag_p)
{
  enum machine_mode cmode, imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    return XEXP (cplx, imag_p);

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* Special case reads from complex constants that got spilled to memory.  */
  if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
    {
      tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
      if (decl && TREE_CODE (decl) == COMPLEX_CST)
        {
          tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
          if (CONSTANT_CLASS_P (part))
            return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
        }
    }

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    return adjust_address_nv (cplx, imode,
                              imag_p ? GET_MODE_SIZE (imode) : 0);

  /* If the sub-object is at least word sized, then we know that subregging
     will work.  This special case is important, since extract_bit_field
     wants to operate on integer modes, and there's rarely an OImode to
     correspond to TCmode.  */
  if (ibitsize >= BITS_PER_WORD
      /* For hard regs we have exact predicates.  Assume we can split
         the original object if it spans an even number of hard regs.
         This special case is important for SCmode on 64-bit platforms
         where the natural size of floating-point regs is 32-bit.  */
      || (REG_P (cplx)
          && REGNO (cplx) < FIRST_PSEUDO_REGISTER
          && hard_regno_nregs[REGNO (cplx)][cmode] % 2 == 0))
    {
      rtx ret = simplify_gen_subreg (imode, cplx, cmode,
                                     imag_p ? GET_MODE_SIZE (imode) : 0);
      if (ret)
        return ret;
      else
        /* simplify_gen_subreg may fail for sub-word MEMs.  */
        gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
    }

  return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
                            true, false, NULL_RTX, imode, imode);
}
\f
/* A subroutine of emit_move_insn_1.  Yet another lowpart generator.
   NEW_MODE and OLD_MODE are the same size.  Return NULL if X cannot be
   represented in NEW_MODE.  If FORCE is true, this will never happen, as
   we'll force-create a SUBREG if needed.  */

static rtx
emit_move_change_mode (enum machine_mode new_mode,
                       enum machine_mode old_mode, rtx x, bool force)
{
  rtx ret;

  if (push_operand (x, GET_MODE (x)))
    {
      ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
      MEM_COPY_ATTRIBUTES (ret, x);
    }
  else if (MEM_P (x))
    {
      /* We don't have to worry about changing the address since the
         size in bytes is supposed to be the same.  */
      if (reload_in_progress)
        {
          /* Copy the MEM to change the mode and move any
             substitutions from the old MEM to the new one.  */
          ret = adjust_address_nv (x, new_mode, 0);
          copy_replacements (x, ret);
        }
      else
        ret = adjust_address (x, new_mode, 0);
    }
  else
    {
      /* Note that we do want simplify_subreg's behavior of validating
         that the new mode is ok for a hard register.  If we were to use
         simplify_gen_subreg, we would create the subreg, but would
         probably run into the target not being able to implement it.  */
      /* Except, of course, when FORCE is true, when this is exactly what
         we want.  Which is needed for CCmodes on some targets.  */
      if (force)
        ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
      else
        ret = simplify_subreg (new_mode, x, old_mode, 0);
    }

  return ret;
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X using
   an integer mode of the same size as MODE.  Returns the instruction
   emitted, or NULL if such a move could not be generated.  */

static rtx
emit_move_via_integer (enum machine_mode mode, rtx x, rtx y, bool force)
{
  enum machine_mode imode;
  enum insn_code code;

  /* There must exist a mode of the exact size we require.  */
  imode = int_mode_for_mode (mode);
  if (imode == BLKmode)
    return NULL_RTX;

  /* The target must support moves in this mode.  */
  code = optab_handler (mov_optab, imode);
  if (code == CODE_FOR_nothing)
    return NULL_RTX;

  x = emit_move_change_mode (imode, mode, x, force);
  if (x == NULL_RTX)
    return NULL_RTX;
  y = emit_move_change_mode (imode, mode, y, force);
  if (y == NULL_RTX)
    return NULL_RTX;
  return emit_insn (GEN_FCN (code) (x, y));
}

/* A subroutine of emit_move_insn_1.  X is a push_operand in MODE.
   Return an equivalent MEM that does not use an auto-increment.  */

static rtx
emit_move_resolve_push (enum machine_mode mode, rtx x)
{
  enum rtx_code code = GET_CODE (XEXP (x, 0));
  HOST_WIDE_INT adjust;
  rtx temp;

  adjust = GET_MODE_SIZE (mode);
#ifdef PUSH_ROUNDING
  adjust = PUSH_ROUNDING (adjust);
#endif
  if (code == PRE_DEC || code == POST_DEC)
    adjust = -adjust;
  else if (code == PRE_MODIFY || code == POST_MODIFY)
    {
      rtx expr = XEXP (XEXP (x, 0), 1);
      HOST_WIDE_INT val;

      gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
      gcc_assert (CONST_INT_P (XEXP (expr, 1)));
      val = INTVAL (XEXP (expr, 1));
      if (GET_CODE (expr) == MINUS)
        val = -val;
      gcc_assert (adjust == val || adjust == -val);
      adjust = val;
    }

  /* Do not use anti_adjust_stack, since we don't want to update
     stack_pointer_delta.  */
  temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
                              GEN_INT (adjust), stack_pointer_rtx,
                              0, OPTAB_LIB_WIDEN);
  if (temp != stack_pointer_rtx)
    emit_move_insn (stack_pointer_rtx, temp);

  switch (code)
    {
    case PRE_INC:
    case PRE_DEC:
    case PRE_MODIFY:
      temp = stack_pointer_rtx;
      break;
    case POST_INC:
    case POST_DEC:
    case POST_MODIFY:
      temp = plus_constant (stack_pointer_rtx, -adjust);
      break;
    default:
      gcc_unreachable ();
    }

  return replace_equiv_address (x, temp);
}

/* A subroutine of emit_move_complex.  Generate a move from Y into X.
   X is known to satisfy push_operand, and MODE is known to be complex.
   Returns the last instruction emitted.  */

rtx
emit_move_complex_push (enum machine_mode mode, rtx x, rtx y)
{
  enum machine_mode submode = GET_MODE_INNER (mode);
  bool imag_first;

#ifdef PUSH_ROUNDING
  unsigned int submodesize = GET_MODE_SIZE (submode);

  /* In case we output to the stack, but the size is smaller than the
     machine can push exactly, we need to use move instructions.  */
  if (PUSH_ROUNDING (submodesize) != submodesize)
    {
      x = emit_move_resolve_push (mode, x);
      return emit_move_insn (x, y);
    }
#endif

  /* Note that the real part always precedes the imag part in memory
     regardless of machine's endianness.  */
  switch (GET_CODE (XEXP (x, 0)))
    {
    case PRE_DEC:
    case POST_DEC:
      imag_first = true;
      break;
    case PRE_INC:
    case POST_INC:
      imag_first = false;
      break;
    default:
      gcc_unreachable ();
    }

  emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
                  read_complex_part (y, imag_first));
  return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
                         read_complex_part (y, !imag_first));
}

/* A subroutine of emit_move_complex.  Perform the move from Y to X
   via two moves of the parts.  Returns the last instruction emitted.  */

rtx
emit_move_complex_parts (rtx x, rtx y)
{
  /* Show the output dies here.  This is necessary for SUBREGs
     of pseudos since we cannot track their lifetimes correctly;
     hard regs shouldn't appear here except as return values.  */
  if (!reload_completed && !reload_in_progress
      && REG_P (x) && !reg_overlap_mentioned_p (x, y))
    emit_clobber (x);

  write_complex_part (x, read_complex_part (y, false), false);
  write_complex_part (x, read_complex_part (y, true), true);

  return get_last_insn ();
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is known to be complex.  Returns the last instruction emitted.  */

static rtx
emit_move_complex (enum machine_mode mode, rtx x, rtx y)
{
  bool try_int;

  /* Need to take special care for pushes, to maintain proper ordering
     of the data, and possibly extra padding.  */
  if (push_operand (x, mode))
    return emit_move_complex_push (mode, x, y);

  /* See if we can coerce the target into moving both values at once.  */

  /* Move floating point as parts.  */
  if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
      && optab_handler (mov_optab, GET_MODE_INNER (mode)) != CODE_FOR_nothing)
    try_int = false;
  /* Not possible if the values are inherently not adjacent.  */
  else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
    try_int = false;
  /* Is possible if both are registers (or subregs of registers).  */
  else if (register_operand (x, mode) && register_operand (y, mode))
    try_int = true;
  /* If one of the operands is a memory, and alignment constraints
     are friendly enough, we may be able to do combined memory operations.
     We do not attempt this if Y is a constant because that combination is
     usually better with the by-parts thing below.  */
  else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
           && (!STRICT_ALIGNMENT
               || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
    try_int = true;
  else
    try_int = false;

  if (try_int)
    {
      rtx ret;

      /* For memory to memory moves, optimal behavior can be had with the
         existing block move logic.  */
      if (MEM_P (x) && MEM_P (y))
        {
          emit_block_move (x, y, GEN_INT (GET_MODE_SIZE (mode)),
                           BLOCK_OP_NO_LIBCALL);
          return get_last_insn ();
        }

      ret = emit_move_via_integer (mode, x, y, true);
      if (ret)
        return ret;
    }

  return emit_move_complex_parts (x, y);
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is known to be MODE_CC.  Returns the last instruction emitted.  */

static rtx
emit_move_ccmode (enum machine_mode mode, rtx x, rtx y)
{
  rtx ret;

  /* Assume all MODE_CC modes are equivalent; if we have movcc, use it.  */
  if (mode != CCmode)
    {
      enum insn_code code = optab_handler (mov_optab, CCmode);
      if (code != CODE_FOR_nothing)
        {
          x = emit_move_change_mode (CCmode, mode, x, true);
          y = emit_move_change_mode (CCmode, mode, y, true);
          return emit_insn (GEN_FCN (code) (x, y));
        }
    }

  /* Otherwise, find the MODE_INT mode of the same width.  */
  ret = emit_move_via_integer (mode, x, y, false);
  gcc_assert (ret != NULL);
  return ret;
}

/* Return true if word I of OP lies entirely in the
   undefined bits of a paradoxical subreg.  */

static bool
undefined_operand_subword_p (const_rtx op, int i)
{
  enum machine_mode innermode, innermostmode;
  int offset;
  if (GET_CODE (op) != SUBREG)
    return false;
  innermode = GET_MODE (op);
  innermostmode = GET_MODE (SUBREG_REG (op));
  offset = i * UNITS_PER_WORD + SUBREG_BYTE (op);
  /* The SUBREG_BYTE represents offset, as if the value were stored in
     memory, except for a paradoxical subreg where we define
     SUBREG_BYTE to be 0; undo this exception as in
     simplify_subreg.  */
  if (SUBREG_BYTE (op) == 0
      && GET_MODE_SIZE (innermostmode) < GET_MODE_SIZE (innermode))
    {
      int difference = (GET_MODE_SIZE (innermostmode) - GET_MODE_SIZE (innermode));
      if (WORDS_BIG_ENDIAN)
        offset += (difference / UNITS_PER_WORD) * UNITS_PER_WORD;
      if (BYTES_BIG_ENDIAN)
        offset += difference % UNITS_PER_WORD;
    }
  if (offset >= GET_MODE_SIZE (innermostmode)
      || offset <= -GET_MODE_SIZE (word_mode))
    return true;
  return false;
}

/* A subroutine of emit_move_insn_1.  Generate a move from Y into X.
   MODE is any multi-word or full-word mode that lacks a move_insn
   pattern.  Note that you will get better code if you define such
   patterns, even if they must turn into multiple assembler instructions.  */

static rtx
emit_move_multi_word (enum machine_mode mode, rtx x, rtx y)
{
  rtx last_insn = 0;
  rtx seq, inner;
  bool need_clobber;
  int i;

  gcc_assert (GET_MODE_SIZE (mode) >= UNITS_PER_WORD);

  /* If X is a push on the stack, do the push now and replace
     X with a reference to the stack pointer.  */
  if (push_operand (x, mode))
    x = emit_move_resolve_push (mode, x);

  /* If we are in reload, see if either operand is a MEM whose address
     is scheduled for replacement.  */
  if (reload_in_progress && MEM_P (x)
      && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
    x = replace_equiv_address_nv (x, inner);
  if (reload_in_progress && MEM_P (y)
      && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
    y = replace_equiv_address_nv (y, inner);

  start_sequence ();

  need_clobber = false;
  for (i = 0;
       i < (GET_MODE_SIZE (mode) + (UNITS_PER_WORD - 1)) / UNITS_PER_WORD;
       i++)
    {
      rtx xpart = operand_subword (x, i, 1, mode);
      rtx ypart;

      /* Do not generate code for a move if it would come entirely
         from the undefined bits of a paradoxical subreg.  */
      if (undefined_operand_subword_p (y, i))
        continue;

      ypart = operand_subword (y, i, 1, mode);

      /* If we can't get a part of Y, put Y into memory if it is a
         constant.  Otherwise, force it into a register.  Then we must
         be able to get a part of Y.  */
      if (ypart == 0 && CONSTANT_P (y))
        {
          y = use_anchored_address (force_const_mem (mode, y));
          ypart = operand_subword (y, i, 1, mode);
        }
      else if (ypart == 0)
        ypart = operand_subword_force (y, i, mode);

      gcc_assert (xpart && ypart);

      need_clobber |= (GET_CODE (xpart) == SUBREG);

      last_insn = emit_move_insn (xpart, ypart);
    }

  seq = get_insns ();
  end_sequence ();

  /* Show the output dies here.  This is necessary for SUBREGs
     of pseudos since we cannot track their lifetimes correctly;
     hard regs shouldn't appear here except as return values.
     We never want to emit such a clobber after reload.  */
  if (x != y
      && ! (reload_in_progress || reload_completed)
      && need_clobber != 0)
    emit_clobber (x);

  emit_insn (seq);

  return last_insn;
}

/* Low level part of emit_move_insn.
   Called just like emit_move_insn, but assumes X and Y
   are basically valid.  */

rtx
emit_move_insn_1 (rtx x, rtx y)
{
  enum machine_mode mode = GET_MODE (x);
  enum insn_code code;

  gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);

  code = optab_handler (mov_optab, mode);
  if (code != CODE_FOR_nothing)
    return emit_insn (GEN_FCN (code) (x, y));

  /* Expand complex moves by moving real part and imag part.  */
  if (COMPLEX_MODE_P (mode))
    return emit_move_complex (mode, x, y);

  if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT
      || ALL_FIXED_POINT_MODE_P (mode))
    {
      rtx result = emit_move_via_integer (mode, x, y, true);

      /* If we can't find an integer mode, use multi words.  */
      if (result)
        return result;
      else
        return emit_move_multi_word (mode, x, y);
    }

  if (GET_MODE_CLASS (mode) == MODE_CC)
    return emit_move_ccmode (mode, x, y);

  /* Try using a move pattern for the corresponding integer mode.  This is
     only safe when simplify_subreg can convert MODE constants into integer
     constants.  At present, it can only do this reliably if the value
     fits within a HOST_WIDE_INT.  */
  if (!CONSTANT_P (y) || GET_MODE_BITSIZE (mode) <= HOST_BITS_PER_WIDE_INT)
    {
      rtx ret = emit_move_via_integer (mode, x, y, false);
      if (ret)
        return ret;
    }

  return emit_move_multi_word (mode, x, y);
}

/* Generate code to copy Y into X.
   Both Y and X must have the same mode, except that
   Y can be a constant with VOIDmode.
   This mode cannot be BLKmode; use emit_block_move for that.

   Return the last instruction emitted.  */

rtx
emit_move_insn (rtx x, rtx y)
{
  enum machine_mode mode = GET_MODE (x);
  rtx y_cst = NULL_RTX;
  rtx last_insn, set;

  gcc_assert (mode != BLKmode
              && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));

  if (CONSTANT_P (y))
    {
      if (optimize
          && SCALAR_FLOAT_MODE_P (GET_MODE (x))
          && (last_insn = compress_float_constant (x, y)))
        return last_insn;

      y_cst = y;

      if (!targetm.legitimate_constant_p (mode, y))
        {
          y = force_const_mem (mode, y);

          /* If the target's cannot_force_const_mem prevented the spill,
             assume that the target's move expanders will also take care
             of the non-legitimate constant.  */
          if (!y)
            y = y_cst;
          else
            y = use_anchored_address (y);
        }
    }

  /* If X or Y are memory references, verify that their addresses are valid
     for the machine.  */
  if (MEM_P (x)
      && (! memory_address_addr_space_p (GET_MODE (x), XEXP (x, 0),
                                         MEM_ADDR_SPACE (x))
          && ! push_operand (x, GET_MODE (x))))
    x = validize_mem (x);

  if (MEM_P (y)
      && ! memory_address_addr_space_p (GET_MODE (y), XEXP (y, 0),
                                        MEM_ADDR_SPACE (y)))
    y = validize_mem (y);

  gcc_assert (mode != BLKmode);

  last_insn = emit_move_insn_1 (x, y);

  if (y_cst && REG_P (x)
      && (set = single_set (last_insn)) != NULL_RTX
      && SET_DEST (set) == x
      && ! rtx_equal_p (y_cst, SET_SRC (set)))
    set_unique_reg_note (last_insn, REG_EQUAL, copy_rtx (y_cst));

  return last_insn;
}

/* If Y is representable exactly in a narrower mode, and the target can
   perform the extension directly from constant or memory, then emit the
   move as an extension.  */

static rtx
compress_float_constant (rtx x, rtx y)
{
  enum machine_mode dstmode = GET_MODE (x);
  enum machine_mode orig_srcmode = GET_MODE (y);
  enum machine_mode srcmode;
  REAL_VALUE_TYPE r;
  int oldcost, newcost;
  bool speed = optimize_insn_for_speed_p ();

  REAL_VALUE_FROM_CONST_DOUBLE (r, y);

  if (targetm.legitimate_constant_p (dstmode, y))
    oldcost = set_src_cost (y, speed);
  else
    oldcost = set_src_cost (force_const_mem (dstmode, y), speed);

  for (srcmode = GET_CLASS_NARROWEST_MODE (GET_MODE_CLASS (orig_srcmode));
       srcmode != orig_srcmode;
       srcmode = GET_MODE_WIDER_MODE (srcmode))
    {
      enum insn_code ic;
      rtx trunc_y, last_insn;

      /* Skip if the target can't extend this way.  */
      ic = can_extend_p (dstmode, srcmode, 0);
      if (ic == CODE_FOR_nothing)
        continue;

      /* Skip if the narrowed value isn't exact.  */
      if (! exact_real_truncate (srcmode, &r))
        continue;

      trunc_y = CONST_DOUBLE_FROM_REAL_VALUE (r, srcmode);

      if (targetm.legitimate_constant_p (srcmode, trunc_y))
        {
          /* Skip if the target needs extra instructions to perform
             the extension.  */
          if (!insn_operand_matches (ic, 1, trunc_y))
            continue;
          /* This is valid, but may not be cheaper than the original. */
          newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
                                  speed);
          if (oldcost < newcost)
            continue;
        }
      else if (float_extend_from_mem[dstmode][srcmode])
        {
          trunc_y = force_const_mem (srcmode, trunc_y);
          /* This is valid, but may not be cheaper than the original. */
          newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
                                  speed);
          if (oldcost < newcost)
            continue;
          trunc_y = validize_mem (trunc_y);
        }
      else
        continue;

      /* For CSE's benefit, force the compressed constant pool entry
         into a new pseudo.  This constant may be used in different modes,
         and if not, combine will put things back together for us.  */
      trunc_y = force_reg (srcmode, trunc_y);
      emit_unop_insn (ic, x, trunc_y, UNKNOWN);
      last_insn = get_last_insn ();

      if (REG_P (x))
        set_unique_reg_note (last_insn, REG_EQUAL, y);

      return last_insn;
    }

  return NULL_RTX;
}
\f
/* Pushing data onto the stack.  */

/* Push a block of length SIZE (perhaps variable)
   and return an rtx to address the beginning of the block.
   The value may be virtual_outgoing_args_rtx.

   EXTRA is the number of bytes of padding to push in addition to SIZE.
   BELOW nonzero means this padding comes at low addresses;
   otherwise, the padding comes at high addresses.  */

rtx
push_block (rtx size, int extra, int below)
{
  rtx temp;

  size = convert_modes (Pmode, ptr_mode, size, 1);
  if (CONSTANT_P (size))
    anti_adjust_stack (plus_constant (size, extra));
  else if (REG_P (size) && extra == 0)
    anti_adjust_stack (size);
  else
    {
      temp = copy_to_mode_reg (Pmode, size);
      if (extra != 0)
        temp = expand_binop (Pmode, add_optab, temp, GEN_INT (extra),
                             temp, 0, OPTAB_LIB_WIDEN);
      anti_adjust_stack (temp);
    }

#ifndef STACK_GROWS_DOWNWARD
  if (0)
#else
  if (1)
#endif
    {
      temp = virtual_outgoing_args_rtx;
      if (extra != 0 && below)
        temp = plus_constant (temp, extra);
    }
  else
    {
      if (CONST_INT_P (size))
        temp = plus_constant (virtual_outgoing_args_rtx,
                              -INTVAL (size) - (below ? 0 : extra));
      else if (extra != 0 && !below)
        temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
                             negate_rtx (Pmode, plus_constant (size, extra)));
      else
        temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
                             negate_rtx (Pmode, size));
    }

  return memory_address (GET_CLASS_NARROWEST_MODE (MODE_INT), temp);
}

/* A utility routine that returns the base of an auto-inc memory, or NULL.  */

static rtx
mem_autoinc_base (rtx mem)
{
  if (MEM_P (mem))
    {
      rtx addr = XEXP (mem, 0);
      if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
        return XEXP (addr, 0);
    }
  return NULL;
}

/* A utility routine used here, in reload, and in try_split.  The insns
   after PREV up to and including LAST are known to adjust the stack,
   with a final value of END_ARGS_SIZE.  Iterate backward from LAST
   placing notes as appropriate.  PREV may be NULL, indicating the
   entire insn sequence prior to LAST should be scanned.

   The set of allowed stack pointer modifications is small:
     (1) One or more auto-inc style memory references (aka pushes),
     (2) One or more addition/subtraction with the SP as destination,
     (3) A single move insn with the SP as destination,
     (4) A call_pop insn,
     (5) Noreturn call insns if !ACCUMULATE_OUTGOING_ARGS.

   Insns in the sequence that do not modify the SP are ignored,
   except for noreturn calls.

   The return value is the amount of adjustment that can be trivially
   verified, via immediate operand or auto-inc.  If the adjustment
   cannot be trivially extracted, the return value is INT_MIN.  */

HOST_WIDE_INT
find_args_size_adjust (rtx insn)
{
  rtx dest, set, pat;
  int i;

  pat = PATTERN (insn);
  set = NULL;

  /* Look for a call_pop pattern.  */
  if (CALL_P (insn))
    {
      /* We have to allow non-call_pop patterns for the case
         of emit_single_push_insn of a TLS address.  */
      if (GET_CODE (pat) != PARALLEL)
        return 0;

      /* All call_pop have a stack pointer adjust in the parallel.
         The call itself is always first, and the stack adjust is
         usually last, so search from the end.  */
      for (i = XVECLEN (pat, 0) - 1; i > 0; --i)
        {
          set = XVECEXP (pat, 0, i);
          if (GET_CODE (set) != SET)
            continue;
          dest = SET_DEST (set);
          if (dest == stack_pointer_rtx)
            break;
        }
      /* We'd better have found the stack pointer adjust.  */
      if (i == 0)
        return 0;
      /* Fall through to process the extracted SET and DEST
         as if it was a standalone insn.  */
    }
  else if (GET_CODE (pat) == SET)
    set = pat;
  else if ((set = single_set (insn)) != NULL)
    ;
  else if (GET_CODE (pat) == PARALLEL)
    {
      /* ??? Some older ports use a parallel with a stack adjust
         and a store for a PUSH_ROUNDING pattern, rather than a
         PRE/POST_MODIFY rtx.  Don't force them to update yet...  */
      /* ??? See h8300 and m68k, pushqi1.  */
      for (i = XVECLEN (pat, 0) - 1; i >= 0; --i)
        {
          set = XVECEXP (pat, 0, i);
          if (GET_CODE (set) != SET)
            continue;
          dest = SET_DEST (set);
          if (dest == stack_pointer_rtx)
            break;

          /* We do not expect an auto-inc of the sp in the parallel.  */
          gcc_checking_assert (mem_autoinc_base (dest) != stack_pointer_rtx);
          gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
                               != stack_pointer_rtx);
        }
      if (i < 0)
        return 0;
    }
  else
    return 0;

  dest = SET_DEST (set);

  /* Look for direct modifications of the stack pointer.  */
  if (REG_P (dest) && REGNO (dest) == STACK_POINTER_REGNUM)
    {
      /* Look for a trivial adjustment, otherwise assume nothing.  */
      /* Note that the SPU restore_stack_block pattern refers to
         the stack pointer in V4SImode.  Consider that non-trivial.  */
      if (SCALAR_INT_MODE_P (GET_MODE (dest))
          && GET_CODE (SET_SRC (set)) == PLUS
          && XEXP (SET_SRC (set), 0) == stack_pointer_rtx
          && CONST_INT_P (XEXP (SET_SRC (set), 1)))
        return INTVAL (XEXP (SET_SRC (set), 1));
      /* ??? Reload can generate no-op moves, which will be cleaned
         up later.  Recognize it and continue searching.  */
      else if (rtx_equal_p (dest, SET_SRC (set)))
        return 0;
      else
        return HOST_WIDE_INT_MIN;
    }
  else
    {
      rtx mem, addr;

      /* Otherwise only think about autoinc patterns.  */
      if (mem_autoinc_base (dest) == stack_pointer_rtx)
        {
          mem = dest;
          gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
                               != stack_pointer_rtx);
        }
      else if (mem_autoinc_base (SET_SRC (set)) == stack_pointer_rtx)
        mem = SET_SRC (set);
      else
        return 0;

      addr = XEXP (mem, 0);
      switch (GET_CODE (addr))
        {
        case PRE_INC:
        case POST_INC:
          return GET_MODE_SIZE (GET_MODE (mem));
        case PRE_DEC:
        case POST_DEC:
          return -GET_MODE_SIZE (GET_MODE (mem));
        case PRE_MODIFY:
        case POST_MODIFY:
          addr = XEXP (addr, 1);
          gcc_assert (GET_CODE (addr) == PLUS);
          gcc_assert (XEXP (addr, 0) == stack_pointer_rtx);
          gcc_assert (CONST_INT_P (XEXP (addr, 1)));
          return INTVAL (XEXP (addr, 1));
        default:
          gcc_unreachable ();
        }
    }
}

int
fixup_args_size_notes (rtx prev, rtx last, int end_args_size)
{
  int args_size = end_args_size;
  bool saw_unknown = false;
  rtx insn;

  for (insn = last; insn != prev; insn = PREV_INSN (insn))
    {
      HOST_WIDE_INT this_delta;

      if (!NONDEBUG_INSN_P (insn))
        continue;

      this_delta = find_args_size_adjust (insn);
      if (this_delta == 0)
        {
          if (!CALL_P (insn)
              || ACCUMULATE_OUTGOING_ARGS
              || find_reg_note (insn, REG_NORETURN, NULL_RTX) == NULL_RTX)
            continue;
        }

      gcc_assert (!saw_unknown);
      if (this_delta == HOST_WIDE_INT_MIN)
        saw_unknown = true;

      add_reg_note (insn, REG_ARGS_SIZE, GEN_INT (args_size));
#ifdef STACK_GROWS_DOWNWARD
      this_delta = -this_delta;
#endif
      args_size -= this_delta;
    }

  return saw_unknown ? INT_MIN : args_size;
}

#ifdef PUSH_ROUNDING
/* Emit single push insn.  */

static void
emit_single_push_insn_1 (enum machine_mode mode, rtx x, tree type)
{
  rtx dest_addr;
  unsigned rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
  rtx dest;
  enum insn_code icode;

  stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
  /* If there is push pattern, use it.  Otherwise try old way of throwing
     MEM representing push operation to move expander.  */
  icode = optab_handler (push_optab, mode);
  if (icode != CODE_FOR_nothing)
    {
      struct expand_operand ops[1];

      create_input_operand (&ops[0], x, mode);
      if (maybe_expand_insn (icode, 1, ops))
        return;
    }
  if (GET_MODE_SIZE (mode) == rounded_size)
    dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
  /* If we are to pad downward, adjust the stack pointer first and
     then store X into the stack location using an offset.  This is
     because emit_move_insn does not know how to pad; it does not have
     access to type.  */
  else if (FUNCTION_ARG_PADDING (mode, type) == downward)
    {
      unsigned padding_size = rounded_size - GET_MODE_SIZE (mode);
      HOST_WIDE_INT offset;

      emit_move_insn (stack_pointer_rtx,
                      expand_binop (Pmode,
#ifdef STACK_GROWS_DOWNWARD
                                    sub_optab,
#else
                                    add_optab,
#endif
                                    stack_pointer_rtx,
                                    GEN_INT (rounded_size),
                                    NULL_RTX, 0, OPTAB_LIB_WIDEN));

      offset = (HOST_WIDE_INT) padding_size;
#ifdef STACK_GROWS_DOWNWARD
      if (STACK_PUSH_CODE == POST_DEC)
        /* We have already decremented the stack pointer, so get the
           previous value.  */
        offset += (HOST_WIDE_INT) rounded_size;
#else
      if (STACK_PUSH_CODE == POST_INC)
        /* We have already incremented the stack pointer, so get the
           previous value.  */
        offset -= (HOST_WIDE_INT) rounded_size;
#endif
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (offset));
    }
  else
    {
#ifdef STACK_GROWS_DOWNWARD
      /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC.  */
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
                                GEN_INT (-(HOST_WIDE_INT) rounded_size));
#else
      /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC.  */
      dest_addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx,
                                GEN_INT (rounded_size));
#endif
      dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
    }

  dest = gen_rtx_MEM (mode, dest_addr);

  if (type != 0)
    {
      set_mem_attributes (dest, type, 1);

      if (flag_optimize_sibling_calls)
        /* Function incoming arguments may overlap with sibling call
           outgoing arguments and we cannot allow reordering of reads
           from function arguments with stores to outgoing arguments
           of sibling calls.  */
        set_mem_alias_set (dest, 0);
    }
  emit_move_insn (dest, x);
}

/* Emit and annotate a single push insn.  */

static void
emit_single_push_insn (enum machine_mode mode, rtx x, tree type)
{
  int delta, old_delta = stack_pointer_delta;
  rtx prev = get_last_insn ();
  rtx last;

  emit_single_push_insn_1 (mode, x, type);

  last = get_last_insn ();

  /* Notice the common case where we emitted exactly one insn.  */
  if (PREV_INSN (last) == prev)
    {
      add_reg_note (last, REG_ARGS_SIZE, GEN_INT (stack_pointer_delta));
      return;
    }

  delta = fixup_args_size_notes (prev, last, stack_pointer_delta);
  gcc_assert (delta == INT_MIN || delta == old_delta);
}
#endif

/* Generate code to push X onto the stack, assuming it has mode MODE and
   type TYPE.
   MODE is redundant except when X is a CONST_INT (since they don't
   carry mode info).
   SIZE is an rtx for the size of data to be copied (in bytes),
   needed only if X is BLKmode.

   ALIGN (in bits) is maximum alignment we can assume.

   If PARTIAL and REG are both nonzero, then copy that many of the first
   bytes of X into registers starting with REG, and push the rest of X.
   The amount of space pushed is decreased by PARTIAL bytes.
   REG must be a hard register in this case.
   If REG is zero but PARTIAL is not, take any all others actions for an
   argument partially in registers, but do not actually load any
   registers.

   EXTRA is the amount in bytes of extra space to leave next to this arg.
   This is ignored if an argument block has already been allocated.

   On a machine that lacks real push insns, ARGS_ADDR is the address of
   the bottom of the argument block for this call.  We use indexing off there
   to store the arg.  On machines with push insns, ARGS_ADDR is 0 when a
   argument block has not been preallocated.

   ARGS_SO_FAR is the size of args previously pushed for this call.

   REG_PARM_STACK_SPACE is nonzero if functions require stack space
   for arguments passed in registers.  If nonzero, it will be the number
   of bytes required.  */

void
emit_push_insn (rtx x, enum machine_mode mode, tree type, rtx size,
                unsigned int align, int partial, rtx reg, int extra,
                rtx args_addr, rtx args_so_far, int reg_parm_stack_space,
                rtx alignment_pad)
{
  rtx xinner;
  enum direction stack_direction
#ifdef STACK_GROWS_DOWNWARD
    = downward;
#else
    = upward;
#endif

  /* Decide where to pad the argument: `downward' for below,
     `upward' for above, or `none' for don't pad it.
     Default is below for small data on big-endian machines; else above.  */
  enum direction where_pad = FUNCTION_ARG_PADDING (mode, type);

  /* Invert direction if stack is post-decrement.
     FIXME: why?  */
  if (STACK_PUSH_CODE == POST_DEC)
    if (where_pad != none)
      where_pad = (where_pad == downward ? upward : downward);

  xinner = x;

  if (mode == BLKmode
      || (STRICT_ALIGNMENT && align < GET_MODE_ALIGNMENT (mode)))
    {
      /* Copy a block into the stack, entirely or partially.  */

      rtx temp;
      int used;
      int offset;
      int skip;

      offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
      used = partial - offset;

      if (mode != BLKmode)
        {
          /* A value is to be stored in an insufficiently aligned
             stack slot; copy via a suitably aligned slot if
             necessary.  */
          size = GEN_INT (GET_MODE_SIZE (mode));
          if (!MEM_P (xinner))
            {
              temp = assign_temp (type, 0, 1, 1);
              emit_move_insn (temp, xinner);
              xinner = temp;
            }
        }

      gcc_assert (size);

      /* USED is now the # of bytes we need not copy to the stack
         because registers will take care of them.  */

      if (partial != 0)
        xinner = adjust_address (xinner, BLKmode, used);

      /* If the partial register-part of the arg counts in its stack size,
         skip the part of stack space corresponding to the registers.
         Otherwise, start copying to the beginning of the stack space,
         by setting SKIP to 0.  */
      skip = (reg_parm_stack_space == 0) ? 0 : used;

#ifdef PUSH_ROUNDING
      /* Do it with several push insns if that doesn't take lots of insns
         and if there is no difficulty with push insns that skip bytes
         on the stack for alignment purposes.  */
      if (args_addr == 0
          && PUSH_ARGS
          && CONST_INT_P (size)
          && skip == 0
          && MEM_ALIGN (xinner) >= align
          && (MOVE_BY_PIECES_P ((unsigned) INTVAL (size) - used, align))
          /* Here we avoid the case of a structure whose weak alignment
             forces many pushes of a small amount of data,
             and such small pushes do rounding that causes trouble.  */
          && ((! SLOW_UNALIGNED_ACCESS (word_mode, align))
              || align >= BIGGEST_ALIGNMENT
              || (PUSH_ROUNDING (align / BITS_PER_UNIT)
                  == (align / BITS_PER_UNIT)))
          && (HOST_WIDE_INT) PUSH_ROUNDING (INTVAL (size)) == INTVAL (size))
        {
          /* Push padding now if padding above and stack grows down,
             or if padding below and stack grows up.
             But if space already allocated, this has already been done.  */
          if (extra && args_addr == 0
              && where_pad != none && where_pad != stack_direction)
            anti_adjust_stack (GEN_INT (extra));

          move_by_pieces (NULL, xinner, INTVAL (size) - used, align, 0);
        }
      else
#endif /* PUSH_ROUNDING  */
        {
          rtx target;

          /* Otherwise make space on the stack and copy the data
             to the address of that space.  */

          /* Deduct words put into registers from the size we must copy.  */
          if (partial != 0)
            {
              if (CONST_INT_P (size))
                size = GEN_INT (INTVAL (size) - used);
              else
                size = expand_binop (GET_MODE (size), sub_optab, size,
                                     GEN_INT (used), NULL_RTX, 0,
                                     OPTAB_LIB_WIDEN);
            }

          /* Get the address of the stack space.
             In this case, we do not deal with EXTRA separately.
             A single stack adjust will do.  */
          if (! args_addr)
            {
              temp = push_block (size, extra, where_pad == downward);
              extra = 0;
            }
          else if (CONST_INT_P (args_so_far))
            temp = memory_address (BLKmode,
                                   plus_constant (args_addr,
                                                  skip + INTVAL (args_so_far)));
          else
            temp = memory_address (BLKmode,
                                   plus_constant (gen_rtx_PLUS (Pmode,
                                                                args_addr,
                                                                args_so_far),
                                                  skip));

          if (!ACCUMULATE_OUTGOING_ARGS)
            {
              /* If the source is referenced relative to the stack pointer,
                 copy it to another register to stabilize it.  We do not need
                 to do this if we know that we won't be changing sp.  */

              if (reg_mentioned_p (virtual_stack_dynamic_rtx, temp)
                  || reg_mentioned_p (virtual_outgoing_args_rtx, temp))
                temp = copy_to_reg (temp);
            }

          target = gen_rtx_MEM (BLKmode, temp);

          /* We do *not* set_mem_attributes here, because incoming arguments
             may overlap with sibling call outgoing arguments and we cannot
             allow reordering of reads from function arguments with stores
             to outgoing arguments of sibling calls.  We do, however, want
             to record the alignment of the stack slot.  */
          /* ALIGN may well be better aligned than TYPE, e.g. due to
             PARM_BOUNDARY.  Assume the caller isn't lying.  */
          set_mem_align (target, align);

          emit_block_move (target, xinner, size, BLOCK_OP_CALL_PARM);
        }
    }
  else if (partial > 0)
    {
      /* Scalar partly in registers.  */

      int size = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
      int i;
      int not_stack;
      /* # bytes of start of argument
         that we must make space for but need not store.  */
      int offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
      int args_offset = INTVAL (args_so_far);
      int skip;

      /* Push padding now if padding above and stack grows down,
         or if padding below and stack grows up.
         But if space already allocated, this has already been done.  */
      if (extra && args_addr == 0
          && where_pad != none && where_pad != stack_direction)
        anti_adjust_stack (GEN_INT (extra));

      /* If we make space by pushing it, we might as well push
         the real data.  Otherwise, we can leave OFFSET nonzero
         and leave the space uninitialized.  */
      if (args_addr == 0)
        offset = 0;

      /* Now NOT_STACK gets the number of words that we don't need to
         allocate on the stack.  Convert OFFSET to words too.  */
      not_stack = (partial - offset) / UNITS_PER_WORD;
      offset /= UNITS_PER_WORD;

      /* If the partial register-part of the arg counts in its stack size,
         skip the part of stack space corresponding to the registers.
         Otherwise, start copying to the beginning of the stack space,
         by setting SKIP to 0.  */
      skip = (reg_parm_stack_space == 0) ? 0 : not_stack;

      if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
        x = validize_mem (force_const_mem (mode, x));

      /* If X is a hard register in a non-integer mode, copy it into a pseudo;
         SUBREGs of such registers are not allowed.  */
      if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER
           && GET_MODE_CLASS (GET_MODE (x)) != MODE_INT))
        x = copy_to_reg (x);

      /* Loop over all the words allocated on the stack for this arg.  */
      /* We can do it by words, because any scalar bigger than a word
         has a size a multiple of a word.  */
#ifndef PUSH_ARGS_REVERSED
      for (i = not_stack; i < size; i++)
#else
      for (i = size - 1; i >= not_stack; i--)
#endif
        if (i >= not_stack + offset)
          emit_push_insn (operand_subword_force (x, i, mode),
                          word_mode, NULL_TREE, NULL_RTX, align, 0, NULL_RTX,
                          0, args_addr,
                          GEN_INT (args_offset + ((i - not_stack + skip)
                                                  * UNITS_PER_WORD)),
                          reg_parm_stack_space, alignment_pad);
    }
  else
    {
      rtx addr;
      rtx dest;

      /* Push padding now if padding above and stack grows down,
         or if padding below and stack grows up.
         But if space already allocated, this has already been done.  */
      if (extra && args_addr == 0
          && where_pad != none && where_pad != stack_direction)
        anti_adjust_stack (GEN_INT (extra));

#ifdef PUSH_ROUNDING
      if (args_addr == 0 && PUSH_ARGS)
        emit_single_push_insn (mode, x, type);
      else
#endif
        {
          if (CONST_INT_P (args_so_far))
            addr
              = memory_address (mode,
                                plus_constant (args_addr,
                                               INTVAL (args_so_far)));
          else
            addr = memory_address (mode, gen_rtx_PLUS (Pmode, args_addr,
                                                       args_so_far));
          dest = gen_rtx_MEM (mode, addr);