gcc/expr.c

/* Convert tree expression to rtl instructions, for GNU compiler.
   Copyright (C) 1988-2015 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "machmode.h"
#include "rtl.h"
#include "hash-set.h"
#include "vec.h"
#include "double-int.h"
#include "input.h"
#include "alias.h"
#include "symtab.h"
#include "wide-int.h"
#include "inchash.h"
#include "tree.h"
#include "fold-const.h"
#include "stringpool.h"
#include "stor-layout.h"
#include "attribs.h"
#include "varasm.h"
#include "flags.h"
#include "regs.h"
#include "hard-reg-set.h"
#include "except.h"
#include "function.h"
#include "insn-config.h"
#include "insn-attr.h"
#include "hashtab.h"
#include "statistics.h"
#include "real.h"
#include "fixed-value.h"
#include "expmed.h"
#include "dojump.h"
#include "explow.h"
#include "calls.h"
#include "emit-rtl.h"
#include "stmt.h"
/* Include expr.h after insn-config.h so we get HAVE_conditional_move.  */
#include "expr.h"
#include "insn-codes.h"
#include "optabs.h"
#include "libfuncs.h"
#include "recog.h"
#include "reload.h"
#include "typeclass.h"
#include "toplev.h"
#include "langhooks.h"
#include "intl.h"
#include "tm_p.h"
#include "tree-iterator.h"
#include "predict.h"
#include "dominance.h"
#include "cfg.h"
#include "basic-block.h"
#include "tree-ssa-alias.h"
#include "internal-fn.h"
#include "gimple-expr.h"
#include "is-a.h"
#include "gimple.h"
#include "gimple-ssa.h"
#include "hash-map.h"
#include "plugin-api.h"
#include "ipa-ref.h"
#include "cgraph.h"
#include "tree-ssanames.h"
#include "target.h"
#include "common/common-target.h"
#include "timevar.h"
#include "df.h"
#include "diagnostic.h"
#include "tree-ssa-live.h"
#include "tree-outof-ssa.h"
#include "target-globals.h"
#include "params.h"
#include "tree-ssa-address.h"
#include "cfgexpand.h"
#include "builtins.h"
#include "tree-chkp.h"
#include "rtl-chkp.h"
#include "ccmp.h"

#ifndef STACK_PUSH_CODE
#ifdef STACK_GROWS_DOWNWARD
#define STACK_PUSH_CODE PRE_DEC
#else
#define STACK_PUSH_CODE PRE_INC
#endif
#endif


/* If this is nonzero, we do not bother generating VOLATILE
   around volatile memory references, and we are willing to
   output indirect addresses.  If cse is to follow, we reject
   indirect addresses so a useful potential cse is generated;
   if it is used only once, instruction combination will produce
   the same indirect address eventually.  */
int cse_not_expected;

/* This structure is used by move_by_pieces to describe the move to
   be performed.  */
struct move_by_pieces_d
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  rtx from;
  rtx from_addr;
  int autinc_from;
  int explicit_inc_from;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  int reverse;
};

/* This structure is used by store_by_pieces to describe the clear to
   be performed.  */

struct store_by_pieces_d
{
  rtx to;
  rtx to_addr;
  int autinc_to;
  int explicit_inc_to;
  unsigned HOST_WIDE_INT len;
  HOST_WIDE_INT offset;
  rtx (*constfun) (void *, HOST_WIDE_INT, machine_mode);
  void *constfundata;
  int reverse;
};

static void move_by_pieces_1 (insn_gen_fn, machine_mode,
			      struct move_by_pieces_d *);
static bool block_move_libcall_safe_for_call_parm (void);
static bool emit_block_move_via_movmem (rtx, rtx, rtx, unsigned, unsigned, HOST_WIDE_INT,
					unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
					unsigned HOST_WIDE_INT);
static tree emit_block_move_libcall_fn (int);
static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned);
static rtx clear_by_pieces_1 (void *, HOST_WIDE_INT, machine_mode);
static void clear_by_pieces (rtx, unsigned HOST_WIDE_INT, unsigned int);
static void store_by_pieces_1 (struct store_by_pieces_d *, unsigned int);
static void store_by_pieces_2 (insn_gen_fn, machine_mode,
			       struct store_by_pieces_d *);
static tree clear_storage_libcall_fn (int);
static rtx_insn *compress_float_constant (rtx, rtx);
static rtx get_subtarget (rtx);
static void store_constructor_field (rtx, unsigned HOST_WIDE_INT,
				     HOST_WIDE_INT, unsigned HOST_WIDE_INT,
				     unsigned HOST_WIDE_INT, machine_mode,
				     tree, int, alias_set_type);
static void store_constructor (tree, rtx, int, HOST_WIDE_INT);
static rtx store_field (rtx, HOST_WIDE_INT, HOST_WIDE_INT,
			unsigned HOST_WIDE_INT, unsigned HOST_WIDE_INT,
			machine_mode, tree, alias_set_type, bool);

static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);

static int is_aligning_offset (const_tree, const_tree);
static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
static rtx do_store_flag (sepops, rtx, machine_mode);
#ifdef PUSH_ROUNDING
static void emit_single_push_insn (machine_mode, rtx, tree);
#endif
static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx, int);
static rtx const_vector_from_tree (tree);
static tree tree_expr_size (const_tree);
static HOST_WIDE_INT int_expr_size (tree);


/* This is run to set up which modes can be used
   directly in memory and to initialize the block move optab.  It is run
   at the beginning of compilation and when the target is reinitialized.  */

void
init_expr_target (void)
{
  rtx insn, pat;
  machine_mode mode;
  int num_clobbers;
  rtx mem, mem1;
  rtx reg;

  /* Try indexing by frame ptr and try by stack ptr.
     It is known that on the Convex the stack ptr isn't a valid index.
     With luck, one or the other is valid on any machine.  */
  mem = gen_rtx_MEM (VOIDmode, stack_pointer_rtx);
  mem1 = gen_rtx_MEM (VOIDmode, frame_pointer_rtx);

  /* A scratch register we can modify in-place below to avoid
     useless RTL allocations.  */
  reg = gen_rtx_REG (VOIDmode, -1);

  insn = rtx_alloc (INSN);
  pat = gen_rtx_SET (VOIDmode, NULL_RTX, NULL_RTX);
  PATTERN (insn) = pat;

  for (mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
       mode = (machine_mode) ((int) mode + 1))
    {
      int regno;

      direct_load[(int) mode] = direct_store[(int) mode] = 0;
      PUT_MODE (mem, mode);
      PUT_MODE (mem1, mode);
      PUT_MODE (reg, mode);

      /* See if there is some register that can be used in this mode and
	 directly loaded or stored from memory.  */

      if (mode != VOIDmode && mode != BLKmode)
	for (regno = 0; regno < FIRST_PSEUDO_REGISTER
	     && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
	     regno++)
	  {
	    if (! HARD_REGNO_MODE_OK (regno, mode))
	      continue;

	    SET_REGNO (reg, regno);

	    SET_SRC (pat) = mem;
	    SET_DEST (pat) = reg;
	    if (recog (pat, insn, &num_clobbers) >= 0)
	      direct_load[(int) mode] = 1;

	    SET_SRC (pat) = mem1;
	    SET_DEST (pat) = reg;
	    if (recog (pat, insn, &num_clobbers) >= 0)
	      direct_load[(int) mode] = 1;

	    SET_SRC (pat) = reg;
	    SET_DEST (pat) = mem;
	    if (recog (pat, insn, &num_clobbers) >= 0)
	      direct_store[(int) mode] = 1;

	    SET_SRC (pat) = reg;
	    SET_DEST (pat) = mem1;
	    if (recog (pat, insn, &num_clobbers) >= 0)
	      direct_store[(int) mode] = 1;
	  }
    }

  mem = gen_rtx_MEM (VOIDmode, gen_rtx_raw_REG (Pmode, 10000));

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      machine_mode srcmode;
      for (srcmode = GET_CLASS_NARROWEST_MODE (MODE_FLOAT); srcmode != mode;
	   srcmode = GET_MODE_WIDER_MODE (srcmode))
	{
	  enum insn_code ic;

	  ic = can_extend_p (mode, srcmode, 0);
	  if (ic == CODE_FOR_nothing)
	    continue;

	  PUT_MODE (mem, srcmode);

	  if (insn_operand_matches (ic, 1, mem))
	    float_extend_from_mem[mode][srcmode] = true;
	}
    }
}

/* This is run at the start of compiling a function.  */

void
init_expr (void)
{
  memset (&crtl->expr, 0, sizeof (crtl->expr));
}

/* Copy data from FROM to TO, where the machine modes are not the same.
   Both modes may be integer, or both may be floating, or both may be
   fixed-point.
   UNSIGNEDP should be nonzero if FROM is an unsigned type.
   This causes zero-extension instead of sign-extension.  */

void
convert_move (rtx to, rtx from, int unsignedp)
{
  machine_mode to_mode = GET_MODE (to);
  machine_mode from_mode = GET_MODE (from);
  int to_real = SCALAR_FLOAT_MODE_P (to_mode);
  int from_real = SCALAR_FLOAT_MODE_P (from_mode);
  enum insn_code code;
  rtx libcall;

  /* rtx code for making an equivalent value.  */
  enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
			      : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));


  gcc_assert (to_real == from_real);
  gcc_assert (to_mode != BLKmode);
  gcc_assert (from_mode != BLKmode);

  /* If the source and destination are already the same, then there's
     nothing to do.  */
  if (to == from)
    return;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  We don't handle such SUBREGs as
     TO here.  */

  if (GET_CODE (from) == SUBREG && SUBREG_PROMOTED_VAR_P (from)
      && (GET_MODE_PRECISION (GET_MODE (SUBREG_REG (from)))
	  >= GET_MODE_PRECISION (to_mode))
      && SUBREG_CHECK_PROMOTED_SIGN (from, unsignedp))
    from = gen_lowpart (to_mode, from), from_mode = to_mode;

  gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));

  if (to_mode == from_mode
      || (from_mode == VOIDmode && CONSTANT_P (from)))
    {
      emit_move_insn (to, from);
      return;
    }

  if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
    {
      gcc_assert (GET_MODE_BITSIZE (from_mode) == GET_MODE_BITSIZE (to_mode));

      if (VECTOR_MODE_P (to_mode))
	from = simplify_gen_subreg (to_mode, from, GET_MODE (from), 0);
      else
	to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);

      emit_move_insn (to, from);
      return;
    }

  if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
    {
      convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
      convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
      return;
    }

  if (to_real)
    {
      rtx value;
      rtx_insn *insns;
      convert_optab tab;

      gcc_assert ((GET_MODE_PRECISION (from_mode)
		   != GET_MODE_PRECISION (to_mode))
		  || (DECIMAL_FLOAT_MODE_P (from_mode)
		      != DECIMAL_FLOAT_MODE_P (to_mode)));

      if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
	/* Conversion between decimal float and binary float, same size.  */
	tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
      else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
	tab = sext_optab;
      else
	tab = trunc_optab;

      /* Try converting directly if the insn is supported.  */

      code = convert_optab_handler (tab, to_mode, from_mode);
      if (code != CODE_FOR_nothing)
	{
	  emit_unop_insn (code, to, from,
			  tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
	  return;
	}

      /* Otherwise use a libcall.  */
      libcall = convert_optab_libfunc (tab, to_mode, from_mode);

      /* Is this conversion implemented yet?  */
      gcc_assert (libcall);

      start_sequence ();
      value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
				       1, from, from_mode);
      insns = get_insns ();
      end_sequence ();
      emit_libcall_block (insns, to, value,
			  tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
								       from)
			  : gen_rtx_FLOAT_EXTEND (to_mode, from));
      return;
    }

  /* Handle pointer conversion.  */			/* SPEE 900220.  */
  /* If the target has a converter from FROM_MODE to TO_MODE, use it.  */
  {
    convert_optab ctab;

    if (GET_MODE_PRECISION (from_mode) > GET_MODE_PRECISION (to_mode))
      ctab = trunc_optab;
    else if (unsignedp)
      ctab = zext_optab;
    else
      ctab = sext_optab;

    if (convert_optab_handler (ctab, to_mode, from_mode)
	!= CODE_FOR_nothing)
      {
	emit_unop_insn (convert_optab_handler (ctab, to_mode, from_mode),
			to, from, UNKNOWN);
	return;
      }
  }

  /* Targets are expected to provide conversion insns between PxImode and
     xImode for all MODE_PARTIAL_INT modes they use, but no others.  */
  if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
    {
      machine_mode full_mode
	= smallest_mode_for_size (GET_MODE_BITSIZE (to_mode), MODE_INT);

      gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
		  != CODE_FOR_nothing);

      if (full_mode != from_mode)
	from = convert_to_mode (full_mode, from, unsignedp);
      emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
		      to, from, UNKNOWN);
      return;
    }
  if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
    {
      rtx new_from;
      machine_mode full_mode
	= smallest_mode_for_size (GET_MODE_BITSIZE (from_mode), MODE_INT);
      convert_optab ctab = unsignedp ? zext_optab : sext_optab;
      enum insn_code icode;

      icode = convert_optab_handler (ctab, full_mode, from_mode);
      gcc_assert (icode != CODE_FOR_nothing);

      if (to_mode == full_mode)
	{
	  emit_unop_insn (icode, to, from, UNKNOWN);
	  return;
	}

      new_from = gen_reg_rtx (full_mode);
      emit_unop_insn (icode, new_from, from, UNKNOWN);

      /* else proceed to integer conversions below.  */
      from_mode = full_mode;
      from = new_from;
    }

   /* Make sure both are fixed-point modes or both are not.  */
   gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
	       ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
   if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
    {
      /* If we widen from_mode to to_mode and they are in the same class,
	 we won't saturate the result.
	 Otherwise, always saturate the result to play safe.  */
      if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
	  && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
	expand_fixed_convert (to, from, 0, 0);
      else
	expand_fixed_convert (to, from, 0, 1);
      return;
    }

  /* Now both modes are integers.  */

  /* Handle expanding beyond a word.  */
  if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
      && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
    {
      rtx_insn *insns;
      rtx lowpart;
      rtx fill_value;
      rtx lowfrom;
      int i;
      machine_mode lowpart_mode;
      int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);

      /* Try converting directly if the insn is supported.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
	  != CODE_FOR_nothing)
	{
	  /* If FROM is a SUBREG, put it into a register.  Do this
	     so that we always generate the same set of insns for
	     better cse'ing; if an intermediate assignment occurred,
	     we won't be doing the operation directly on the SUBREG.  */
	  if (optimize > 0 && GET_CODE (from) == SUBREG)
	    from = force_reg (from_mode, from);
	  emit_unop_insn (code, to, from, equiv_code);
	  return;
	}
      /* Next, try converting via full word.  */
      else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
	       && ((code = can_extend_p (to_mode, word_mode, unsignedp))
		   != CODE_FOR_nothing))
	{
	  rtx word_to = gen_reg_rtx (word_mode);
	  if (REG_P (to))
	    {
	      if (reg_overlap_mentioned_p (to, from))
		from = force_reg (from_mode, from);
	      emit_clobber (to);
	    }
	  convert_move (word_to, from, unsignedp);
	  emit_unop_insn (code, to, word_to, equiv_code);
	  return;
	}

      /* No special multiword conversion insn; do it by hand.  */
      start_sequence ();

      /* Since we will turn this into a no conflict block, we must ensure the
         the source does not overlap the target so force it into an isolated
         register when maybe so.  Likewise for any MEM input, since the
         conversion sequence might require several references to it and we
         must ensure we're getting the same value every time.  */

      if (MEM_P (from) || reg_overlap_mentioned_p (to, from))
	from = force_reg (from_mode, from);

      /* Get a copy of FROM widened to a word, if necessary.  */
      if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
	lowpart_mode = word_mode;
      else
	lowpart_mode = from_mode;

      lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);

      lowpart = gen_lowpart (lowpart_mode, to);
      emit_move_insn (lowpart, lowfrom);

      /* Compute the value to put in each remaining word.  */
      if (unsignedp)
	fill_value = const0_rtx;
      else
	fill_value = emit_store_flag_force (gen_reg_rtx (word_mode),
					    LT, lowfrom, const0_rtx,
					    lowpart_mode, 0, -1);

      /* Fill the remaining words.  */
      for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
	{
	  int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
	  rtx subword = operand_subword (to, index, 1, to_mode);

	  gcc_assert (subword);

	  if (fill_value != subword)
	    emit_move_insn (subword, fill_value);
	}

      insns = get_insns ();
      end_sequence ();

      emit_insn (insns);
      return;
    }

  /* Truncating multi-word to a word or less.  */
  if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
      && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
    {
      if (!((MEM_P (from)
	     && ! MEM_VOLATILE_P (from)
	     && direct_load[(int) to_mode]
	     && ! mode_dependent_address_p (XEXP (from, 0),
					    MEM_ADDR_SPACE (from)))
	    || REG_P (from)
	    || GET_CODE (from) == SUBREG))
	from = force_reg (from_mode, from);
      convert_move (to, gen_lowpart (word_mode, from), 0);
      return;
    }

  /* Now follow all the conversions between integers
     no more than a word long.  */

  /* For truncation, usually we can just refer to FROM in a narrower mode.  */
  if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
      && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
    {
      if (!((MEM_P (from)
	     && ! MEM_VOLATILE_P (from)
	     && direct_load[(int) to_mode]
	     && ! mode_dependent_address_p (XEXP (from, 0),
					    MEM_ADDR_SPACE (from)))
	    || REG_P (from)
	    || GET_CODE (from) == SUBREG))
	from = force_reg (from_mode, from);
      if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
	  && ! HARD_REGNO_MODE_OK (REGNO (from), to_mode))
	from = copy_to_reg (from);
      emit_move_insn (to, gen_lowpart (to_mode, from));
      return;
    }

  /* Handle extension.  */
  if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
    {
      /* Convert directly if that works.  */
      if ((code = can_extend_p (to_mode, from_mode, unsignedp))
	  != CODE_FOR_nothing)
	{
	  emit_unop_insn (code, to, from, equiv_code);
	  return;
	}
      else
	{
	  machine_mode intermediate;
	  rtx tmp;
	  int shift_amount;

	  /* Search for a mode to convert via.  */
	  for (intermediate = from_mode; intermediate != VOIDmode;
	       intermediate = GET_MODE_WIDER_MODE (intermediate))
	    if (((can_extend_p (to_mode, intermediate, unsignedp)
		  != CODE_FOR_nothing)
		 || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
		     && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, intermediate)))
		&& (can_extend_p (intermediate, from_mode, unsignedp)
		    != CODE_FOR_nothing))
	      {
		convert_move (to, convert_to_mode (intermediate, from,
						   unsignedp), unsignedp);
		return;
	      }

	  /* No suitable intermediate mode.
	     Generate what we need with	shifts.  */
	  shift_amount = (GET_MODE_PRECISION (to_mode)
			  - GET_MODE_PRECISION (from_mode));
	  from = gen_lowpart (to_mode, force_reg (from_mode, from));
	  tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
			      to, unsignedp);
	  tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
			      to, unsignedp);
	  if (tmp != to)
	    emit_move_insn (to, tmp);
	  return;
	}
    }

  /* Support special truncate insns for certain modes.  */
  if (convert_optab_handler (trunc_optab, to_mode,
			     from_mode) != CODE_FOR_nothing)
    {
      emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
		      to, from, UNKNOWN);
      return;
    }

  /* Handle truncation of volatile memrefs, and so on;
     the things that couldn't be truncated directly,
     and for which there was no special instruction.

     ??? Code above formerly short-circuited this, for most integer
     mode pairs, with a force_reg in from_mode followed by a recursive
     call to this routine.  Appears always to have been wrong.  */
  if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
    {
      rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
      emit_move_insn (to, temp);
      return;
    }

  /* Mode combination is not recognized.  */
  gcc_unreachable ();
}

/* Return an rtx for a value that would result
   from converting X to mode MODE.
   Both X and MODE may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.
   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.  */

rtx
convert_to_mode (machine_mode mode, rtx x, int unsignedp)
{
  return convert_modes (mode, VOIDmode, x, unsignedp);
}

/* Return an rtx for a value that would result
   from converting X from mode OLDMODE to mode MODE.
   Both modes may be floating, or both integer.
   UNSIGNEDP is nonzero if X is an unsigned value.

   This can be done by referring to a part of X in place
   or by copying to a new temporary with conversion.

   You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode.  */

rtx
convert_modes (machine_mode mode, machine_mode oldmode, rtx x, int unsignedp)
{
  rtx temp;

  /* If FROM is a SUBREG that indicates that we have already done at least
     the required extension, strip it.  */

  if (GET_CODE (x) == SUBREG && SUBREG_PROMOTED_VAR_P (x)
      && GET_MODE_SIZE (GET_MODE (SUBREG_REG (x))) >= GET_MODE_SIZE (mode)
      && SUBREG_CHECK_PROMOTED_SIGN (x, unsignedp))
    x = gen_lowpart (mode, SUBREG_REG (x));

  if (GET_MODE (x) != VOIDmode)
    oldmode = GET_MODE (x);

  if (mode == oldmode)
    return x;

  if (CONST_SCALAR_INT_P (x) && GET_MODE_CLASS (mode) == MODE_INT)
    {
      /* If the caller did not tell us the old mode, then there is not
	 much to do with respect to canonicalization.  We have to
	 assume that all the bits are significant.  */
      if (GET_MODE_CLASS (oldmode) != MODE_INT)
	oldmode = MAX_MODE_INT;
      wide_int w = wide_int::from (std::make_pair (x, oldmode),
				   GET_MODE_PRECISION (mode),
				   unsignedp ? UNSIGNED : SIGNED);
      return immed_wide_int_const (w, mode);
    }

  /* We can do this with a gen_lowpart if both desired and current modes
     are integer, and this is either a constant integer, a register, or a
     non-volatile MEM. */
  if (GET_MODE_CLASS (mode) == MODE_INT
      && GET_MODE_CLASS (oldmode) == MODE_INT
      && GET_MODE_PRECISION (mode) <= GET_MODE_PRECISION (oldmode)
      && ((MEM_P (x) && !MEM_VOLATILE_P (x) && direct_load[(int) mode])
          || (REG_P (x)
              && (!HARD_REGISTER_P (x)
                  || HARD_REGNO_MODE_OK (REGNO (x), mode))
              && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (x)))))

   return gen_lowpart (mode, x);

  /* Converting from integer constant into mode is always equivalent to an
     subreg operation.  */
  if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
    {
      gcc_assert (GET_MODE_BITSIZE (mode) == GET_MODE_BITSIZE (oldmode));
      return simplify_gen_subreg (mode, x, oldmode, 0);
    }

  temp = gen_reg_rtx (mode);
  convert_move (temp, x, unsignedp);
  return temp;
}

/* Return the largest alignment we can use for doing a move (or store)
   of MAX_PIECES.  ALIGN is the largest alignment we could use.  */

static unsigned int
alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
{
  machine_mode tmode;

  tmode = mode_for_size (max_pieces * BITS_PER_UNIT, MODE_INT, 1);
  if (align >= GET_MODE_ALIGNMENT (tmode))
    align = GET_MODE_ALIGNMENT (tmode);
  else
    {
      machine_mode tmode, xmode;

      for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT), xmode = tmode;
	   tmode != VOIDmode;
	   xmode = tmode, tmode = GET_MODE_WIDER_MODE (tmode))
	if (GET_MODE_SIZE (tmode) > max_pieces
	    || SLOW_UNALIGNED_ACCESS (tmode, align))
	  break;

      align = MAX (align, GET_MODE_ALIGNMENT (xmode));
    }

  return align;
}

/* Return the widest integer mode no wider than SIZE.  If no such mode
   can be found, return VOIDmode.  */

static machine_mode
widest_int_mode_for_size (unsigned int size)
{
  machine_mode tmode, mode = VOIDmode;

  for (tmode = GET_CLASS_NARROWEST_MODE (MODE_INT);
       tmode != VOIDmode; tmode = GET_MODE_WIDER_MODE (tmode))
    if (GET_MODE_SIZE (tmode) < size)
      mode = tmode;

  return mode;
}

/* Determine whether the LEN bytes can be moved by using several move
   instructions.  Return nonzero if a call to move_by_pieces should
   succeed.  */

int
can_move_by_pieces (unsigned HOST_WIDE_INT len,
		    unsigned int align)
{
  return targetm.use_by_pieces_infrastructure_p (len, align, MOVE_BY_PIECES,
						 optimize_insn_for_speed_p ());
}

/* Generate several move instructions to copy LEN bytes from block FROM to
   block TO.  (These are MEM rtx's with BLKmode).

   If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
   used to push FROM to the stack.

   ALIGN is maximum stack alignment we can assume.

   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
		unsigned int align, int endp)
{
  struct move_by_pieces_d data;
  machine_mode to_addr_mode;
  machine_mode from_addr_mode = get_address_mode (from);
  rtx to_addr, from_addr = XEXP (from, 0);
  unsigned int max_size = MOVE_MAX_PIECES + 1;
  enum insn_code icode;

  align = MIN (to ? MEM_ALIGN (to) : align, MEM_ALIGN (from));

  data.offset = 0;
  data.from_addr = from_addr;
  if (to)
    {
      to_addr_mode = get_address_mode (to);
      to_addr = XEXP (to, 0);
      data.to = to;
      data.autinc_to
	= (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
	   || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);
      data.reverse
	= (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
    }
  else
    {
      to_addr_mode = VOIDmode;
      to_addr = NULL_RTX;
      data.to = NULL_RTX;
      data.autinc_to = 1;
#ifdef STACK_GROWS_DOWNWARD
      data.reverse = 1;
#else
      data.reverse = 0;
#endif
    }
  data.to_addr = to_addr;
  data.from = from;
  data.autinc_from
    = (GET_CODE (from_addr) == PRE_INC || GET_CODE (from_addr) == PRE_DEC
       || GET_CODE (from_addr) == POST_INC
       || GET_CODE (from_addr) == POST_DEC);

  data.explicit_inc_from = 0;
  data.explicit_inc_to = 0;
  if (data.reverse) data.offset = len;
  data.len = len;

  /* If copying requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!(data.autinc_from && data.autinc_to)
      && move_by_pieces_ninsns (len, align, max_size) > 2)
    {
      /* Find the mode of the largest move...
	 MODE might not be used depending on the definitions of the
	 USE_* macros below.  */
      machine_mode mode ATTRIBUTE_UNUSED
	= widest_int_mode_for_size (max_size);

      if (USE_LOAD_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_from)
	{
	  data.from_addr = copy_to_mode_reg (from_addr_mode,
					     plus_constant (from_addr_mode,
							    from_addr, len));
	  data.autinc_from = 1;
	  data.explicit_inc_from = -1;
	}
      if (USE_LOAD_POST_INCREMENT (mode) && ! data.autinc_from)
	{
	  data.from_addr = copy_to_mode_reg (from_addr_mode, from_addr);
	  data.autinc_from = 1;
	  data.explicit_inc_from = 1;
	}
      if (!data.autinc_from && CONSTANT_P (from_addr))
	data.from_addr = copy_to_mode_reg (from_addr_mode, from_addr);
      if (USE_STORE_PRE_DECREMENT (mode) && data.reverse && ! data.autinc_to)
	{
	  data.to_addr = copy_to_mode_reg (to_addr_mode,
					   plus_constant (to_addr_mode,
							  to_addr, len));
	  data.autinc_to = 1;
	  data.explicit_inc_to = -1;
	}
      if (USE_STORE_POST_INCREMENT (mode) && ! data.reverse && ! data.autinc_to)
	{
	  data.to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
	  data.autinc_to = 1;
	  data.explicit_inc_to = 1;
	}
      if (!data.autinc_to && CONSTANT_P (to_addr))
	data.to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
    }

  align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);

  /* First move what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1 && data.len > 0)
    {
      machine_mode mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
	break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
	move_by_pieces_1 (GEN_FCN (icode), mode, &data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data.len);

  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
	{
	  if (endp == 2)
	    {
	      if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
		emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
	      else
		data.to_addr = copy_to_mode_reg (to_addr_mode,
						 plus_constant (to_addr_mode,
								data.to_addr,
								-1));
	    }
	  to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
					   data.offset);
	}
      else
	{
	  if (endp == 2)
	    --data.offset;
	  to1 = adjust_address (data.to, QImode, data.offset);
	}
      return to1;
    }
  else
    return data.to;
}

/* Return number of insns required to move L bytes by pieces.
   ALIGN (in bits) is maximum alignment we can assume.  */

unsigned HOST_WIDE_INT
move_by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
		       unsigned int max_size)
{
  unsigned HOST_WIDE_INT n_insns = 0;

  align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);

  while (max_size > 1 && l > 0)
    {
      machine_mode mode;
      enum insn_code icode;

      mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
	break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
	n_insns += l / GET_MODE_SIZE (mode), l %= GET_MODE_SIZE (mode);

      max_size = GET_MODE_SIZE (mode);
    }

  gcc_assert (!l);
  return n_insns;
}

/* Subroutine of move_by_pieces.  Move as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
move_by_pieces_1 (insn_gen_fn genfun, machine_mode mode,
		  struct move_by_pieces_d *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1 = NULL_RTX, from1;

  while (data->len >= size)
    {
      if (data->reverse)
	data->offset -= size;

      if (data->to)
	{
	  if (data->autinc_to)
	    to1 = adjust_automodify_address (data->to, mode, data->to_addr,
					     data->offset);
	  else
	    to1 = adjust_address (data->to, mode, data->offset);
	}

      if (data->autinc_from)
	from1 = adjust_automodify_address (data->from, mode, data->from_addr,
					   data->offset);
      else
	from1 = adjust_address (data->from, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
	emit_insn (gen_add2_insn (data->to_addr,
				  gen_int_mode (-(HOST_WIDE_INT) size,
						GET_MODE (data->to_addr))));
      if (HAVE_PRE_DECREMENT && data->explicit_inc_from < 0)
	emit_insn (gen_add2_insn (data->from_addr,
				  gen_int_mode (-(HOST_WIDE_INT) size,
						GET_MODE (data->from_addr))));

      if (data->to)
	emit_insn ((*genfun) (to1, from1));
      else
	{
#ifdef PUSH_ROUNDING
	  emit_single_push_insn (mode, from1, NULL);
#else
	  gcc_unreachable ();
#endif
	}

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
	emit_insn (gen_add2_insn (data->to_addr,
				  gen_int_mode (size,
						GET_MODE (data->to_addr))));
      if (HAVE_POST_INCREMENT && data->explicit_inc_from > 0)
	emit_insn (gen_add2_insn (data->from_addr,
				  gen_int_mode (size,
						GET_MODE (data->from_addr))));

      if (! data->reverse)
	data->offset += size;

      data->len -= size;
    }
}

/* Emit code to move a block Y to a block X.  This may be done with
   string-move instructions, with multiple scalar move instructions,
   or with a library call.

   Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
   SIZE is an rtx that says how long they are.
   ALIGN is the maximum alignment we can assume they have.
   METHOD describes what kind of copy this is, and what mechanisms may be used.
   MIN_SIZE is the minimal size of block to move
   MAX_SIZE is the maximal size of block to move, if it can not be represented
   in unsigned HOST_WIDE_INT, than it is mask of all ones.

   Return the address of the new block, if memcpy is called and returns it,
   0 otherwise.  */

rtx
emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
		       unsigned int expected_align, HOST_WIDE_INT expected_size,
		       unsigned HOST_WIDE_INT min_size,
		       unsigned HOST_WIDE_INT max_size,
		       unsigned HOST_WIDE_INT probable_max_size)
{
  bool may_use_call;
  rtx retval = 0;
  unsigned int align;

  gcc_assert (size);
  if (CONST_INT_P (size)
      && INTVAL (size) == 0)
    return 0;

  switch (method)
    {
    case BLOCK_OP_NORMAL:
    case BLOCK_OP_TAILCALL:
      may_use_call = true;
      break;

    case BLOCK_OP_CALL_PARM:
      may_use_call = block_move_libcall_safe_for_call_parm ();

      /* Make inhibit_defer_pop nonzero around the library call
	 to force it to pop the arguments right away.  */
      NO_DEFER_POP;
      break;

    case BLOCK_OP_NO_LIBCALL:
      may_use_call = false;
      break;

    default:
      gcc_unreachable ();
    }

  gcc_assert (MEM_P (x) && MEM_P (y));
  align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
  gcc_assert (align >= BITS_PER_UNIT);

  /* Make sure we've got BLKmode addresses; store_one_arg can decide that
     block copy is more efficient for other large modes, e.g. DCmode.  */
  x = adjust_address (x, BLKmode, 0);
  y = adjust_address (y, BLKmode, 0);

  /* Set MEM_SIZE as appropriate for this block copy.  The main place this
     can be incorrect is coming from __builtin_memcpy.  */
  if (CONST_INT_P (size))
    {
      x = shallow_copy_rtx (x);
      y = shallow_copy_rtx (y);
      set_mem_size (x, INTVAL (size));
      set_mem_size (y, INTVAL (size));
    }

  if (CONST_INT_P (size) && can_move_by_pieces (INTVAL (size), align))
    move_by_pieces (x, y, INTVAL (size), align, 0);
  else if (emit_block_move_via_movmem (x, y, size, align,
				       expected_align, expected_size,
				       min_size, max_size, probable_max_size))
    ;
  else if (may_use_call
	   && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
	   && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
    {
      /* Since x and y are passed to a libcall, mark the corresponding
	 tree EXPR as addressable.  */
      tree y_expr = MEM_EXPR (y);
      tree x_expr = MEM_EXPR (x);
      if (y_expr)
	mark_addressable (y_expr);
      if (x_expr)
	mark_addressable (x_expr);
      retval = emit_block_move_via_libcall (x, y, size,
					    method == BLOCK_OP_TAILCALL);
    }

  else
    emit_block_move_via_loop (x, y, size, align);

  if (method == BLOCK_OP_CALL_PARM)
    OK_DEFER_POP;

  return retval;
}

rtx
emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method)
{
  unsigned HOST_WIDE_INT max, min = 0;
  if (GET_CODE (size) == CONST_INT)
    min = max = UINTVAL (size);
  else
    max = GET_MODE_MASK (GET_MODE (size));
  return emit_block_move_hints (x, y, size, method, 0, -1,
				min, max, max);
}

/* A subroutine of emit_block_move.  Returns true if calling the
   block move libcall will not clobber any parameters which may have
   already been placed on the stack.  */

static bool
block_move_libcall_safe_for_call_parm (void)
{
#if defined (REG_PARM_STACK_SPACE)
  tree fn;
#endif

  /* If arguments are pushed on the stack, then they're safe.  */
  if (PUSH_ARGS)
    return true;

  /* If registers go on the stack anyway, any argument is sure to clobber
     an outgoing argument.  */
#if defined (REG_PARM_STACK_SPACE)
  fn = emit_block_move_libcall_fn (false);
  /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
     depend on its argument.  */
  (void) fn;
  if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
      && REG_PARM_STACK_SPACE (fn) != 0)
    return false;
#endif

  /* If any argument goes in memory, then it might clobber an outgoing
     argument.  */
  {
    CUMULATIVE_ARGS args_so_far_v;
    cumulative_args_t args_so_far;
    tree fn, arg;

    fn = emit_block_move_libcall_fn (false);
    INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
    args_so_far = pack_cumulative_args (&args_so_far_v);

    arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
    for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
      {
	machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
	rtx tmp = targetm.calls.function_arg (args_so_far, mode,
					      NULL_TREE, true);
	if (!tmp || !REG_P (tmp))
	  return false;
	if (targetm.calls.arg_partial_bytes (args_so_far, mode, NULL, 1))
	  return false;
	targetm.calls.function_arg_advance (args_so_far, mode,
					    NULL_TREE, true);
      }
  }
  return true;
}

/* A subroutine of emit_block_move.  Expand a movmem pattern;
   return true if successful.  */

static bool
emit_block_move_via_movmem (rtx x, rtx y, rtx size, unsigned int align,
			    unsigned int expected_align, HOST_WIDE_INT expected_size,
			    unsigned HOST_WIDE_INT min_size,
			    unsigned HOST_WIDE_INT max_size,
			    unsigned HOST_WIDE_INT probable_max_size)
{
  int save_volatile_ok = volatile_ok;
  machine_mode mode;

  if (expected_align < align)
    expected_align = align;
  if (expected_size != -1)
    {
      if ((unsigned HOST_WIDE_INT)expected_size > probable_max_size)
	expected_size = probable_max_size;
      if ((unsigned HOST_WIDE_INT)expected_size < min_size)
	expected_size = min_size;
    }

  /* Since this is a move insn, we don't care about volatility.  */
  volatile_ok = 1;

  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = direct_optab_handler (movmem_optab, mode);

      if (code != CODE_FOR_nothing
	  /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
	     here because if SIZE is less than the mode mask, as it is
	     returned by the macro, it will definitely be less than the
	     actual mode mask.  Since SIZE is within the Pmode address
	     space, we limit MODE to Pmode.  */
	  && ((CONST_INT_P (size)
	       && ((unsigned HOST_WIDE_INT) INTVAL (size)
		   <= (GET_MODE_MASK (mode) >> 1)))
	      || max_size <= (GET_MODE_MASK (mode) >> 1)
	      || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
	{
	  struct expand_operand ops[9];
	  unsigned int nops;

	  /* ??? When called via emit_block_move_for_call, it'd be
	     nice if there were some way to inform the backend, so
	     that it doesn't fail the expansion because it thinks
	     emitting the libcall would be more efficient.  */
	  nops = insn_data[(int) code].n_generator_args;
	  gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);

	  create_fixed_operand (&ops[0], x);
	  create_fixed_operand (&ops[1], y);
	  /* The check above guarantees that this size conversion is valid.  */
	  create_convert_operand_to (&ops[2], size, mode, true);
	  create_integer_operand (&ops[3], align / BITS_PER_UNIT);
	  if (nops >= 6)
	    {
	      create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
	      create_integer_operand (&ops[5], expected_size);
	    }
	  if (nops >= 8)
	    {
	      create_integer_operand (&ops[6], min_size);
	      /* If we can not represent the maximal size,
		 make parameter NULL.  */
	      if ((HOST_WIDE_INT) max_size != -1)
	        create_integer_operand (&ops[7], max_size);
	      else
		create_fixed_operand (&ops[7], NULL);
	    }
	  if (nops == 9)
	    {
	      /* If we can not represent the maximal size,
		 make parameter NULL.  */
	      if ((HOST_WIDE_INT) probable_max_size != -1)
	        create_integer_operand (&ops[8], probable_max_size);
	      else
		create_fixed_operand (&ops[8], NULL);
	    }
	  if (maybe_expand_insn (code, nops, ops))
	    {
	      volatile_ok = save_volatile_ok;
	      return true;
	    }
	}
    }

  volatile_ok = save_volatile_ok;
  return false;
}

/* A subroutine of emit_block_move.  Expand a call to memcpy.
   Return the return value from memcpy, 0 otherwise.  */

rtx
emit_block_move_via_libcall (rtx dst, rtx src, rtx size, bool tailcall)
{
  rtx dst_addr, src_addr;
  tree call_expr, fn, src_tree, dst_tree, size_tree;
  machine_mode size_mode;
  rtx retval;

  /* Emit code to copy the addresses of DST and SRC and SIZE into new
     pseudos.  We can then place those new pseudos into a VAR_DECL and
     use them later.  */

  dst_addr = copy_addr_to_reg (XEXP (dst, 0));
  src_addr = copy_addr_to_reg (XEXP (src, 0));

  dst_addr = convert_memory_address (ptr_mode, dst_addr);
  src_addr = convert_memory_address (ptr_mode, src_addr);

  dst_tree = make_tree (ptr_type_node, dst_addr);
  src_tree = make_tree (ptr_type_node, src_addr);

  size_mode = TYPE_MODE (sizetype);

  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memcpy in this context.  This could be a user call to memcpy and
     the user may wish to examine the return value from memcpy.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  size_tree = make_tree (sizetype, size);

  fn = emit_block_move_libcall_fn (true);
  call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of emit_block_move_via_libcall.  Create the tree node
   for the function we use for block copies.  */

static GTY(()) tree block_move_fn;

void
init_block_move_fn (const char *asmspec)
{
  if (!block_move_fn)
    {
      tree args, fn, attrs, attr_args;

      fn = get_identifier ("memcpy");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
				       const_ptr_type_node, sizetype,
				       NULL_TREE);

      fn = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      attr_args = build_tree_list (NULL_TREE, build_string (1, "1"));
      attrs = tree_cons (get_identifier ("fn spec"), attr_args, NULL);

      decl_attributes (&fn, attrs, ATTR_FLAG_BUILT_IN);

      block_move_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_move_fn, asmspec);
}

static tree
emit_block_move_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_move_fn)
    init_block_move_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_move_fn);
    }

  return block_move_fn;
}

/* A subroutine of emit_block_move.  Copy the data via an explicit
   loop.  This is used only when libcalls are forbidden.  */
/* ??? It'd be nice to copy in hunks larger than QImode.  */

static void
emit_block_move_via_loop (rtx x, rtx y, rtx size,
			  unsigned int align ATTRIBUTE_UNUSED)
{
  rtx_code_label *cmp_label, *top_label;
  rtx iter, x_addr, y_addr, tmp;
  machine_mode x_addr_mode = get_address_mode (x);
  machine_mode y_addr_mode = get_address_mode (y);
  machine_mode iter_mode;

  iter_mode = GET_MODE (size);
  if (iter_mode == VOIDmode)
    iter_mode = word_mode;

  top_label = gen_label_rtx ();
  cmp_label = gen_label_rtx ();
  iter = gen_reg_rtx (iter_mode);

  emit_move_insn (iter, const0_rtx);

  x_addr = force_operand (XEXP (x, 0), NULL_RTX);
  y_addr = force_operand (XEXP (y, 0), NULL_RTX);
  do_pending_stack_adjust ();

  emit_jump (cmp_label);
  emit_label (top_label);

  tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
  x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);

  if (x_addr_mode != y_addr_mode)
    tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
  y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);

  x = change_address (x, QImode, x_addr);
  y = change_address (y, QImode, y_addr);

  emit_move_insn (x, y);

  tmp = expand_simple_binop (iter_mode, PLUS, iter, const1_rtx, iter,
			     true, OPTAB_LIB_WIDEN);
  if (tmp != iter)
    emit_move_insn (iter, tmp);

  emit_label (cmp_label);

  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
			   true, top_label, REG_BR_PROB_BASE * 90 / 100);
}

/* Copy all or part of a value X into registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_to_reg (int regno, rtx x, int nregs, machine_mode mode)
{
  int i;
#ifdef HAVE_load_multiple
  rtx pat;
  rtx_insn *last;
#endif

  if (nregs == 0)
    return;

  if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
    x = validize_mem (force_const_mem (mode, x));

  /* See if the machine can do this with a load multiple insn.  */
#ifdef HAVE_load_multiple
  if (HAVE_load_multiple)
    {
      last = get_last_insn ();
      pat = gen_load_multiple (gen_rtx_REG (word_mode, regno), x,
			       GEN_INT (nregs));
      if (pat)
	{
	  emit_insn (pat);
	  return;
	}
      else
	delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    emit_move_insn (gen_rtx_REG (word_mode, regno + i),
		    operand_subword_force (x, i, mode));
}

/* Copy all or part of a BLKmode value X out of registers starting at REGNO.
   The number of registers to be filled is NREGS.  */

void
move_block_from_reg (int regno, rtx x, int nregs)
{
  int i;

  if (nregs == 0)
    return;

  /* See if the machine can do this with a store multiple insn.  */
#ifdef HAVE_store_multiple
  if (HAVE_store_multiple)
    {
      rtx_insn *last = get_last_insn ();
      rtx pat = gen_store_multiple (x, gen_rtx_REG (word_mode, regno),
				    GEN_INT (nregs));
      if (pat)
	{
	  emit_insn (pat);
	  return;
	}
      else
	delete_insns_since (last);
    }
#endif

  for (i = 0; i < nregs; i++)
    {
      rtx tem = operand_subword (x, i, 1, BLKmode);

      gcc_assert (tem);

      emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
    }
}

/* Generate a PARALLEL rtx for a new non-consecutive group of registers from
   ORIG, where ORIG is a non-consecutive group of registers represented by
   a PARALLEL.  The clone is identical to the original except in that the
   original set of registers is replaced by a new set of pseudo registers.
   The new set has the same modes as the original set.  */

rtx
gen_group_rtx (rtx orig)
{
  int i, length;
  rtx *tmps;

  gcc_assert (GET_CODE (orig) == PARALLEL);

  length = XVECLEN (orig, 0);
  tmps = XALLOCAVEC (rtx, length);

  /* Skip a NULL entry in first slot.  */
  i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;

  if (i)
    tmps[0] = 0;

  for (; i < length; i++)
    {
      machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
      rtx offset = XEXP (XVECEXP (orig, 0, i), 1);

      tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
    }

  return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
}

/* A subroutine of emit_group_load.  Arguments as for emit_group_load,
   except that values are placed in TMPS[i], and must later be moved
   into corresponding XEXP (XVECEXP (DST, 0, i), 0) element.  */

static void
emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type, int ssize)
{
  rtx src;
  int start, i;
  machine_mode m = GET_MODE (orig_src);

  gcc_assert (GET_CODE (dst) == PARALLEL);

  if (m != VOIDmode
      && !SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_src)
      && GET_CODE (orig_src) != CONCAT)
    {
      machine_mode imode = int_mode_for_mode (GET_MODE (orig_src));
      if (imode == BLKmode)
	src = assign_stack_temp (GET_MODE (orig_src), ssize);
      else
	src = gen_reg_rtx (imode);
      if (imode != BLKmode)
	src = gen_lowpart (GET_MODE (orig_src), src);
      emit_move_insn (src, orig_src);
      /* ...and back again.  */
      if (imode != BLKmode)
	src = gen_lowpart (imode, src);
      emit_group_load_1 (tmps, dst, src, type, ssize);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (dst, 0, 0), 0))
    start = 0;
  else
    start = 1;

  /* Process the pieces.  */
  for (i = start; i < XVECLEN (dst, 0); i++)
    {
      machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (dst, 0, i), 1));
      unsigned int bytelen = GET_MODE_SIZE (mode);
      int shift = 0;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
	{
	  /* Arrange to shift the fragment to where it belongs.
	     extract_bit_field loads to the lsb of the reg.  */
	  if (
#ifdef BLOCK_REG_PADDING
	      BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
	      == (BYTES_BIG_ENDIAN ? upward : downward)
#else
	      BYTES_BIG_ENDIAN
#endif
	      )
	    shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
	  bytelen = ssize - bytepos;
	  gcc_assert (bytelen > 0);
	}

      /* If we won't be loading directly from memory, protect the real source
	 from strange tricks we might play; but make sure that the source can
	 be loaded directly into the destination.  */
      src = orig_src;
      if (!MEM_P (orig_src)
	  && (!CONSTANT_P (orig_src)
	      || (GET_MODE (orig_src) != mode
		  && GET_MODE (orig_src) != VOIDmode)))
	{
	  if (GET_MODE (orig_src) == VOIDmode)
	    src = gen_reg_rtx (mode);
	  else
	    src = gen_reg_rtx (GET_MODE (orig_src));

	  emit_move_insn (src, orig_src);
	}

      /* Optimize the access just a bit.  */
      if (MEM_P (src)
	  && (! SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (src))
	      || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
	  && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
	  && bytelen == GET_MODE_SIZE (mode))
	{
	  tmps[i] = gen_reg_rtx (mode);
	  emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
	}
      else if (COMPLEX_MODE_P (mode)
	       && GET_MODE (src) == mode
	       && bytelen == GET_MODE_SIZE (mode))
	/* Let emit_move_complex do the bulk of the work.  */
	tmps[i] = src;
      else if (GET_CODE (src) == CONCAT)
	{
	  unsigned int slen = GET_MODE_SIZE (GET_MODE (src));
	  unsigned int slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));

	  if ((bytepos == 0 && bytelen == slen0)
	      || (bytepos != 0 && bytepos + bytelen <= slen))
	    {
	      /* The following assumes that the concatenated objects all
		 have the same size.  In this case, a simple calculation
		 can be used to determine the object and the bit field
		 to be extracted.  */
	      tmps[i] = XEXP (src, bytepos / slen0);
	      if (! CONSTANT_P (tmps[i])
		  && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode))
		tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
					     (bytepos % slen0) * BITS_PER_UNIT,
					     1, NULL_RTX, mode, mode);
	    }
	  else
	    {
	      rtx mem;

	      gcc_assert (!bytepos);
	      mem = assign_stack_temp (GET_MODE (src), slen);
	      emit_move_insn (mem, src);
	      tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
					   0, 1, NULL_RTX, mode, mode);
	    }
	}
      /* FIXME: A SIMD parallel will eventually lead to a subreg of a
	 SIMD register, which is currently broken.  While we get GCC
	 to emit proper RTL for these cases, let's dump to memory.  */
      else if (VECTOR_MODE_P (GET_MODE (dst))
	       && REG_P (src))
	{
	  int slen = GET_MODE_SIZE (GET_MODE (src));
	  rtx mem;

	  mem = assign_stack_temp (GET_MODE (src), slen);
	  emit_move_insn (mem, src);
	  tmps[i] = adjust_address (mem, mode, (int) bytepos);
	}
      else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
               && XVECLEN (dst, 0) > 1)
        tmps[i] = simplify_gen_subreg (mode, src, GET_MODE (dst), bytepos);
      else if (CONSTANT_P (src))
	{
	  HOST_WIDE_INT len = (HOST_WIDE_INT) bytelen;

	  if (len == ssize)
	    tmps[i] = src;
	  else
	    {
	      rtx first, second;

	      /* TODO: const_wide_int can have sizes other than this...  */
	      gcc_assert (2 * len == ssize);
	      split_double (src, &first, &second);
	      if (i)
		tmps[i] = second;
	      else
		tmps[i] = first;
	    }
	}
      else if (REG_P (src) && GET_MODE (src) == mode)
	tmps[i] = src;
      else
	tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
				     bytepos * BITS_PER_UNIT, 1, NULL_RTX,
				     mode, mode);

      if (shift)
	tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
				shift, tmps[i], 0);
    }
}

/* Emit code to move a block SRC of type TYPE to a block DST,
   where DST is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_SRC in bytes, or -1
   if not known.  */

void
emit_group_load (rtx dst, rtx src, tree type, int ssize)
{
  rtx *tmps;
  int i;

  tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
  emit_group_load_1 (tmps, dst, src, type, ssize);

  /* Copy the extracted pieces into the proper (probable) hard regs.  */
  for (i = 0; i < XVECLEN (dst, 0); i++)
    {
      rtx d = XEXP (XVECEXP (dst, 0, i), 0);
      if (d == NULL)
	continue;
      emit_move_insn (d, tmps[i]);
    }
}

/* Similar, but load SRC into new pseudos in a format that looks like
   PARALLEL.  This can later be fed to emit_group_move to get things
   in the right place.  */

rtx
emit_group_load_into_temps (rtx parallel, rtx src, tree type, int ssize)
{
  rtvec vec;
  int i;

  vec = rtvec_alloc (XVECLEN (parallel, 0));
  emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);

  /* Convert the vector to look just like the original PARALLEL, except
     with the computed values.  */
  for (i = 0; i < XVECLEN (parallel, 0); i++)
    {
      rtx e = XVECEXP (parallel, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
	{
	  d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
	  e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
	}
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
}

/* Emit code to move a block SRC to block DST, where SRC and DST are
   non-consecutive groups of registers, each represented by a PARALLEL.  */

void
emit_group_move (rtx dst, rtx src)
{
  int i;

  gcc_assert (GET_CODE (src) == PARALLEL
	      && GET_CODE (dst) == PARALLEL
	      && XVECLEN (src, 0) == XVECLEN (dst, 0));

  /* Skip first entry if NULL.  */
  for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
    emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
		    XEXP (XVECEXP (src, 0, i), 0));
}

/* Move a group of registers represented by a PARALLEL into pseudos.  */

rtx
emit_group_move_into_temps (rtx src)
{
  rtvec vec = rtvec_alloc (XVECLEN (src, 0));
  int i;

  for (i = 0; i < XVECLEN (src, 0); i++)
    {
      rtx e = XVECEXP (src, 0, i);
      rtx d = XEXP (e, 0);

      if (d)
	e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
      RTVEC_ELT (vec, i) = e;
    }

  return gen_rtx_PARALLEL (GET_MODE (src), vec);
}

/* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
   where SRC is non-consecutive registers represented by a PARALLEL.
   SSIZE represents the total size of block ORIG_DST, or -1 if not
   known.  */

void
emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED, int ssize)
{
  rtx *tmps, dst;
  int start, finish, i;
  machine_mode m = GET_MODE (orig_dst);

  gcc_assert (GET_CODE (src) == PARALLEL);

  if (!SCALAR_INT_MODE_P (m)
      && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
    {
      machine_mode imode = int_mode_for_mode (GET_MODE (orig_dst));
      if (imode == BLKmode)
        dst = assign_stack_temp (GET_MODE (orig_dst), ssize);
      else
        dst = gen_reg_rtx (imode);
      emit_group_store (dst, src, type, ssize);
      if (imode != BLKmode)
        dst = gen_lowpart (GET_MODE (orig_dst), dst);
      emit_move_insn (orig_dst, dst);
      return;
    }

  /* Check for a NULL entry, used to indicate that the parameter goes
     both on the stack and in registers.  */
  if (XEXP (XVECEXP (src, 0, 0), 0))
    start = 0;
  else
    start = 1;
  finish = XVECLEN (src, 0);

  tmps = XALLOCAVEC (rtx, finish);

  /* Copy the (probable) hard regs into pseudos.  */
  for (i = start; i < finish; i++)
    {
      rtx reg = XEXP (XVECEXP (src, 0, i), 0);
      if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
	{
	  tmps[i] = gen_reg_rtx (GET_MODE (reg));
	  emit_move_insn (tmps[i], reg);
	}
      else
	tmps[i] = reg;
    }

  /* If we won't be storing directly into memory, protect the real destination
     from strange tricks we might play.  */
  dst = orig_dst;
  if (GET_CODE (dst) == PARALLEL)
    {
      rtx temp;

      /* We can get a PARALLEL dst if there is a conditional expression in
	 a return statement.  In that case, the dst and src are the same,
	 so no action is necessary.  */
      if (rtx_equal_p (dst, src))
	return;

      /* It is unclear if we can ever reach here, but we may as well handle
	 it.  Allocate a temporary, and split this into a store/load to/from
	 the temporary.  */
      temp = assign_stack_temp (GET_MODE (dst), ssize);
      emit_group_store (temp, src, type, ssize);
      emit_group_load (dst, temp, type, ssize);
      return;
    }
  else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
    {
      machine_mode outer = GET_MODE (dst);
      machine_mode inner;
      HOST_WIDE_INT bytepos;
      bool done = false;
      rtx temp;

      if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
	dst = gen_reg_rtx (outer);

      /* Make life a bit easier for combine.  */
      /* If the first element of the vector is the low part
	 of the destination mode, use a paradoxical subreg to
	 initialize the destination.  */
      if (start < finish)
	{
	  inner = GET_MODE (tmps[start]);
	  bytepos = subreg_lowpart_offset (inner, outer);
	  if (INTVAL (XEXP (XVECEXP (src, 0, start), 1)) == bytepos)
	    {
	      temp = simplify_gen_subreg (outer, tmps[start],
					  inner, 0);
	      if (temp)
		{
		  emit_move_insn (dst, temp);
		  done = true;
		  start++;
		}
	    }
	}

      /* If the first element wasn't the low part, try the last.  */
      if (!done
	  && start < finish - 1)
	{
	  inner = GET_MODE (tmps[finish - 1]);
	  bytepos = subreg_lowpart_offset (inner, outer);
	  if (INTVAL (XEXP (XVECEXP (src, 0, finish - 1), 1)) == bytepos)
	    {
	      temp = simplify_gen_subreg (outer, tmps[finish - 1],
					  inner, 0);
	      if (temp)
		{
		  emit_move_insn (dst, temp);
		  done = true;
		  finish--;
		}
	    }
	}

      /* Otherwise, simply initialize the result to zero.  */
      if (!done)
        emit_move_insn (dst, CONST0_RTX (outer));
    }

  /* Process the pieces.  */
  for (i = start; i < finish; i++)
    {
      HOST_WIDE_INT bytepos = INTVAL (XEXP (XVECEXP (src, 0, i), 1));
      machine_mode mode = GET_MODE (tmps[i]);
      unsigned int bytelen = GET_MODE_SIZE (mode);
      unsigned int adj_bytelen;
      rtx dest = dst;

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
	adj_bytelen = ssize - bytepos;
      else
	adj_bytelen = bytelen;

      if (GET_CODE (dst) == CONCAT)
	{
	  if (bytepos + adj_bytelen
	      <= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
	    dest = XEXP (dst, 0);
	  else if (bytepos >= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0))))
	    {
	      bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
	      dest = XEXP (dst, 1);
	    }
	  else
	    {
	      machine_mode dest_mode = GET_MODE (dest);
	      machine_mode tmp_mode = GET_MODE (tmps[i]);

	      gcc_assert (bytepos == 0 && XVECLEN (src, 0));

	      if (GET_MODE_ALIGNMENT (dest_mode)
		  >= GET_MODE_ALIGNMENT (tmp_mode))
		{
		  dest = assign_stack_temp (dest_mode,
					    GET_MODE_SIZE (dest_mode));
		  emit_move_insn (adjust_address (dest,
						  tmp_mode,
						  bytepos),
				  tmps[i]);
		  dst = dest;
		}
	      else
		{
		  dest = assign_stack_temp (tmp_mode,
					    GET_MODE_SIZE (tmp_mode));
		  emit_move_insn (dest, tmps[i]);
		  dst = adjust_address (dest, dest_mode, bytepos);
		}
	      break;
	    }
	}

      /* Handle trailing fragments that run over the size of the struct.  */
      if (ssize >= 0 && bytepos + (HOST_WIDE_INT) bytelen > ssize)
	{
	  /* store_bit_field always takes its value from the lsb.
	     Move the fragment to the lsb if it's not already there.  */
	  if (
#ifdef BLOCK_REG_PADDING
	      BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
	      == (BYTES_BIG_ENDIAN ? upward : downward)
#else
	      BYTES_BIG_ENDIAN
#endif
	      )
	    {
	      int shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
	      tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
				      shift, tmps[i], 0);
	    }

	  /* Make sure not to write past the end of the struct.  */
	  store_bit_field (dest,
			   adj_bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
			   bytepos * BITS_PER_UNIT, ssize * BITS_PER_UNIT - 1,
			   VOIDmode, tmps[i]);
	}

      /* Optimize the access just a bit.  */
      else if (MEM_P (dest)
	       && (!SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (dest))
		   || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
	       && bytepos * BITS_PER_UNIT % GET_MODE_ALIGNMENT (mode) == 0
	       && bytelen == GET_MODE_SIZE (mode))
	emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);

      else
	store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
			 0, 0, mode, tmps[i]);
    }

  /* Copy from the pseudo into the (probable) hard reg.  */
  if (orig_dst != dst)
    emit_move_insn (orig_dst, dst);
}

/* Return a form of X that does not use a PARALLEL.  TYPE is the type
   of the value stored in X.  */

rtx
maybe_emit_group_store (rtx x, tree type)
{
  machine_mode mode = TYPE_MODE (type);
  gcc_checking_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
  if (GET_CODE (x) == PARALLEL)
    {
      rtx result = gen_reg_rtx (mode);
      emit_group_store (result, x, type, int_size_in_bytes (type));
      return result;
    }
  return x;
}

/* Copy a BLKmode object of TYPE out of a register SRCREG into TARGET.

   This is used on targets that return BLKmode values in registers.  */

void
copy_blkmode_from_reg (rtx target, rtx srcreg, tree type)
{
  unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
  rtx src = NULL, dst = NULL;
  unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
  unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
  machine_mode mode = GET_MODE (srcreg);
  machine_mode tmode = GET_MODE (target);
  machine_mode copy_mode;

  /* BLKmode registers created in the back-end shouldn't have survived.  */
  gcc_assert (mode != BLKmode);

  /* If the structure doesn't take up a whole number of words, see whether
     SRCREG is padded on the left or on the right.  If it's on the left,
     set PADDING_CORRECTION to the number of bits to skip.

     In most ABIs, the structure will be returned at the least end of
     the register, which translates to right padding on little-endian
     targets and left padding on big-endian targets.  The opposite
     holds if the structure is returned at the most significant
     end of the register.  */
  if (bytes % UNITS_PER_WORD != 0
      && (targetm.calls.return_in_msb (type)
	  ? !BYTES_BIG_ENDIAN
	  : BYTES_BIG_ENDIAN))
    padding_correction
      = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));

  /* We can use a single move if we have an exact mode for the size.  */
  else if (MEM_P (target)
	   && (!SLOW_UNALIGNED_ACCESS (mode, MEM_ALIGN (target))
	       || MEM_ALIGN (target) >= GET_MODE_ALIGNMENT (mode))
	   && bytes == GET_MODE_SIZE (mode))
  {
    emit_move_insn (adjust_address (target, mode, 0), srcreg);
    return;
  }

  /* And if we additionally have the same mode for a register.  */
  else if (REG_P (target)
	   && GET_MODE (target) == mode
	   && bytes == GET_MODE_SIZE (mode))
  {
    emit_move_insn (target, srcreg);
    return;
  }

  /* This code assumes srcreg is at least a full word.  If it isn't, copy it
     into a new pseudo which is a full word.  */
  if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
    {
      srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));
      mode = word_mode;
    }

  /* Copy the structure BITSIZE bits at a time.  If the target lives in
     memory, take care of not reading/writing past its end by selecting
     a copy mode suited to BITSIZE.  This should always be possible given
     how it is computed.

     If the target lives in register, make sure not to select a copy mode
     larger than the mode of the register.

     We could probably emit more efficient code for machines which do not use
     strict alignment, but it doesn't seem worth the effort at the current
     time.  */

  copy_mode = word_mode;
  if (MEM_P (target))
    {
      machine_mode mem_mode = mode_for_size (bitsize, MODE_INT, 1);
      if (mem_mode != BLKmode)
	copy_mode = mem_mode;
    }
  else if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
    copy_mode = tmode;

  for (bitpos = 0, xbitpos = padding_correction;
       bitpos < bytes * BITS_PER_UNIT;
       bitpos += bitsize, xbitpos += bitsize)
    {
      /* We need a new source operand each time xbitpos is on a
	 word boundary and when xbitpos == padding_correction
	 (the first time through).  */
      if (xbitpos % BITS_PER_WORD == 0 || xbitpos == padding_correction)
	src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD, mode);

      /* We need a new destination operand each time bitpos is on
	 a word boundary.  */
      if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
	dst = target;
      else if (bitpos % BITS_PER_WORD == 0)
	dst = operand_subword (target, bitpos / BITS_PER_WORD, 1, tmode);

      /* Use xbitpos for the source extraction (right justified) and
	 bitpos for the destination store (left justified).  */
      store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
		       extract_bit_field (src, bitsize,
					  xbitpos % BITS_PER_WORD, 1,
					  NULL_RTX, copy_mode, copy_mode));
    }
}

/* Copy BLKmode value SRC into a register of mode MODE.  Return the
   register if it contains any data, otherwise return null.

   This is used on targets that return BLKmode values in registers.  */

rtx
copy_blkmode_to_reg (machine_mode mode, tree src)
{
  int i, n_regs;
  unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
  unsigned int bitsize;
  rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
  machine_mode dst_mode;

  gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);

  x = expand_normal (src);

  bytes = int_size_in_bytes (TREE_TYPE (src));
  if (bytes == 0)
    return NULL_RTX;

  /* If the structure doesn't take up a whole number of words, see
     whether the register value should be padded on the left or on
     the right.  Set PADDING_CORRECTION to the number of padding
     bits needed on the left side.

     In most ABIs, the structure will be returned at the least end of
     the register, which translates to right padding on little-endian
     targets and left padding on big-endian targets.  The opposite
     holds if the structure is returned at the most significant
     end of the register.  */
  if (bytes % UNITS_PER_WORD != 0
      && (targetm.calls.return_in_msb (TREE_TYPE (src))
	  ? !BYTES_BIG_ENDIAN
	  : BYTES_BIG_ENDIAN))
    padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
					   * BITS_PER_UNIT));

  n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
  dst_words = XALLOCAVEC (rtx, n_regs);
  bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);

  /* Copy the structure BITSIZE bits at a time.  */
  for (bitpos = 0, xbitpos = padding_correction;
       bitpos < bytes * BITS_PER_UNIT;
       bitpos += bitsize, xbitpos += bitsize)
    {
      /* We need a new destination pseudo each time xbitpos is
	 on a word boundary and when xbitpos == padding_correction
	 (the first time through).  */
      if (xbitpos % BITS_PER_WORD == 0
	  || xbitpos == padding_correction)
	{
	  /* Generate an appropriate register.  */
	  dst_word = gen_reg_rtx (word_mode);
	  dst_words[xbitpos / BITS_PER_WORD] = dst_word;

	  /* Clear the destination before we move anything into it.  */
	  emit_move_insn (dst_word, CONST0_RTX (word_mode));
	}

      /* We need a new source operand each time bitpos is on a word
	 boundary.  */
      if (bitpos % BITS_PER_WORD == 0)
	src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);

      /* Use bitpos for the source extraction (left justified) and
	 xbitpos for the destination store (right justified).  */
      store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
		       0, 0, word_mode,
		       extract_bit_field (src_word, bitsize,
					  bitpos % BITS_PER_WORD, 1,
					  NULL_RTX, word_mode, word_mode));
    }

  if (mode == BLKmode)
    {
      /* Find the smallest integer mode large enough to hold the
	 entire structure.  */
      for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
	   mode != VOIDmode;
	   mode = GET_MODE_WIDER_MODE (mode))
	/* Have we found a large enough mode?  */
	if (GET_MODE_SIZE (mode) >= bytes)
	  break;

      /* A suitable mode should have been found.  */
      gcc_assert (mode != VOIDmode);
    }

  if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
    dst_mode = word_mode;
  else
    dst_mode = mode;
  dst = gen_reg_rtx (dst_mode);

  for (i = 0; i < n_regs; i++)
    emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);

  if (mode != dst_mode)
    dst = gen_lowpart (mode, dst);

  return dst;
}

/* Add a USE expression for REG to the (possibly empty) list pointed
   to by CALL_FUSAGE.  REG must denote a hard register.  */

void
use_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
{
  gcc_assert (REG_P (reg));

  if (!HARD_REGISTER_P (reg))
    return;

  *call_fusage
    = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
}

/* Add a CLOBBER expression for REG to the (possibly empty) list pointed
   to by CALL_FUSAGE.  REG must denote a hard register.  */

void
clobber_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
{
  gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);

  *call_fusage
    = gen_rtx_EXPR_LIST (mode, gen_rtx_CLOBBER (VOIDmode, reg), *call_fusage);
}

/* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
   starting at REGNO.  All of these registers must be hard registers.  */

void
use_regs (rtx *call_fusage, int regno, int nregs)
{
  int i;

  gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);

  for (i = 0; i < nregs; i++)
    use_reg (call_fusage, regno_reg_rtx[regno + i]);
}

/* Add USE expressions to *CALL_FUSAGE for each REG contained in the
   PARALLEL REGS.  This is for calls that pass values in multiple
   non-contiguous locations.  The Irix 6 ABI has examples of this.  */

void
use_group_regs (rtx *call_fusage, rtx regs)
{
  int i;

  for (i = 0; i < XVECLEN (regs, 0); i++)
    {
      rtx reg = XEXP (XVECEXP (regs, 0, i), 0);

      /* A NULL entry means the parameter goes both on the stack and in
	 registers.  This can also be a MEM for targets that pass values
	 partially on the stack and partially in registers.  */
      if (reg != 0 && REG_P (reg))
	use_reg (call_fusage, reg);
    }
}

/* Return the defining gimple statement for SSA_NAME NAME if it is an
   assigment and the code of the expresion on the RHS is CODE.  Return
   NULL otherwise.  */

static gimple
get_def_for_expr (tree name, enum tree_code code)
{
  gimple def_stmt;

  if (TREE_CODE (name) != SSA_NAME)
    return NULL;

  def_stmt = get_gimple_for_ssa_name (name);
  if (!def_stmt
      || gimple_assign_rhs_code (def_stmt) != code)
    return NULL;

  return def_stmt;
}

#ifdef HAVE_conditional_move
/* Return the defining gimple statement for SSA_NAME NAME if it is an
   assigment and the class of the expresion on the RHS is CLASS.  Return
   NULL otherwise.  */

static gimple
get_def_for_expr_class (tree name, enum tree_code_class tclass)
{
  gimple def_stmt;

  if (TREE_CODE (name) != SSA_NAME)
    return NULL;

  def_stmt = get_gimple_for_ssa_name (name);
  if (!def_stmt
      || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
    return NULL;

  return def_stmt;
}
#endif


/* Determine whether the LEN bytes generated by CONSTFUN can be
   stored to memory using several move instructions.  CONSTFUNDATA is
   a pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.  MEMSETP is true if this is
   a memset operation and false if it's a copy of a constant string.
   Return nonzero if a call to store_by_pieces should succeed.  */

int
can_store_by_pieces (unsigned HOST_WIDE_INT len,
		     rtx (*constfun) (void *, HOST_WIDE_INT, machine_mode),
		     void *constfundata, unsigned int align, bool memsetp)
{
  unsigned HOST_WIDE_INT l;
  unsigned int max_size;
  HOST_WIDE_INT offset = 0;
  machine_mode mode;
  enum insn_code icode;
  int reverse;
  /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it.  */
  rtx cst ATTRIBUTE_UNUSED;

  if (len == 0)
    return 1;

  if (!targetm.use_by_pieces_infrastructure_p (len, align,
					       memsetp
						 ? SET_BY_PIECES
						 : STORE_BY_PIECES,
					       optimize_insn_for_speed_p ()))
    return 0;

  align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);

  /* We would first store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  for (reverse = 0;
       reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
       reverse++)
    {
      l = len;
      max_size = STORE_MAX_PIECES + 1;
      while (max_size > 1 && l > 0)
	{
	  mode = widest_int_mode_for_size (max_size);

	  if (mode == VOIDmode)
	    break;

	  icode = optab_handler (mov_optab, mode);
	  if (icode != CODE_FOR_nothing
	      && align >= GET_MODE_ALIGNMENT (mode))
	    {
	      unsigned int size = GET_MODE_SIZE (mode);

	      while (l >= size)
		{
		  if (reverse)
		    offset -= size;

		  cst = (*constfun) (constfundata, offset, mode);
		  if (!targetm.legitimate_constant_p (mode, cst))
		    return 0;

		  if (!reverse)
		    offset += size;

		  l -= size;
		}
	    }

	  max_size = GET_MODE_SIZE (mode);
	}

      /* The code above should have handled everything.  */
      gcc_assert (!l);
    }

  return 1;
}

/* Generate several move instructions to store LEN bytes generated by
   CONSTFUN to block TO.  (A MEM rtx with BLKmode).  CONSTFUNDATA is a
   pointer which will be passed as argument in every CONSTFUN call.
   ALIGN is maximum alignment we can assume.  MEMSETP is true if this is
   a memset operation and false if it's a copy of a constant string.
   If ENDP is 0 return to, if ENDP is 1 return memory at the end ala
   mempcpy, and if ENDP is 2 return memory the end minus one byte ala
   stpcpy.  */

rtx
store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
		 rtx (*constfun) (void *, HOST_WIDE_INT, machine_mode),
		 void *constfundata, unsigned int align, bool memsetp, int endp)
{
  machine_mode to_addr_mode = get_address_mode (to);
  struct store_by_pieces_d data;

  if (len == 0)
    {
      gcc_assert (endp != 2);
      return to;
    }

  gcc_assert (targetm.use_by_pieces_infrastructure_p
		(len, align,
		 memsetp
		   ? SET_BY_PIECES
		   : STORE_BY_PIECES,
		 optimize_insn_for_speed_p ()));

  data.constfun = constfun;
  data.constfundata = constfundata;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
  if (endp)
    {
      rtx to1;

      gcc_assert (!data.reverse);
      if (data.autinc_to)
	{
	  if (endp == 2)
	    {
	      if (HAVE_POST_INCREMENT && data.explicit_inc_to > 0)
		emit_insn (gen_add2_insn (data.to_addr, constm1_rtx));
	      else
		data.to_addr = copy_to_mode_reg (to_addr_mode,
						 plus_constant (to_addr_mode,
								data.to_addr,
								-1));
	    }
	  to1 = adjust_automodify_address (data.to, QImode, data.to_addr,
					   data.offset);
	}
      else
	{
	  if (endp == 2)
	    --data.offset;
	  to1 = adjust_address (data.to, QImode, data.offset);
	}
      return to1;
    }
  else
    return data.to;
}

/* Generate several move instructions to clear LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
{
  struct store_by_pieces_d data;

  if (len == 0)
    return;

  data.constfun = clear_by_pieces_1;
  data.constfundata = NULL;
  data.len = len;
  data.to = to;
  store_by_pieces_1 (&data, align);
}

/* Callback routine for clear_by_pieces.
   Return const0_rtx unconditionally.  */

static rtx
clear_by_pieces_1 (void *data ATTRIBUTE_UNUSED,
		   HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
		   machine_mode mode ATTRIBUTE_UNUSED)
{
  return const0_rtx;
}

/* Subroutine of clear_by_pieces and store_by_pieces.
   Generate several move instructions to store LEN bytes of block TO.  (A MEM
   rtx with BLKmode).  ALIGN is maximum alignment we can assume.  */

static void
store_by_pieces_1 (struct store_by_pieces_d *data ATTRIBUTE_UNUSED,
		   unsigned int align ATTRIBUTE_UNUSED)
{
  machine_mode to_addr_mode = get_address_mode (data->to);
  rtx to_addr = XEXP (data->to, 0);
  unsigned int max_size = STORE_MAX_PIECES + 1;
  enum insn_code icode;

  data->offset = 0;
  data->to_addr = to_addr;
  data->autinc_to
    = (GET_CODE (to_addr) == PRE_INC || GET_CODE (to_addr) == PRE_DEC
       || GET_CODE (to_addr) == POST_INC || GET_CODE (to_addr) == POST_DEC);

  data->explicit_inc_to = 0;
  data->reverse
    = (GET_CODE (to_addr) == PRE_DEC || GET_CODE (to_addr) == POST_DEC);
  if (data->reverse)
    data->offset = data->len;

  /* If storing requires more than two move insns,
     copy addresses to registers (to make displacements shorter)
     and use post-increment if available.  */
  if (!data->autinc_to
      && move_by_pieces_ninsns (data->len, align, max_size) > 2)
    {
      /* Determine the main mode we'll be using.
	 MODE might not be used depending on the definitions of the
	 USE_* macros below.  */
      machine_mode mode ATTRIBUTE_UNUSED
	= widest_int_mode_for_size (max_size);

      if (USE_STORE_PRE_DECREMENT (mode) && data->reverse && ! data->autinc_to)
	{
	  data->to_addr = copy_to_mode_reg (to_addr_mode,
					    plus_constant (to_addr_mode,
							   to_addr,
							   data->len));
	  data->autinc_to = 1;
	  data->explicit_inc_to = -1;
	}

      if (USE_STORE_POST_INCREMENT (mode) && ! data->reverse
	  && ! data->autinc_to)
	{
	  data->to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
	  data->autinc_to = 1;
	  data->explicit_inc_to = 1;
	}

      if ( !data->autinc_to && CONSTANT_P (to_addr))
	data->to_addr = copy_to_mode_reg (to_addr_mode, to_addr);
    }

  align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);

  /* First store what we can in the largest integer mode, then go to
     successively smaller modes.  */

  while (max_size > 1 && data->len > 0)
    {
      machine_mode mode = widest_int_mode_for_size (max_size);

      if (mode == VOIDmode)
	break;

      icode = optab_handler (mov_optab, mode);
      if (icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode))
	store_by_pieces_2 (GEN_FCN (icode), mode, data);

      max_size = GET_MODE_SIZE (mode);
    }

  /* The code above should have handled everything.  */
  gcc_assert (!data->len);
}

/* Subroutine of store_by_pieces_1.  Store as many bytes as appropriate
   with move instructions for mode MODE.  GENFUN is the gen_... function
   to make a move insn for that mode.  DATA has all the other info.  */

static void
store_by_pieces_2 (insn_gen_fn genfun, machine_mode mode,
		   struct store_by_pieces_d *data)
{
  unsigned int size = GET_MODE_SIZE (mode);
  rtx to1, cst;

  while (data->len >= size)
    {
      if (data->reverse)
	data->offset -= size;

      if (data->autinc_to)
	to1 = adjust_automodify_address (data->to, mode, data->to_addr,
					 data->offset);
      else
	to1 = adjust_address (data->to, mode, data->offset);

      if (HAVE_PRE_DECREMENT && data->explicit_inc_to < 0)
	emit_insn (gen_add2_insn (data->to_addr,
				  gen_int_mode (-(HOST_WIDE_INT) size,
						GET_MODE (data->to_addr))));

      cst = (*data->constfun) (data->constfundata, data->offset, mode);
      emit_insn ((*genfun) (to1, cst));

      if (HAVE_POST_INCREMENT && data->explicit_inc_to > 0)
	emit_insn (gen_add2_insn (data->to_addr,
				  gen_int_mode (size,
						GET_MODE (data->to_addr))));

      if (! data->reverse)
	data->offset += size;

      data->len -= size;
    }
}

/* Write zeros through the storage of OBJECT.  If OBJECT has BLKmode, SIZE is
   its length in bytes.  */

rtx
clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
		     unsigned int expected_align, HOST_WIDE_INT expected_size,
		     unsigned HOST_WIDE_INT min_size,
		     unsigned HOST_WIDE_INT max_size,
		     unsigned HOST_WIDE_INT probable_max_size)
{
  machine_mode mode = GET_MODE (object);
  unsigned int align;

  gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);

  /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
     just move a zero.  Otherwise, do this a piece at a time.  */
  if (mode != BLKmode
      && CONST_INT_P (size)
      && INTVAL (size) == (HOST_WIDE_INT) GET_MODE_SIZE (mode))
    {
      rtx zero = CONST0_RTX (mode);
      if (zero != NULL)
	{
	  emit_move_insn (object, zero);
	  return NULL;
	}

      if (COMPLEX_MODE_P (mode))
	{
	  zero = CONST0_RTX (GET_MODE_INNER (mode));
	  if (zero != NULL)
	    {
	      write_complex_part (object, zero, 0);
	      write_complex_part (object, zero, 1);
	      return NULL;
	    }
	}
    }

  if (size == const0_rtx)
    return NULL;

  align = MEM_ALIGN (object);

  if (CONST_INT_P (size)
      && targetm.use_by_pieces_infrastructure_p (INTVAL (size), align,
						 CLEAR_BY_PIECES,
						 optimize_insn_for_speed_p ()))
    clear_by_pieces (object, INTVAL (size), align);
  else if (set_storage_via_setmem (object, size, const0_rtx, align,
				   expected_align, expected_size,
				   min_size, max_size, probable_max_size))
    ;
  else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
    return set_storage_via_libcall (object, size, const0_rtx,
				    method == BLOCK_OP_TAILCALL);
  else
    gcc_unreachable ();

  return NULL;
}

rtx
clear_storage (rtx object, rtx size, enum block_op_methods method)
{
  unsigned HOST_WIDE_INT max, min = 0;
  if (GET_CODE (size) == CONST_INT)
    min = max = UINTVAL (size);
  else
    max = GET_MODE_MASK (GET_MODE (size));
  return clear_storage_hints (object, size, method, 0, -1, min, max, max);
}


/* A subroutine of clear_storage.  Expand a call to memset.
   Return the return value of memset, 0 otherwise.  */

rtx
set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
{
  tree call_expr, fn, object_tree, size_tree, val_tree;
  machine_mode size_mode;
  rtx retval;

  /* Emit code to copy OBJECT and SIZE into new pseudos.  We can then
     place those into new pseudos into a VAR_DECL and use them later.  */

  object = copy_addr_to_reg (XEXP (object, 0));

  size_mode = TYPE_MODE (sizetype);
  size = convert_to_mode (size_mode, size, 1);
  size = copy_to_mode_reg (size_mode, size);

  /* It is incorrect to use the libcall calling conventions to call
     memset in this context.  This could be a user call to memset and
     the user may wish to examine the return value from memset.  For
     targets where libcalls and normal calls have different conventions
     for returning pointers, we could end up generating incorrect code.  */

  object_tree = make_tree (ptr_type_node, object);
  if (!CONST_INT_P (val))
    val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
  size_tree = make_tree (sizetype, size);
  val_tree = make_tree (integer_type_node, val);

  fn = clear_storage_libcall_fn (true);
  call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
  CALL_EXPR_TAILCALL (call_expr) = tailcall;

  retval = expand_normal (call_expr);

  return retval;
}

/* A subroutine of set_storage_via_libcall.  Create the tree node
   for the function we use for block clears.  */

tree block_clear_fn;

void
init_block_clear_fn (const char *asmspec)
{
  if (!block_clear_fn)
    {
      tree fn, args;

      fn = get_identifier ("memset");
      args = build_function_type_list (ptr_type_node, ptr_type_node,
				       integer_type_node, sizetype,
				       NULL_TREE);

      fn = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, fn, args);
      DECL_EXTERNAL (fn) = 1;
      TREE_PUBLIC (fn) = 1;
      DECL_ARTIFICIAL (fn) = 1;
      TREE_NOTHROW (fn) = 1;
      DECL_VISIBILITY (fn) = VISIBILITY_DEFAULT;
      DECL_VISIBILITY_SPECIFIED (fn) = 1;

      block_clear_fn = fn;
    }

  if (asmspec)
    set_user_assembler_name (block_clear_fn, asmspec);
}

static tree
clear_storage_libcall_fn (int for_call)
{
  static bool emitted_extern;

  if (!block_clear_fn)
    init_block_clear_fn (NULL);

  if (for_call && !emitted_extern)
    {
      emitted_extern = true;
      make_decl_rtl (block_clear_fn);
    }

  return block_clear_fn;
}

/* Expand a setmem pattern; return true if successful.  */

bool
set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
			unsigned int expected_align, HOST_WIDE_INT expected_size,
			unsigned HOST_WIDE_INT min_size,
			unsigned HOST_WIDE_INT max_size,
			unsigned HOST_WIDE_INT probable_max_size)
{
  /* Try the most limited insn first, because there's no point
     including more than one in the machine description unless
     the more limited one has some advantage.  */

  machine_mode mode;

  if (expected_align < align)
    expected_align = align;
  if (expected_size != -1)
    {
      if ((unsigned HOST_WIDE_INT)expected_size > max_size)
	expected_size = max_size;
      if ((unsigned HOST_WIDE_INT)expected_size < min_size)
	expected_size = min_size;
    }

  for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
       mode = GET_MODE_WIDER_MODE (mode))
    {
      enum insn_code code = direct_optab_handler (setmem_optab, mode);

      if (code != CODE_FOR_nothing
	  /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
	     here because if SIZE is less than the mode mask, as it is
	     returned by the macro, it will definitely be less than the
	     actual mode mask.  Since SIZE is within the Pmode address
	     space, we limit MODE to Pmode.  */
	  && ((CONST_INT_P (size)
	       && ((unsigned HOST_WIDE_INT) INTVAL (size)
		   <= (GET_MODE_MASK (mode) >> 1)))
	      || max_size <= (GET_MODE_MASK (mode) >> 1)
	      || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
	{
	  struct expand_operand ops[9];
	  unsigned int nops;

	  nops = insn_data[(int) code].n_generator_args;
	  gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);

	  create_fixed_operand (&ops[0], object);
	  /* The check above guarantees that this size conversion is valid.  */
	  create_convert_operand_to (&ops[1], size, mode, true);
	  create_convert_operand_from (&ops[2], val, byte_mode, true);
	  create_integer_operand (&ops[3], align / BITS_PER_UNIT);
	  if (nops >= 6)
	    {
	      create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
	      create_integer_operand (&ops[5], expected_size);
	    }
	  if (nops >= 8)
	    {
	      create_integer_operand (&ops[6], min_size);
	      /* If we can not represent the maximal size,
		 make parameter NULL.  */
	      if ((HOST_WIDE_INT) max_size != -1)
	        create_integer_operand (&ops[7], max_size);
	      else
		create_fixed_operand (&ops[7], NULL);
	    }
	  if (nops == 9)
	    {
	      /* If we can not represent the maximal size,
		 make parameter NULL.  */
	      if ((HOST_WIDE_INT) probable_max_size != -1)
	        create_integer_operand (&ops[8], probable_max_size);
	      else
		create_fixed_operand (&ops[8], NULL);
	    }
	  if (maybe_expand_insn (code, nops, ops))
	    return true;
	}
    }

  return false;
}


/* Write to one of the components of the complex value CPLX.  Write VAL to
   the real part if IMAG_P is false, and the imaginary part if its true.  */

void
write_complex_part (rtx cplx, rtx val, bool imag_p)
{
  machine_mode cmode;
  machine_mode imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    {
      emit_move_insn (XEXP (cplx, imag_p), val);
      return;
    }

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    {
      emit_move_insn (adjust_address_nv (cplx, imode,
					 imag_p ? GET_MODE_SIZE (imode) : 0),
		      val);
      return;
    }

  /* If the sub-object is at least word sized, then we know that subregging
     will work.  This special case is important, since store_bit_field
     wants to operate on integer modes, and there's rarely an OImode to
     correspond to TCmode.  */
  if (ibitsize >= BITS_PER_WORD
      /* For hard regs we have exact predicates.  Assume we can split
	 the original object if it spans an even number of hard regs.
	 This special case is important for SCmode on 64-bit platforms
	 where the natural size of floating-point regs is 32-bit.  */
      || (REG_P (cplx)
	  && REGNO (cplx) < FIRST_PSEUDO_REGISTER
	  && hard_regno_nregs[REGNO (cplx)][cmode] % 2 == 0))
    {
      rtx part = simplify_gen_subreg (imode, cplx, cmode,
				      imag_p ? GET_MODE_SIZE (imode) : 0);
      if (part)
        {
	  emit_move_insn (part, val);
	  return;
	}
      else
	/* simplify_gen_subreg may fail for sub-word MEMs.  */
	gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
    }

  store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val);
}

/* Extract one of the components of the complex value CPLX.  Extract the
   real part if IMAG_P is false, and the imaginary part if it's true.  */

static rtx
read_complex_part (rtx cplx, bool imag_p)
{
  machine_mode cmode, imode;
  unsigned ibitsize;

  if (GET_CODE (cplx) == CONCAT)
    return XEXP (cplx, imag_p);

  cmode = GET_MODE (cplx);
  imode = GET_MODE_INNER (cmode);
  ibitsize = GET_MODE_BITSIZE (imode);

  /* Special case reads from complex constants that got spilled to memory.  */
  if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
    {
      tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
      if (decl && TREE_CODE (decl) == COMPLEX_CST)
	{
	  tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
	  if (CONSTANT_CLASS_P (part))
	    return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
	}
    }

  /* For MEMs simplify_gen_subreg may generate an invalid new address
     because, e.g., the original address is considered mode-dependent
     by the target, which restricts simplify_subreg from invoking
     adjust_address_nv.  Instead of preparing fallback support for an
     invalid address, we call adjust_address_nv directly.  */
  if (MEM_P (cplx))
    return adjust_address_nv (cplx, imode,
			      imag_p ? GET_MODE_SIZE (imode) : 0);

  /* If the sub-object is at least word sized, then we know that subregging