Google Git
Sign in
gnu / gcc / f49e3d28be44179f07b8a06159139ce77096dda7 / . / gcc / config / i386 / i386-expand.c
blob: 3e6f7d8ef7e942793ed2ee658bd4debbfebf048f [file] [log] [blame]
/* Copyright (C) 1988-2021 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.
GCC is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
#define IN_TARGET_CODE 1
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "memmodel.h"
#include "gimple.h"
#include "cfghooks.h"
#include "cfgloop.h"
#include "df.h"
#include "tm_p.h"
#include "stringpool.h"
#include "expmed.h"
#include "optabs.h"
#include "regs.h"
#include "emit-rtl.h"
#include "recog.h"
#include "cgraph.h"
#include "diagnostic.h"
#include "cfgbuild.h"
#include "alias.h"
#include "fold-const.h"
#include "attribs.h"
#include "calls.h"
#include "stor-layout.h"
#include "varasm.h"
#include "output.h"
#include "insn-attr.h"
#include "flags.h"
#include "except.h"
#include "explow.h"
#include "expr.h"
#include "cfgrtl.h"
#include "common/common-target.h"
#include "langhooks.h"
#include "reload.h"
#include "gimplify.h"
#include "dwarf2.h"
#include "tm-constrs.h"
#include "cselib.h"
#include "sched-int.h"
#include "opts.h"
#include "tree-pass.h"
#include "context.h"
#include "pass_manager.h"
#include "target-globals.h"
#include "gimple-iterator.h"
#include "tree-vectorizer.h"
#include "shrink-wrap.h"
#include "builtins.h"
#include "rtl-iter.h"
#include "tree-iterator.h"
#include "dbgcnt.h"
#include "case-cfn-macros.h"
#include "dojump.h"
#include "fold-const-call.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
#include "selftest.h"
#include "selftest-rtl.h"
#include "print-rtl.h"
#include "intl.h"
#include "ifcvt.h"
#include "symbol-summary.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
#include "wide-int-bitmask.h"
#include "tree-vector-builder.h"
#include "debug.h"
#include "dwarf2out.h"
#include "i386-options.h"
#include "i386-builtins.h"
#include "i386-expand.h"
/* Split one or more double-mode RTL references into pairs of half-mode
references. The RTL can be REG, offsettable MEM, integer constant, or
CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
split and "num" is its length. lo_half and hi_half are output arrays
that parallel "operands". */
void
split_double_mode (machine_mode mode, rtx operands[],
int num, rtx lo_half[], rtx hi_half[])
{
machine_mode half_mode;
unsigned int byte;
rtx mem_op = NULL_RTX;
int mem_num = 0;
switch (mode)
{
case E_TImode:
half_mode = DImode;
break;
case E_DImode:
half_mode = SImode;
break;
case E_P2HImode:
half_mode = HImode;
break;
case E_P2QImode:
half_mode = QImode;
break;
default:
gcc_unreachable ();
}
byte = GET_MODE_SIZE (half_mode);
while (num--)
{
rtx op = operands[num];
/* simplify_subreg refuse to split volatile memory addresses,
but we still have to handle it. */
if (MEM_P (op))
{
if (mem_op && rtx_equal_p (op, mem_op))
{
lo_half[num] = lo_half[mem_num];
hi_half[num] = hi_half[mem_num];
}
else
{
mem_op = op;
mem_num = num;
lo_half[num] = adjust_address (op, half_mode, 0);
hi_half[num] = adjust_address (op, half_mode, byte);
}
}
else
{
lo_half[num] = simplify_gen_subreg (half_mode, op,
GET_MODE (op) == VOIDmode
? mode : GET_MODE (op), 0);
rtx tmp = simplify_gen_subreg (half_mode, op,
GET_MODE (op) == VOIDmode
? mode : GET_MODE (op), byte);
/* simplify_gen_subreg will return NULL RTX for the
high half of the paradoxical subreg. */
hi_half[num] = tmp ? tmp : gen_reg_rtx (half_mode);
}
}
}
/* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
for the target. */
void
ix86_expand_clear (rtx dest)
{
rtx tmp;
/* We play register width games, which are only valid after reload. */
gcc_assert (reload_completed);
/* Avoid HImode and its attendant prefix byte. */
if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
dest = gen_rtx_REG (SImode, REGNO (dest));
tmp = gen_rtx_SET (dest, const0_rtx);
if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
{
rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
}
emit_insn (tmp);
}
/* Return true if V can be broadcasted from an integer of WIDTH bits
which is returned in VAL_BROADCAST. Otherwise, return false. */
static bool
ix86_broadcast (HOST_WIDE_INT v, unsigned int width,
HOST_WIDE_INT &val_broadcast)
{
wide_int val = wi::uhwi (v, HOST_BITS_PER_WIDE_INT);
val_broadcast = wi::extract_uhwi (val, 0, width);
for (unsigned int i = width; i < HOST_BITS_PER_WIDE_INT; i += width)
{
HOST_WIDE_INT each = wi::extract_uhwi (val, i, width);
if (val_broadcast != each)
return false;
}
val_broadcast = sext_hwi (val_broadcast, width);
return true;
}
/* Convert the CONST_WIDE_INT operand OP to broadcast in MODE. */
static rtx
ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
{
/* Don't use integer vector broadcast if we can't move from GPR to SSE
register directly. */
if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
return nullptr;
/* Convert CONST_WIDE_INT to a non-standard SSE constant integer
broadcast only if vector broadcast is available. */
if (!TARGET_AVX
|| !CONST_WIDE_INT_P (op)
|| standard_sse_constant_p (op, mode))
return nullptr;
HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
HOST_WIDE_INT val_broadcast;
scalar_int_mode broadcast_mode;
if (TARGET_AVX2
&& ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
val_broadcast))
broadcast_mode = QImode;
else if (TARGET_AVX2
&& ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
val_broadcast))
broadcast_mode = HImode;
else if (ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
val_broadcast))
broadcast_mode = SImode;
else if (TARGET_64BIT
&& ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
val_broadcast))
broadcast_mode = DImode;
else
return nullptr;
/* Check if OP can be broadcasted from VAL. */
for (int i = 1; i < CONST_WIDE_INT_NUNITS (op); i++)
if (val != CONST_WIDE_INT_ELT (op, i))
return nullptr;
unsigned int nunits = (GET_MODE_SIZE (mode)
/ GET_MODE_SIZE (broadcast_mode));
machine_mode vector_mode;
if (!mode_for_vector (broadcast_mode, nunits).exists (&vector_mode))
gcc_unreachable ();
rtx target = ix86_gen_scratch_sse_rtx (vector_mode);
bool ok = ix86_expand_vector_init_duplicate (false, vector_mode,
target,
GEN_INT (val_broadcast));
gcc_assert (ok);
target = lowpart_subreg (mode, target, vector_mode);
return target;
}
void
ix86_expand_move (machine_mode mode, rtx operands[])
{
rtx op0, op1;
rtx tmp, addend = NULL_RTX;
enum tls_model model;
op0 = operands[0];
op1 = operands[1];
/* Avoid complex sets of likely spilled hard registers before reload. */
if (!ix86_hardreg_mov_ok (op0, op1))
{
tmp = gen_reg_rtx (mode);
operands[0] = tmp;
ix86_expand_move (mode, operands);
operands[0] = op0;
operands[1] = tmp;
op1 = tmp;
}
switch (GET_CODE (op1))
{
case CONST:
tmp = XEXP (op1, 0);
if (GET_CODE (tmp) != PLUS
|| GET_CODE (XEXP (tmp, 0)) != SYMBOL_REF)
break;
op1 = XEXP (tmp, 0);
addend = XEXP (tmp, 1);
/* FALLTHRU */
case SYMBOL_REF:
model = SYMBOL_REF_TLS_MODEL (op1);
if (model)
op1 = legitimize_tls_address (op1, model, true);
else if (ix86_force_load_from_GOT_p (op1))
{
/* Load the external function address via GOT slot to avoid PLT. */
op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op1),
(TARGET_64BIT
? UNSPEC_GOTPCREL
: UNSPEC_GOT));
op1 = gen_rtx_CONST (Pmode, op1);
op1 = gen_const_mem (Pmode, op1);
set_mem_alias_set (op1, ix86_GOT_alias_set ());
}
else
{
tmp = legitimize_pe_coff_symbol (op1, addend != NULL_RTX);
if (tmp)
{
op1 = tmp;
if (!addend)
break;
}
else
{
op1 = operands[1];
break;
}
}
if (addend)
{
op1 = force_operand (op1, NULL_RTX);
op1 = expand_simple_binop (Pmode, PLUS, op1, addend,
op0, 1, OPTAB_DIRECT);
}
else
op1 = force_operand (op1, op0);
if (op1 == op0)
return;
op1 = convert_to_mode (mode, op1, 1);
default:
break;
}
if ((flag_pic || MACHOPIC_INDIRECT)
&& symbolic_operand (op1, mode))
{
if (TARGET_MACHO && !TARGET_64BIT)
{
#if TARGET_MACHO
/* dynamic-no-pic */
if (MACHOPIC_INDIRECT)
{
rtx temp = (op0 && REG_P (op0) && mode == Pmode)
? op0 : gen_reg_rtx (Pmode);
op1 = machopic_indirect_data_reference (op1, temp);
if (MACHOPIC_PURE)
op1 = machopic_legitimize_pic_address (op1, mode,
temp == op1 ? 0 : temp);
}
if (op0 != op1 && GET_CODE (op0) != MEM)
{
rtx insn = gen_rtx_SET (op0, op1);
emit_insn (insn);
return;
}
if (GET_CODE (op0) == MEM)
op1 = force_reg (Pmode, op1);
else
{
rtx temp = op0;
if (GET_CODE (temp) != REG)
temp = gen_reg_rtx (Pmode);
temp = legitimize_pic_address (op1, temp);
if (temp == op0)
return;
op1 = temp;
}
/* dynamic-no-pic */
#endif
}
else
{
if (MEM_P (op0))
op1 = force_reg (mode, op1);
else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
{
rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
op1 = legitimize_pic_address (op1, reg);
if (op0 == op1)
return;
op1 = convert_to_mode (mode, op1, 1);
}
}
}
else
{
if (MEM_P (op0)
&& (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
|| !push_operand (op0, mode))
&& MEM_P (op1))
op1 = force_reg (mode, op1);
if (push_operand (op0, mode)
&& ! general_no_elim_operand (op1, mode))
op1 = copy_to_mode_reg (mode, op1);
/* Force large constants in 64bit compilation into register
to get them CSEed. */
if (can_create_pseudo_p ()
&& (mode == DImode) && TARGET_64BIT
&& immediate_operand (op1, mode)
&& !x86_64_zext_immediate_operand (op1, VOIDmode)
&& !register_operand (op0, mode)
&& optimize)
op1 = copy_to_mode_reg (mode, op1);
if (can_create_pseudo_p ())
{
if (CONST_DOUBLE_P (op1))
{
/* If we are loading a floating point constant to a
register, force the value to memory now, since we'll
get better code out the back end. */
op1 = validize_mem (force_const_mem (mode, op1));
if (!register_operand (op0, mode))
{
rtx temp = gen_reg_rtx (mode);
emit_insn (gen_rtx_SET (temp, op1));
emit_move_insn (op0, temp);
return;
}
}
else if (GET_MODE_SIZE (mode) >= 16)
{
rtx tmp = ix86_convert_const_wide_int_to_broadcast
(GET_MODE (op0), op1);
if (tmp != nullptr)
op1 = tmp;
}
}
}
emit_insn (gen_rtx_SET (op0, op1));
}
/* OP is a memref of CONST_VECTOR, return scalar constant mem
if CONST_VECTOR is a vec_duplicate, else return NULL. */
static rtx
ix86_broadcast_from_constant (machine_mode mode, rtx op)
{
int nunits = GET_MODE_NUNITS (mode);
if (nunits < 2)
return nullptr;
/* Don't use integer vector broadcast if we can't move from GPR to SSE
register directly. */
if (!TARGET_INTER_UNIT_MOVES_TO_VEC
&& INTEGRAL_MODE_P (mode))
return nullptr;
/* Convert CONST_VECTOR to a non-standard SSE constant integer
broadcast only if vector broadcast is available. */
if (!(TARGET_AVX2
|| (TARGET_AVX
&& (GET_MODE_INNER (mode) == SImode
|| GET_MODE_INNER (mode) == DImode))
|| FLOAT_MODE_P (mode))
|| standard_sse_constant_p (op, mode))
return nullptr;
/* Don't broadcast from a 64-bit integer constant in 32-bit mode.
We can still put 64-bit integer constant in memory when
avx512 embed broadcast is available. */
if (GET_MODE_INNER (mode) == DImode && !TARGET_64BIT
&& (!TARGET_AVX512F
|| (GET_MODE_SIZE (mode) < 64 && !TARGET_AVX512VL)))
return nullptr;
if (GET_MODE_INNER (mode) == TImode)
return nullptr;
rtx constant = get_pool_constant (XEXP (op, 0));
if (GET_CODE (constant) != CONST_VECTOR)
return nullptr;
/* There could be some rtx like
(mem/u/c:V16QI (symbol_ref/u:DI ("*.LC1")))
but with "*.LC1" refer to V2DI constant vector. */
if (GET_MODE (constant) != mode)
{
constant = simplify_subreg (mode, constant, GET_MODE (constant),
0);
if (constant == nullptr || GET_CODE (constant) != CONST_VECTOR)
return nullptr;
}
rtx first = XVECEXP (constant, 0, 0);
for (int i = 1; i < nunits; ++i)
{
rtx tmp = XVECEXP (constant, 0, i);
/* Vector duplicate value. */
if (!rtx_equal_p (tmp, first))
return nullptr;
}
return first;
}
void
ix86_expand_vector_move (machine_mode mode, rtx operands[])
{
rtx op0 = operands[0], op1 = operands[1];
/* Use GET_MODE_BITSIZE instead of GET_MODE_ALIGNMENT for IA MCU
psABI since the biggest alignment is 4 byte for IA MCU psABI. */
unsigned int align = (TARGET_IAMCU
? GET_MODE_BITSIZE (mode)
: GET_MODE_ALIGNMENT (mode));
if (push_operand (op0, VOIDmode))
op0 = emit_move_resolve_push (mode, op0);
/* Force constants other than zero into memory. We do not know how
the instructions used to build constants modify the upper 64 bits
of the register, once we have that information we may be able
to handle some of them more efficiently. */
if (can_create_pseudo_p ()
&& (CONSTANT_P (op1)
|| (SUBREG_P (op1)
&& CONSTANT_P (SUBREG_REG (op1))))
&& ((register_operand (op0, mode)
&& !standard_sse_constant_p (op1, mode))
/* ix86_expand_vector_move_misalign() does not like constants. */
|| (SSE_REG_MODE_P (mode)
&& MEM_P (op0)
&& MEM_ALIGN (op0) < align)))
{
if (SUBREG_P (op1))
{
machine_mode imode = GET_MODE (SUBREG_REG (op1));
rtx r = force_const_mem (imode, SUBREG_REG (op1));
if (r)
r = validize_mem (r);
else
r = force_reg (imode, SUBREG_REG (op1));
op1 = simplify_gen_subreg (mode, r, imode, SUBREG_BYTE (op1));
}
else
{
machine_mode mode = GET_MODE (op0);
rtx tmp = ix86_convert_const_wide_int_to_broadcast
(mode, op1);
if (tmp == nullptr)
op1 = validize_mem (force_const_mem (mode, op1));
else
op1 = tmp;
}
}
if (can_create_pseudo_p ()
&& GET_MODE_SIZE (mode) >= 16
&& VECTOR_MODE_P (mode)
&& (MEM_P (op1)
&& SYMBOL_REF_P (XEXP (op1, 0))
&& CONSTANT_POOL_ADDRESS_P (XEXP (op1, 0))))
{
rtx first = ix86_broadcast_from_constant (mode, op1);
if (first != nullptr)
{
/* Broadcast to XMM/YMM/ZMM register from an integer
constant or scalar mem. */
op1 = gen_reg_rtx (mode);
if (FLOAT_MODE_P (mode)
|| (!TARGET_64BIT && GET_MODE_INNER (mode) == DImode))
first = force_const_mem (GET_MODE_INNER (mode), first);
bool ok = ix86_expand_vector_init_duplicate (false, mode,
op1, first);
gcc_assert (ok);
emit_move_insn (op0, op1);
return;
}
}
/* We need to check memory alignment for SSE mode since attribute
can make operands unaligned. */
if (can_create_pseudo_p ()
&& SSE_REG_MODE_P (mode)
&& ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
|| (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
{
rtx tmp[2];
/* ix86_expand_vector_move_misalign() does not like both
arguments in memory. */
if (!register_operand (op0, mode)
&& !register_operand (op1, mode))
{
rtx scratch = ix86_gen_scratch_sse_rtx (mode);
emit_move_insn (scratch, op1);
op1 = scratch;
}
tmp[0] = op0; tmp[1] = op1;
ix86_expand_vector_move_misalign (mode, tmp);
return;
}
/* Make operand1 a register if it isn't already. */
if (can_create_pseudo_p ()
&& !register_operand (op0, mode)
&& !register_operand (op1, mode))
{
rtx tmp = ix86_gen_scratch_sse_rtx (GET_MODE (op0));
emit_move_insn (tmp, op1);
emit_move_insn (op0, tmp);
return;
}
emit_insn (gen_rtx_SET (op0, op1));
}
/* Split 32-byte AVX unaligned load and store if needed. */
static void
ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
{
rtx m;
rtx (*extract) (rtx, rtx, rtx);
machine_mode mode;
if ((MEM_P (op1) && !TARGET_AVX256_SPLIT_UNALIGNED_LOAD)
|| (MEM_P (op0) && !TARGET_AVX256_SPLIT_UNALIGNED_STORE))
{
emit_insn (gen_rtx_SET (op0, op1));
return;
}
rtx orig_op0 = NULL_RTX;
mode = GET_MODE (op0);
switch (GET_MODE_CLASS (mode))
{
case MODE_VECTOR_INT:
case MODE_INT:
if (mode != V32QImode)
{
if (!MEM_P (op0))
{
orig_op0 = op0;
op0 = gen_reg_rtx (V32QImode);
}
else
op0 = gen_lowpart (V32QImode, op0);
op1 = gen_lowpart (V32QImode, op1);
mode = V32QImode;
}
break;
case MODE_VECTOR_FLOAT:
break;
default:
gcc_unreachable ();
}
switch (mode)
{
default:
gcc_unreachable ();
case E_V32QImode:
extract = gen_avx_vextractf128v32qi;
mode = V16QImode;
break;
case E_V16HFmode:
extract = gen_avx_vextractf128v16hf;
mode = V8HFmode;
break;
case E_V8SFmode:
extract = gen_avx_vextractf128v8sf;
mode = V4SFmode;
break;
case E_V4DFmode:
extract = gen_avx_vextractf128v4df;
mode = V2DFmode;
break;
}
if (MEM_P (op1))
{
rtx r = gen_reg_rtx (mode);
m = adjust_address (op1, mode, 0);
emit_move_insn (r, m);
m = adjust_address (op1, mode, 16);
r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
emit_move_insn (op0, r);
}
else if (MEM_P (op0))
{
m = adjust_address (op0, mode, 0);
emit_insn (extract (m, op1, const0_rtx));
m = adjust_address (op0, mode, 16);
emit_insn (extract (m, copy_rtx (op1), const1_rtx));
}
else
gcc_unreachable ();
if (orig_op0)
emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
}
/* Implement the movmisalign patterns for SSE. Non-SSE modes go
straight to ix86_expand_vector_move. */
/* Code generation for scalar reg-reg moves of single and double precision data:
if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
movaps reg, reg
else
movss reg, reg
if (x86_sse_partial_reg_dependency == true)
movapd reg, reg
else
movsd reg, reg
Code generation for scalar loads of double precision data:
if (x86_sse_split_regs == true)
movlpd mem, reg (gas syntax)
else
movsd mem, reg
Code generation for unaligned packed loads of single precision data
(x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
if (x86_sse_unaligned_move_optimal)
movups mem, reg
if (x86_sse_partial_reg_dependency == true)
{
xorps reg, reg
movlps mem, reg
movhps mem+8, reg
}
else
{
movlps mem, reg
movhps mem+8, reg
}
Code generation for unaligned packed loads of double precision data
(x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
if (x86_sse_unaligned_move_optimal)
movupd mem, reg
if (x86_sse_split_regs == true)
{
movlpd mem, reg
movhpd mem+8, reg
}
else
{
movsd mem, reg
movhpd mem+8, reg
}
*/
void
ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
{
rtx op0, op1, m;
op0 = operands[0];
op1 = operands[1];
/* Use unaligned load/store for AVX512 or when optimizing for size. */
if (GET_MODE_SIZE (mode) == 64 || optimize_insn_for_size_p ())
{
emit_insn (gen_rtx_SET (op0, op1));
return;
}
if (TARGET_AVX)
{
if (GET_MODE_SIZE (mode) == 32)
ix86_avx256_split_vector_move_misalign (op0, op1);
else
/* Always use 128-bit mov<mode>_internal pattern for AVX. */
emit_insn (gen_rtx_SET (op0, op1));
return;
}
if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
|| TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
{
emit_insn (gen_rtx_SET (op0, op1));
return;
}
/* ??? If we have typed data, then it would appear that using
movdqu is the only way to get unaligned data loaded with
integer type. */
if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
{
emit_insn (gen_rtx_SET (op0, op1));
return;
}
if (MEM_P (op1))
{
if (TARGET_SSE2 && mode == V2DFmode)
{
rtx zero;
/* When SSE registers are split into halves, we can avoid
writing to the top half twice. */
if (TARGET_SSE_SPLIT_REGS)
{
emit_clobber (op0);
zero = op0;
}
else
{
/* ??? Not sure about the best option for the Intel chips.
The following would seem to satisfy; the register is
entirely cleared, breaking the dependency chain. We
then store to the upper half, with a dependency depth
of one. A rumor has it that Intel recommends two movsd
followed by an unpacklpd, but this is unconfirmed. And
given that the dependency depth of the unpacklpd would
still be one, I'm not sure why this would be better. */
zero = CONST0_RTX (V2DFmode);
}
m = adjust_address (op1, DFmode, 0);
emit_insn (gen_sse2_loadlpd (op0, zero, m));
m = adjust_address (op1, DFmode, 8);
emit_insn (gen_sse2_loadhpd (op0, op0, m));
}
else
{
rtx t;
if (mode != V4SFmode)
t = gen_reg_rtx (V4SFmode);
else
t = op0;
if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
emit_move_insn (t, CONST0_RTX (V4SFmode));
else
emit_clobber (t);
m = adjust_address (op1, V2SFmode, 0);
emit_insn (gen_sse_loadlps (t, t, m));
m = adjust_address (op1, V2SFmode, 8);
emit_insn (gen_sse_loadhps (t, t, m));
if (mode != V4SFmode)
emit_move_insn (op0, gen_lowpart (mode, t));
}
}
else if (MEM_P (op0))
{
if (TARGET_SSE2 && mode == V2DFmode)
{
m = adjust_address (op0, DFmode, 0);
emit_insn (gen_sse2_storelpd (m, op1));
m = adjust_address (op0, DFmode, 8);
emit_insn (gen_sse2_storehpd (m, op1));
}
else
{
if (mode != V4SFmode)
op1 = gen_lowpart (V4SFmode, op1);
m = adjust_address (op0, V2SFmode, 0);
emit_insn (gen_sse_storelps (m, op1));
m = adjust_address (op0, V2SFmode, 8);
emit_insn (gen_sse_storehps (m, copy_rtx (op1)));
}
}
else
gcc_unreachable ();
}
/* Move bits 64:95 to bits 32:63. */
void
ix86_move_vector_high_sse_to_mmx (rtx op)
{
rtx mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4, GEN_INT (0), GEN_INT (2),
GEN_INT (0), GEN_INT (0)));
rtx dest = lowpart_subreg (V4SImode, op, GET_MODE (op));
op = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
rtx insn = gen_rtx_SET (dest, op);
emit_insn (insn);
}
/* Split MMX pack with signed/unsigned saturation with SSE/SSE2. */
void
ix86_split_mmx_pack (rtx operands[], enum rtx_code code)
{
rtx op0 = operands[0];
rtx op1 = operands[1];
rtx op2 = operands[2];
machine_mode dmode = GET_MODE (op0);
machine_mode smode = GET_MODE (op1);
machine_mode inner_dmode = GET_MODE_INNER (dmode);
machine_mode inner_smode = GET_MODE_INNER (smode);
/* Get the corresponding SSE mode for destination. */
int nunits = 16 / GET_MODE_SIZE (inner_dmode);
machine_mode sse_dmode = mode_for_vector (GET_MODE_INNER (dmode),
nunits).require ();
machine_mode sse_half_dmode = mode_for_vector (GET_MODE_INNER (dmode),
nunits / 2).require ();
/* Get the corresponding SSE mode for source. */
nunits = 16 / GET_MODE_SIZE (inner_smode);
machine_mode sse_smode = mode_for_vector (GET_MODE_INNER (smode),
nunits).require ();
/* Generate SSE pack with signed/unsigned saturation. */
rtx dest = lowpart_subreg (sse_dmode, op0, GET_MODE (op0));
op1 = lowpart_subreg (sse_smode, op1, GET_MODE (op1));
op2 = lowpart_subreg (sse_smode, op2, GET_MODE (op2));
op1 = gen_rtx_fmt_e (code, sse_half_dmode, op1);
op2 = gen_rtx_fmt_e (code, sse_half_dmode, op2);
rtx insn = gen_rtx_SET (dest, gen_rtx_VEC_CONCAT (sse_dmode,
op1, op2));
emit_insn (insn);
ix86_move_vector_high_sse_to_mmx (op0);
}
/* Split MMX punpcklXX/punpckhXX with SSE punpcklXX. */
void
ix86_split_mmx_punpck (rtx operands[], bool high_p)
{
rtx op0 = operands[0];
rtx op1 = operands[1];
rtx op2 = operands[2];
machine_mode mode = GET_MODE (op0);
rtx mask;
/* The corresponding SSE mode. */
machine_mode sse_mode, double_sse_mode;
switch (mode)
{
case E_V4QImode:
case E_V8QImode:
sse_mode = V16QImode;
double_sse_mode = V32QImode;
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (16,
GEN_INT (0), GEN_INT (16),
GEN_INT (1), GEN_INT (17),
GEN_INT (2), GEN_INT (18),
GEN_INT (3), GEN_INT (19),
GEN_INT (4), GEN_INT (20),
GEN_INT (5), GEN_INT (21),
GEN_INT (6), GEN_INT (22),
GEN_INT (7), GEN_INT (23)));
break;
case E_V4HImode:
case E_V2HImode:
sse_mode = V8HImode;
double_sse_mode = V16HImode;
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (8,
GEN_INT (0), GEN_INT (8),
GEN_INT (1), GEN_INT (9),
GEN_INT (2), GEN_INT (10),
GEN_INT (3), GEN_INT (11)));
break;
case E_V2SImode:
sse_mode = V4SImode;
double_sse_mode = V8SImode;
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4,
GEN_INT (0), GEN_INT (4),
GEN_INT (1), GEN_INT (5)));
break;
case E_V2SFmode:
sse_mode = V4SFmode;
double_sse_mode = V8SFmode;
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4,
GEN_INT (0), GEN_INT (4),
GEN_INT (1), GEN_INT (5)));
break;
default:
gcc_unreachable ();
}
/* Generate SSE punpcklXX. */
rtx dest = lowpart_subreg (sse_mode, op0, GET_MODE (op0));
op1 = lowpart_subreg (sse_mode, op1, GET_MODE (op1));
op2 = lowpart_subreg (sse_mode, op2, GET_MODE (op2));
op1 = gen_rtx_VEC_CONCAT (double_sse_mode, op1, op2);
op2 = gen_rtx_VEC_SELECT (sse_mode, op1, mask);
rtx insn = gen_rtx_SET (dest, op2);
emit_insn (insn);
/* Move high bits to low bits. */
if (high_p)
{
if (sse_mode == V4SFmode)
{
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4, GEN_INT (2), GEN_INT (3),
GEN_INT (4), GEN_INT (5)));
op2 = gen_rtx_VEC_CONCAT (V8SFmode, dest, dest);
op1 = gen_rtx_VEC_SELECT (V4SFmode, op2, mask);
}
else
{
int sz = GET_MODE_SIZE (mode);
if (sz == 4)
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4, GEN_INT (1), GEN_INT (0),
GEN_INT (0), GEN_INT (1)));
else if (sz == 8)
mask = gen_rtx_PARALLEL (VOIDmode,
gen_rtvec (4, GEN_INT (2), GEN_INT (3),
GEN_INT (0), GEN_INT (1)));
else
gcc_unreachable ();
dest = lowpart_subreg (V4SImode, dest, GET_MODE (dest));
op1 = gen_rtx_VEC_SELECT (V4SImode, dest, mask);
}
insn = gen_rtx_SET (dest, op1);
emit_insn (insn);
}
}
/* Helper function of ix86_fixup_binary_operands to canonicalize
operand order. Returns true if the operands should be swapped. */
static bool
ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
rtx operands[])
{
rtx dst = operands[0];
rtx src1 = operands[1];
rtx src2 = operands[2];
/* If the operation is not commutative, we can't do anything. */
if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
&& GET_RTX_CLASS (code) != RTX_COMM_COMPARE)
return false;
/* Highest priority is that src1 should match dst. */
if (rtx_equal_p (dst, src1))
return false;
if (rtx_equal_p (dst, src2))
return true;
/* Next highest priority is that immediate constants come second. */
if (immediate_operand (src2, mode))
return false;
if (immediate_operand (src1, mode))
return true;
/* Lowest priority is that memory references should come second. */
if (MEM_P (src2))
return false;
if (MEM_P (src1))
return true;
return false;
}
/* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
destination to use for the operation. If different from the true
destination in operands[0], a copy operation will be required. */
rtx
ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
rtx operands[])
{
rtx dst = operands[0];
rtx src1 = operands[1];
rtx src2 = operands[2];
/* Canonicalize operand order. */
if (ix86_swap_binary_operands_p (code, mode, operands))
{
/* It is invalid to swap operands of different modes. */
gcc_assert (GET_MODE (src1) == GET_MODE (src2));
std::swap (src1, src2);
}
/* Both source operands cannot be in memory. */
if (MEM_P (src1) && MEM_P (src2))
{
/* Optimization: Only read from memory once. */
if (rtx_equal_p (src1, src2))
{
src2 = force_reg (mode, src2);
src1 = src2;
}
else if (rtx_equal_p (dst, src1))
src2 = force_reg (mode, src2);
else
src1 = force_reg (mode, src1);
}
/* If the destination is memory, and we do not have matching source
operands, do things in registers. */
if (MEM_P (dst) && !rtx_equal_p (dst, src1))
dst = gen_reg_rtx (mode);
/* Source 1 cannot be a constant. */
if (CONSTANT_P (src1))
src1 = force_reg (mode, src1);
/* Source 1 cannot be a non-matching memory. */
if (MEM_P (src1) && !rtx_equal_p (dst, src1))
src1 = force_reg (mode, src1);
/* Improve address combine. */
if (code == PLUS
&& GET_MODE_CLASS (mode) == MODE_INT
&& MEM_P (src2))
src2 = force_reg (mode, src2);
operands[1] = src1;
operands[2] = src2;
return dst;
}
/* Similarly, but assume that the destination has already been
set up properly. */
void
ix86_fixup_binary_operands_no_copy (enum rtx_code code,
machine_mode mode, rtx operands[])
{
rtx dst = ix86_fixup_binary_operands (code, mode, operands);
gcc_assert (dst == operands[0]);
}
/* Attempt to expand a binary operator. Make the expansion closer to the
actual machine, then just general_operand, which will allow 3 separate
memory references (one output, two input) in a single insn. */
void
ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
rtx operands[])
{
rtx src1, src2, dst, op, clob;
dst = ix86_fixup_binary_operands (code, mode, operands);
src1 = operands[1];
src2 = operands[2];
/* Emit the instruction. */
op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
if (reload_completed
&& code == PLUS
&& !rtx_equal_p (dst, src1))
{
/* This is going to be an LEA; avoid splitting it later. */
emit_insn (op);
}
else
{
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
}
/* Fix up the destination if needed. */
if (dst != operands[0])
emit_move_insn (operands[0], dst);
}
/* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
the given OPERANDS. */
void
ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
rtx operands[])
{
rtx op1 = NULL_RTX, op2 = NULL_RTX;
if (SUBREG_P (operands[1]))
{
op1 = operands[1];
op2 = operands[2];
}
else if (SUBREG_P (operands[2]))
{
op1 = operands[2];
op2 = operands[1];
}
/* Optimize (__m128i) d | (__m128i) e and similar code
when d and e are float vectors into float vector logical
insn. In C/C++ without using intrinsics there is no other way
to express vector logical operation on float vectors than
to cast them temporarily to integer vectors. */
if (op1
&& !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
&& (SUBREG_P (op2) || GET_CODE (op2) == CONST_VECTOR)
&& GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
&& GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
&& SUBREG_BYTE (op1) == 0
&& (GET_CODE (op2) == CONST_VECTOR
|| (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
&& SUBREG_BYTE (op2) == 0))
&& can_create_pseudo_p ())
{
rtx dst;
switch (GET_MODE (SUBREG_REG (op1)))
{
case E_V4SFmode:
case E_V8SFmode:
case E_V16SFmode:
case E_V2DFmode:
case E_V4DFmode:
case E_V8DFmode:
dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
if (GET_CODE (op2) == CONST_VECTOR)
{
op2 = gen_lowpart (GET_MODE (dst), op2);
op2 = force_reg (GET_MODE (dst), op2);
}
else
{
op1 = operands[1];
op2 = SUBREG_REG (operands[2]);
if (!vector_operand (op2, GET_MODE (dst)))
op2 = force_reg (GET_MODE (dst), op2);
}
op1 = SUBREG_REG (op1);
if (!vector_operand (op1, GET_MODE (dst)))
op1 = force_reg (GET_MODE (dst), op1);
emit_insn (gen_rtx_SET (dst,
gen_rtx_fmt_ee (code, GET_MODE (dst),
op1, op2)));
emit_move_insn (operands[0], gen_lowpart (mode, dst));
return;
default:
break;
}
}
if (!vector_operand (operands[1], mode))
operands[1] = force_reg (mode, operands[1]);
if (!vector_operand (operands[2], mode))
operands[2] = force_reg (mode, operands[2]);
ix86_fixup_binary_operands_no_copy (code, mode, operands);
emit_insn (gen_rtx_SET (operands[0],
gen_rtx_fmt_ee (code, mode, operands[1],
operands[2])));
}
/* Return TRUE or FALSE depending on whether the binary operator meets the
appropriate constraints. */
bool
ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
rtx operands[3])
{
rtx dst = operands[0];
rtx src1 = operands[1];
rtx src2 = operands[2];
/* Both source operands cannot be in memory. */
if ((MEM_P (src1) || bcst_mem_operand (src1, mode))
&& (MEM_P (src2) || bcst_mem_operand (src2, mode)))
return false;
/* Canonicalize operand order for commutative operators. */
if (ix86_swap_binary_operands_p (code, mode, operands))
std::swap (src1, src2);
/* If the destination is memory, we must have a matching source operand. */
if (MEM_P (dst) && !rtx_equal_p (dst, src1))
return false;
/* Source 1 cannot be a constant. */
if (CONSTANT_P (src1))
return false;
/* Source 1 cannot be a non-matching memory. */
if (MEM_P (src1) && !rtx_equal_p (dst, src1))
/* Support "andhi/andsi/anddi" as a zero-extending move. */
return (code == AND
&& (mode == HImode
|| mode == SImode
|| (TARGET_64BIT && mode == DImode))
&& satisfies_constraint_L (src2));
return true;
}
/* Attempt to expand a unary operator. Make the expansion closer to the
actual machine, then just general_operand, which will allow 2 separate
memory references (one output, one input) in a single insn. */
void
ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
rtx operands[])
{
bool matching_memory = false;
rtx src, dst, op, clob;
dst = operands[0];
src = operands[1];
/* If the destination is memory, and we do not have matching source
operands, do things in registers. */
if (MEM_P (dst))
{
if (rtx_equal_p (dst, src))
matching_memory = true;
else
dst = gen_reg_rtx (mode);
}
/* When source operand is memory, destination must match. */
if (MEM_P (src) && !matching_memory)
src = force_reg (mode, src);
/* Emit the instruction. */
op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
if (code == NOT)
emit_insn (op);
else
{
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
}
/* Fix up the destination if needed. */
if (dst != operands[0])
emit_move_insn (operands[0], dst);
}
/* Predict just emitted jump instruction to be taken with probability PROB. */
static void
predict_jump (int prob)
{
rtx_insn *insn = get_last_insn ();
gcc_assert (JUMP_P (insn));
add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob));
}
/* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
divisor are within the range [0-255]. */
void
ix86_split_idivmod (machine_mode mode, rtx operands[],
bool unsigned_p)
{
rtx_code_label *end_label, *qimode_label;
rtx div, mod;
rtx_insn *insn;
rtx scratch, tmp0, tmp1, tmp2;
rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
switch (mode)
{
case E_SImode:
if (GET_MODE (operands[0]) == SImode)
{
if (GET_MODE (operands[1]) == SImode)
gen_divmod4_1 = unsigned_p ? gen_udivmodsi4_1 : gen_divmodsi4_1;
else
gen_divmod4_1
= unsigned_p ? gen_udivmodsi4_zext_2 : gen_divmodsi4_zext_2;
}
else
gen_divmod4_1
= unsigned_p ? gen_udivmodsi4_zext_1 : gen_divmodsi4_zext_1;
break;
case E_DImode:
gen_divmod4_1 = unsigned_p ? gen_udivmoddi4_1 : gen_divmoddi4_1;
break;
default:
gcc_unreachable ();
}
end_label = gen_label_rtx ();
qimode_label = gen_label_rtx ();
scratch = gen_reg_rtx (mode);
/* Use 8bit unsigned divimod if dividend and divisor are within
the range [0-255]. */
emit_move_insn (scratch, operands[2]);
scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
scratch, 1, OPTAB_DIRECT);
emit_insn (gen_test_ccno_1 (mode, scratch, GEN_INT (-0x100)));
tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
gen_rtx_LABEL_REF (VOIDmode, qimode_label),
pc_rtx);
insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
predict_jump (REG_BR_PROB_BASE * 50 / 100);
JUMP_LABEL (insn) = qimode_label;
/* Generate original signed/unsigned divimod. */
emit_insn (gen_divmod4_1 (operands[0], operands[1],
operands[2], operands[3]));
/* Branch to the end. */
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
/* Generate 8bit unsigned divide. */
emit_label (qimode_label);
/* Don't use operands[0] for result of 8bit divide since not all
registers support QImode ZERO_EXTRACT. */
tmp0 = lowpart_subreg (HImode, scratch, mode);
tmp1 = lowpart_subreg (HImode, operands[2], mode);
tmp2 = lowpart_subreg (QImode, operands[3], mode);
emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
if (unsigned_p)
{
div = gen_rtx_UDIV (mode, operands[2], operands[3]);
mod = gen_rtx_UMOD (mode, operands[2], operands[3]);
}
else
{
div = gen_rtx_DIV (mode, operands[2], operands[3]);
mod = gen_rtx_MOD (mode, operands[2], operands[3]);
}
if (mode == SImode)
{
if (GET_MODE (operands[0]) != SImode)
div = gen_rtx_ZERO_EXTEND (DImode, div);
if (GET_MODE (operands[1]) != SImode)
mod = gen_rtx_ZERO_EXTEND (DImode, mod);
}
/* Extract remainder from AH. */
scratch = gen_lowpart (GET_MODE (operands[1]), scratch);
tmp1 = gen_rtx_ZERO_EXTRACT (GET_MODE (operands[1]), scratch,
GEN_INT (8), GEN_INT (8));
insn = emit_move_insn (operands[1], tmp1);
set_unique_reg_note (insn, REG_EQUAL, mod);
/* Zero extend quotient from AL. */
tmp1 = gen_lowpart (QImode, tmp0);
insn = emit_insn (gen_extend_insn
(operands[0], tmp1,
GET_MODE (operands[0]), QImode, 1));
set_unique_reg_note (insn, REG_EQUAL, div);
emit_label (end_label);
}
/* Emit x86 binary operand CODE in mode MODE, where the first operand
matches destination. RTX includes clobber of FLAGS_REG. */
void
ix86_emit_binop (enum rtx_code code, machine_mode mode,
rtx dst, rtx src)
{
rtx op, clob;
op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
}
/* Return true if regno1 def is nearest to the insn. */
static bool
find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
{
rtx_insn *prev = insn;
rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
if (insn == start)
return false;
while (prev && prev != start)
{
if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
{
prev = PREV_INSN (prev);
continue;
}
if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
return true;
else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
return false;
prev = PREV_INSN (prev);
}
/* None of the regs is defined in the bb. */
return false;
}
/* INSN_UID of the last insn emitted by zero store peephole2s. */
int ix86_last_zero_store_uid;
/* Split lea instructions into a sequence of instructions
which are executed on ALU to avoid AGU stalls.
It is assumed that it is allowed to clobber flags register
at lea position. */
void
ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
{
unsigned int regno0, regno1, regno2;
struct ix86_address parts;
rtx target, tmp;
int ok, adds;
ok = ix86_decompose_address (operands[1], &parts);
gcc_assert (ok);
target = gen_lowpart (mode, operands[0]);
regno0 = true_regnum (target);
regno1 = INVALID_REGNUM;
regno2 = INVALID_REGNUM;
if (parts.base)
{
parts.base = gen_lowpart (mode, parts.base);
regno1 = true_regnum (parts.base);
}
if (parts.index)
{
parts.index = gen_lowpart (mode, parts.index);
regno2 = true_regnum (parts.index);
}
if (parts.disp)
parts.disp = gen_lowpart (mode, parts.disp);
if (parts.scale > 1)
{
/* Case r1 = r1 + ... */
if (regno1 == regno0)
{
/* If we have a case r1 = r1 + C * r2 then we
should use multiplication which is very
expensive. Assume cost model is wrong if we
have such case here. */
gcc_assert (regno2 != regno0);
for (adds = parts.scale; adds > 0; adds--)
ix86_emit_binop (PLUS, mode, target, parts.index);
}
else
{
/* r1 = r2 + r3 * C case. Need to move r3 into r1. */
if (regno0 != regno2)
emit_insn (gen_rtx_SET (target, parts.index));
/* Use shift for scaling, but emit it as MULT instead
to avoid it being immediately peephole2 optimized back
into lea. */
ix86_emit_binop (MULT, mode, target, GEN_INT (parts.scale));
if (parts.base)
ix86_emit_binop (PLUS, mode, target, parts.base);
if (parts.disp && parts.disp != const0_rtx)
ix86_emit_binop (PLUS, mode, target, parts.disp);
}
}
else if (!parts.base && !parts.index)
{
gcc_assert(parts.disp);
emit_insn (gen_rtx_SET (target, parts.disp));
}
else
{
if (!parts.base)
{
if (regno0 != regno2)
emit_insn (gen_rtx_SET (target, parts.index));
}
else if (!parts.index)
{
if (regno0 != regno1)
emit_insn (gen_rtx_SET (target, parts.base));
}
else
{
if (regno0 == regno1)
tmp = parts.index;
else if (regno0 == regno2)
tmp = parts.base;
else
{
rtx tmp1;
/* Find better operand for SET instruction, depending
on which definition is farther from the insn. */
if (find_nearest_reg_def (insn, regno1, regno2))
tmp = parts.index, tmp1 = parts.base;
else
tmp = parts.base, tmp1 = parts.index;
emit_insn (gen_rtx_SET (target, tmp));
if (parts.disp && parts.disp != const0_rtx)
ix86_emit_binop (PLUS, mode, target, parts.disp);
ix86_emit_binop (PLUS, mode, target, tmp1);
return;
}
ix86_emit_binop (PLUS, mode, target, tmp);
}
if (parts.disp && parts.disp != const0_rtx)
ix86_emit_binop (PLUS, mode, target, parts.disp);
}
}
/* Post-reload splitter for converting an SF or DFmode value in an
SSE register into an unsigned SImode. */
void
ix86_split_convert_uns_si_sse (rtx operands[])
{
machine_mode vecmode;
rtx value, large, zero_or_two31, input, two31, x;
large = operands[1];
zero_or_two31 = operands[2];
input = operands[3];
two31 = operands[4];
vecmode = GET_MODE (large);
value = gen_rtx_REG (vecmode, REGNO (operands[0]));
/* Load up the value into the low element. We must ensure that the other
elements are valid floats -- zero is the easiest such value. */
if (MEM_P (input))
{
if (vecmode == V4SFmode)
emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
else
emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
}
else
{
input = gen_rtx_REG (vecmode, REGNO (input));
emit_move_insn (value, CONST0_RTX (vecmode));
if (vecmode == V4SFmode)
emit_insn (gen_sse_movss (value, value, input));
else
emit_insn (gen_sse2_movsd (value, value, input));
}
emit_move_insn (large, two31);
emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
x = gen_rtx_fmt_ee (LE, vecmode, large, value);
emit_insn (gen_rtx_SET (large, x));
x = gen_rtx_AND (vecmode, zero_or_two31, large);
emit_insn (gen_rtx_SET (zero_or_two31, x));
x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
emit_insn (gen_rtx_SET (value, x));
large = gen_rtx_REG (V4SImode, REGNO (large));
emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
x = gen_rtx_REG (V4SImode, REGNO (value));
if (vecmode == V4SFmode)
emit_insn (gen_fix_truncv4sfv4si2 (x, value));
else
emit_insn (gen_sse2_cvttpd2dq (x, value));
value = x;
emit_insn (gen_xorv4si3 (value, value, large));
}
static bool ix86_expand_vector_init_one_nonzero (bool mmx_ok,
machine_mode mode, rtx target,
rtx var, int one_var);
/* Convert an unsigned DImode value into a DFmode, using only SSE.
Expects the 64-bit DImode to be supplied in a pair of integral
registers. Requires SSE2; will use SSE3 if available. For x86_32,
-mfpmath=sse, !optimize_size only. */
void
ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
{
REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
rtx int_xmm, fp_xmm;
rtx biases, exponents;
rtx x;
int_xmm = gen_reg_rtx (V4SImode);
if (TARGET_INTER_UNIT_MOVES_TO_VEC)
emit_insn (gen_movdi_to_sse (int_xmm, input));
else if (TARGET_SSE_SPLIT_REGS)
{
emit_clobber (int_xmm);
emit_move_insn (gen_lowpart (DImode, int_xmm), input);
}
else
{
x = gen_reg_rtx (V2DImode);
ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
}
x = gen_rtx_CONST_VECTOR (V4SImode,
gen_rtvec (4, GEN_INT (0x43300000UL),
GEN_INT (0x45300000UL),
const0_rtx, const0_rtx));
exponents = validize_mem (force_const_mem (V4SImode, x));
/* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
/* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
Similarly (0x45300000UL ## fp_value_hi_xmm) yields
(0x1.0p84 + double(fp_value_hi_xmm)).
Note these exponents differ by 32. */
fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
/* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
real_ldexp (&bias_lo_rvt, &dconst1, 52);
real_ldexp (&bias_hi_rvt, &dconst1, 84);
biases = const_double_from_real_value (bias_lo_rvt, DFmode);
x = const_double_from_real_value (bias_hi_rvt, DFmode);
biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
biases = validize_mem (force_const_mem (V2DFmode, biases));
emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
/* Add the upper and lower DFmode values together. */
if (TARGET_SSE3)
emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
else
{
x = copy_to_mode_reg (V2DFmode, fp_xmm);
emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
}
ix86_expand_vector_extract (false, target, fp_xmm, 0);
}
/* Not used, but eases macroization of patterns. */
void
ix86_expand_convert_uns_sixf_sse (rtx, rtx)
{
gcc_unreachable ();
}
static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask);
/* Convert an unsigned SImode value into a DFmode. Only currently used
for SSE, but applicable anywhere. */
void
ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
{
REAL_VALUE_TYPE TWO31r;
rtx x, fp;
x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
NULL, 1, OPTAB_DIRECT);
fp = gen_reg_rtx (DFmode);
emit_insn (gen_floatsidf2 (fp, x));
real_ldexp (&TWO31r, &dconst1, 31);
x = const_double_from_real_value (TWO31r, DFmode);
x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
/* Remove the sign with FE_DOWNWARD, where x - x = -0.0. */
if (HONOR_SIGNED_ZEROS (DFmode) && flag_rounding_math)
x = ix86_expand_sse_fabs (x, NULL);
if (x != target)
emit_move_insn (target, x);
}
/* Convert a signed DImode value into a DFmode. Only used for SSE in
32-bit mode; otherwise we have a direct convert instruction. */
void
ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
{
REAL_VALUE_TYPE TWO32r;
rtx fp_lo, fp_hi, x;
fp_lo = gen_reg_rtx (DFmode);
fp_hi = gen_reg_rtx (DFmode);
emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
real_ldexp (&TWO32r, &dconst1, 32);
x = const_double_from_real_value (TWO32r, DFmode);
fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
0, OPTAB_DIRECT);
if (x != target)
emit_move_insn (target, x);
}
/* Convert an unsigned SImode value into a SFmode, using only SSE.
For x86_32, -mfpmath=sse, !optimize_size only. */
void
ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
{
REAL_VALUE_TYPE ONE16r;
rtx fp_hi, fp_lo, int_hi, int_lo, x;
real_ldexp (&ONE16r, &dconst1, 16);
x = const_double_from_real_value (ONE16r, SFmode);
int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
NULL, 0, OPTAB_DIRECT);
int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
NULL, 0, OPTAB_DIRECT);
fp_hi = gen_reg_rtx (SFmode);
fp_lo = gen_reg_rtx (SFmode);
emit_insn (gen_floatsisf2 (fp_hi, int_hi));
emit_insn (gen_floatsisf2 (fp_lo, int_lo));
if (TARGET_FMA)
{
x = validize_mem (force_const_mem (SFmode, x));
fp_hi = gen_rtx_FMA (SFmode, fp_hi, x, fp_lo);
emit_move_insn (target, fp_hi);
}
else
{
fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
0, OPTAB_DIRECT);
fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
0, OPTAB_DIRECT);
if (!rtx_equal_p (target, fp_hi))
emit_move_insn (target, fp_hi);
}
}
/* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
a vector of unsigned ints VAL to vector of floats TARGET. */
void
ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
{
rtx tmp[8];
REAL_VALUE_TYPE TWO16r;
machine_mode intmode = GET_MODE (val);
machine_mode fltmode = GET_MODE (target);
rtx (*cvt) (rtx, rtx);
if (intmode == V4SImode)
cvt = gen_floatv4siv4sf2;
else
cvt = gen_floatv8siv8sf2;
tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
tmp[0] = force_reg (intmode, tmp[0]);
tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
OPTAB_DIRECT);
tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
NULL_RTX, 1, OPTAB_DIRECT);
tmp[3] = gen_reg_rtx (fltmode);
emit_insn (cvt (tmp[3], tmp[1]));
tmp[4] = gen_reg_rtx (fltmode);
emit_insn (cvt (tmp[4], tmp[2]));
real_ldexp (&TWO16r, &dconst1, 16);
tmp[5] = const_double_from_real_value (TWO16r, SFmode);
tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
if (TARGET_FMA)
{
tmp[6] = gen_rtx_FMA (fltmode, tmp[4], tmp[5], tmp[3]);
emit_move_insn (target, tmp[6]);
}
else
{
tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5],
NULL_RTX, 1, OPTAB_DIRECT);
tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6],
target, 1, OPTAB_DIRECT);
if (tmp[7] != target)
emit_move_insn (target, tmp[7]);
}
}
/* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
This is done by doing just signed conversion if < 0x1p31, and otherwise by
subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
rtx
ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
{
REAL_VALUE_TYPE TWO31r;
rtx two31r, tmp[4];
machine_mode mode = GET_MODE (val);
machine_mode scalarmode = GET_MODE_INNER (mode);
machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
rtx (*cmp) (rtx, rtx, rtx, rtx);
int i;
for (i = 0; i < 3; i++)
tmp[i] = gen_reg_rtx (mode);
real_ldexp (&TWO31r, &dconst1, 31);
two31r = const_double_from_real_value (TWO31r, scalarmode);
two31r = ix86_build_const_vector (mode, 1, two31r);
two31r = force_reg (mode, two31r);
switch (mode)
{
case E_V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
case E_V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
case E_V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
case E_V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
default: gcc_unreachable ();
}
tmp[3] = gen_rtx_LE (mode, two31r, val);
emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
0, OPTAB_DIRECT);
if (intmode == V4SImode || TARGET_AVX2)
*xorp = expand_simple_binop (intmode, ASHIFT,
gen_lowpart (intmode, tmp[0]),
GEN_INT (31), NULL_RTX, 0,
OPTAB_DIRECT);
else
{
rtx two31 = gen_int_mode (HOST_WIDE_INT_1U << 31, SImode);
two31 = ix86_build_const_vector (intmode, 1, two31);
*xorp = expand_simple_binop (intmode, AND,
gen_lowpart (intmode, tmp[0]),
two31, NULL_RTX, 0,
OPTAB_DIRECT);
}
return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
0, OPTAB_DIRECT);
}
/* Generate code for floating point ABS or NEG. */
void
ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
rtx operands[])
{
rtx set, dst, src;
bool use_sse = false;
bool vector_mode = VECTOR_MODE_P (mode);
machine_mode vmode = mode;
rtvec par;
if (vector_mode || mode == TFmode || mode == HFmode)
{
use_sse = true;
if (mode == HFmode)
vmode = V8HFmode;
}
else if (TARGET_SSE_MATH)
{
use_sse = SSE_FLOAT_MODE_P (mode);
if (mode == SFmode)
vmode = V4SFmode;
else if (mode == DFmode)
vmode = V2DFmode;
}
dst = operands[0];
src = operands[1];
set = gen_rtx_fmt_e (code, mode, src);
set = gen_rtx_SET (dst, set);
if (use_sse)
{
rtx mask, use, clob;
/* NEG and ABS performed with SSE use bitwise mask operations.
Create the appropriate mask now. */
mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
use = gen_rtx_USE (VOIDmode, mask);
if (vector_mode || mode == TFmode)
par = gen_rtvec (2, set, use);
else
{
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
par = gen_rtvec (3, set, use, clob);
}
}
else
{
rtx clob;
/* Changing of sign for FP values is doable using integer unit too. */
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
par = gen_rtvec (2, set, clob);
}
emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
}
/* Deconstruct a floating point ABS or NEG operation
with integer registers into integer operations. */
void
ix86_split_fp_absneg_operator (enum rtx_code code, machine_mode mode,
rtx operands[])
{
enum rtx_code absneg_op;
rtx dst, set;
gcc_assert (operands_match_p (operands[0], operands[1]));
switch (mode)
{
case E_SFmode:
dst = gen_lowpart (SImode, operands[0]);
if (code == ABS)
{
set = gen_int_mode (0x7fffffff, SImode);
absneg_op = AND;
}
else
{
set = gen_int_mode (0x80000000, SImode);
absneg_op = XOR;
}
set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
break;
case E_DFmode:
if (TARGET_64BIT)
{
dst = gen_lowpart (DImode, operands[0]);
dst = gen_rtx_ZERO_EXTRACT (DImode, dst, const1_rtx, GEN_INT (63));
if (code == ABS)
set = const0_rtx;
else
set = gen_rtx_NOT (DImode, dst);
}
else
{
dst = gen_highpart (SImode, operands[0]);
if (code == ABS)
{
set = gen_int_mode (0x7fffffff, SImode);
absneg_op = AND;
}
else
{
set = gen_int_mode (0x80000000, SImode);
absneg_op = XOR;
}
set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
}
break;
case E_XFmode:
dst = gen_rtx_REG (SImode,
REGNO (operands[0]) + (TARGET_64BIT ? 1 : 2));
if (code == ABS)
{
set = GEN_INT (0x7fff);
absneg_op = AND;
}
else
{
set = GEN_INT (0x8000);
absneg_op = XOR;
}
set = gen_rtx_fmt_ee (absneg_op, SImode, dst, set);
break;
default:
gcc_unreachable ();
}
set = gen_rtx_SET (dst, set);
rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
rtvec par = gen_rtvec (2, set, clob);
emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
}
/* Expand a copysign operation. Special case operand 0 being a constant. */
void
ix86_expand_copysign (rtx operands[])
{
machine_mode mode, vmode;
rtx dest, op0, op1, mask, op2, op3;
mode = GET_MODE (operands[0]);
if (mode == HFmode)
vmode = V8HFmode;
else if (mode == SFmode)
vmode = V4SFmode;
else if (mode == DFmode)
vmode = V2DFmode;
else if (mode == TFmode)
vmode = mode;
else
gcc_unreachable ();
if (rtx_equal_p (operands[1], operands[2]))
{
emit_move_insn (operands[0], operands[1]);
return;
}
dest = lowpart_subreg (vmode, operands[0], mode);
op1 = lowpart_subreg (vmode, operands[2], mode);
mask = ix86_build_signbit_mask (vmode, 0, 0);
if (CONST_DOUBLE_P (operands[1]))
{
op0 = simplify_unary_operation (ABS, mode, operands[1], mode);
/* Optimize for 0, simplify b = copy_signf (0.0f, a) to b = mask & a. */
if (op0 == CONST0_RTX (mode))
{
emit_move_insn (dest, gen_rtx_AND (vmode, mask, op1));
return;
}
if (GET_MODE_SIZE (mode) < 16)
op0 = ix86_build_const_vector (vmode, false, op0);
op0 = force_reg (vmode, op0);
}
else
op0 = lowpart_subreg (vmode, operands[1], mode);
op2 = gen_reg_rtx (vmode);
op3 = gen_reg_rtx (vmode);
emit_move_insn (op2, gen_rtx_AND (vmode,
gen_rtx_NOT (vmode, mask),
op0));
emit_move_insn (op3, gen_rtx_AND (vmode, mask, op1));
emit_move_insn (dest, gen_rtx_IOR (vmode, op2, op3));
}
/* Expand an xorsign operation. */
void
ix86_expand_xorsign (rtx operands[])
{
machine_mode mode, vmode;
rtx dest, op0, op1, mask, x, temp;
dest = operands[0];
op0 = operands[1];
op1 = operands[2];
mode = GET_MODE (dest);
if (mode == HFmode)
vmode = V8HFmode;
else if (mode == SFmode)
vmode = V4SFmode;
else if (mode == DFmode)
vmode = V2DFmode;
else
gcc_unreachable ();
temp = gen_reg_rtx (vmode);
mask = ix86_build_signbit_mask (vmode, 0, 0);
op1 = lowpart_subreg (vmode, op1, mode);
x = gen_rtx_AND (vmode, op1, mask);
emit_insn (gen_rtx_SET (temp, x));
op0 = lowpart_subreg (vmode, op0, mode);
x = gen_rtx_XOR (vmode, temp, op0);
dest = lowpart_subreg (vmode, dest, mode);
emit_insn (gen_rtx_SET (dest, x));
}
static rtx ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1);
void
ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
{
machine_mode mode = GET_MODE (op0);
rtx tmp;
/* Handle special case - vector comparsion with boolean result, transform
it using ptest instruction. */
if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
{
rtx flag = gen_rtx_REG (CCZmode, FLAGS_REG);
machine_mode p_mode = GET_MODE_SIZE (mode) == 32 ? V4DImode : V2DImode;
gcc_assert (code == EQ || code == NE);
/* Generate XOR since we can't check that one operand is zero vector. */
tmp = gen_reg_rtx (mode);
emit_insn (gen_rtx_SET (tmp, gen_rtx_XOR (mode, op0, op1)));
tmp = gen_lowpart (p_mode, tmp);
emit_insn (gen_rtx_SET (gen_rtx_REG (CCmode, FLAGS_REG),
gen_rtx_UNSPEC (CCmode,
gen_rtvec (2, tmp, tmp),
UNSPEC_PTEST)));
tmp = gen_rtx_fmt_ee (code, VOIDmode, flag, const0_rtx);
tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
gen_rtx_LABEL_REF (VOIDmode, label),
pc_rtx);
emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
return;
}
switch (mode)
{
case E_HFmode:
case E_SFmode:
case E_DFmode:
case E_XFmode:
case E_QImode:
case E_HImode:
case E_SImode:
simple:
tmp = ix86_expand_compare (code, op0, op1);
tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
gen_rtx_LABEL_REF (VOIDmode, label),
pc_rtx);
emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
return;
case E_DImode:
if (TARGET_64BIT)
goto simple;
/* For 32-bit target DI comparison may be performed on
SSE registers. To allow this we should avoid split
to SI mode which is achieved by doing xor in DI mode
and then comparing with zero (which is recognized by
STV pass). We don't compare using xor when optimizing
for size. */
if (!optimize_insn_for_size_p ()
&& TARGET_STV
&& (code == EQ || code == NE))
{
op0 = force_reg (mode, gen_rtx_XOR (mode, op0, op1));
op1 = const0_rtx;
}
/* FALLTHRU */
case E_TImode:
/* Expand DImode branch into multiple compare+branch. */
{
rtx lo[2], hi[2];
rtx_code_label *label2;
enum rtx_code code1, code2, code3;
machine_mode submode;
if (CONSTANT_P (op0) && !CONSTANT_P (op1))
{
std::swap (op0, op1);
code = swap_condition (code);
}
split_double_mode (mode, &op0, 1, lo+0, hi+0);
split_double_mode (mode, &op1, 1, lo+1, hi+1);
submode = mode == DImode ? SImode : DImode;
/* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
avoid two branches. This costs one extra insn, so disable when
optimizing for size. */
if ((code == EQ || code == NE)
&& (!optimize_insn_for_size_p ()
|| hi[1] == const0_rtx || lo[1] == const0_rtx))
{
rtx xor0, xor1;
xor1 = hi[0];
if (hi[1] != const0_rtx)
xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
NULL_RTX, 0, OPTAB_WIDEN);
xor0 = lo[0];
if (lo[1] != const0_rtx)
xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
NULL_RTX, 0, OPTAB_WIDEN);
tmp = expand_binop (submode, ior_optab, xor1, xor0,
NULL_RTX, 0, OPTAB_WIDEN);
ix86_expand_branch (code, tmp, const0_rtx, label);
return;
}
/* Otherwise, if we are doing less-than or greater-or-equal-than,
op1 is a constant and the low word is zero, then we can just
examine the high word. Similarly for low word -1 and
less-or-equal-than or greater-than. */
if (CONST_INT_P (hi[1]))
switch (code)
{
case LT: case LTU: case GE: case GEU:
if (lo[1] == const0_rtx)
{
ix86_expand_branch (code, hi[0], hi[1], label);
return;
}
break;
case LE: case LEU: case GT: case GTU:
if (lo[1] == constm1_rtx)
{
ix86_expand_branch (code, hi[0], hi[1], label);
return;
}
break;
default:
break;
}
/* Emulate comparisons that do not depend on Zero flag with
double-word subtraction. Note that only Overflow, Sign
and Carry flags are valid, so swap arguments and condition
of comparisons that would otherwise test Zero flag. */
switch (code)
{
case LE: case LEU: case GT: case GTU:
std::swap (lo[0], lo[1]);
std::swap (hi[0], hi[1]);
code = swap_condition (code);
/* FALLTHRU */
case LT: case LTU: case GE: case GEU:
{
bool uns = (code == LTU || code == GEU);
rtx (*sbb_insn) (machine_mode, rtx, rtx, rtx)
= uns ? gen_sub3_carry_ccc : gen_sub3_carry_ccgz;
if (!nonimmediate_operand (lo[0], submode))
lo[0] = force_reg (submode, lo[0]);
if (!x86_64_general_operand (lo[1], submode))
lo[1] = force_reg (submode, lo[1]);
if (!register_operand (hi[0], submode))
hi[0] = force_reg (submode, hi[0]);
if ((uns && !nonimmediate_operand (hi[1], submode))
|| (!uns && !x86_64_general_operand (hi[1], submode)))
hi[1] = force_reg (submode, hi[1]);
emit_insn (gen_cmp_1 (submode, lo[0], lo[1]));
tmp = gen_rtx_SCRATCH (submode);
emit_insn (sbb_insn (submode, tmp, hi[0], hi[1]));
tmp = gen_rtx_REG (uns ? CCCmode : CCGZmode, FLAGS_REG);
ix86_expand_branch (code, tmp, const0_rtx, label);
return;
}
default:
break;
}
/* Otherwise, we need two or three jumps. */
label2 = gen_label_rtx ();
code1 = code;
code2 = swap_condition (code);
code3 = unsigned_condition (code);
switch (code)
{
case LT: case GT: case LTU: case GTU:
break;
case LE: code1 = LT; code2 = GT; break;
case GE: code1 = GT; code2 = LT; break;
case LEU: code1 = LTU; code2 = GTU; break;
case GEU: code1 = GTU; code2 = LTU; break;
case EQ: code1 = UNKNOWN; code2 = NE; break;
case NE: code2 = UNKNOWN; break;
default:
gcc_unreachable ();
}
/*
* a < b =>
* if (hi(a) < hi(b)) goto true;
* if (hi(a) > hi(b)) goto false;
* if (lo(a) < lo(b)) goto true;
* false:
*/
if (code1 != UNKNOWN)
ix86_expand_branch (code1, hi[0], hi[1], label);
if (code2 != UNKNOWN)
ix86_expand_branch (code2, hi[0], hi[1], label2);
ix86_expand_branch (code3, lo[0], lo[1], label);
if (code2 != UNKNOWN)
emit_label (label2);
return;
}
default:
gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
goto simple;
}
}
/* Figure out whether to use unordered fp comparisons. */
static bool
ix86_unordered_fp_compare (enum rtx_code code)
{
if (!TARGET_IEEE_FP)
return false;
switch (code)
{
case LT:
case LE:
case GT:
case GE:
case LTGT:
return false;
case EQ:
case NE:
case UNORDERED:
case ORDERED:
case UNLT:
case UNLE:
case UNGT:
case UNGE:
case UNEQ:
return true;
default:
gcc_unreachable ();
}
}
/* Return a comparison we can do and that it is equivalent to
swap_condition (code) apart possibly from orderedness.
But, never change orderedness if TARGET_IEEE_FP, returning
UNKNOWN in that case if necessary. */
static enum rtx_code
ix86_fp_swap_condition (enum rtx_code code)
{
switch (code)
{
case GT: /* GTU - CF=0 & ZF=0 */
return TARGET_IEEE_FP ? UNKNOWN : UNLT;
case GE: /* GEU - CF=0 */
return TARGET_IEEE_FP ? UNKNOWN : UNLE;
case UNLT: /* LTU - CF=1 */
return TARGET_IEEE_FP ? UNKNOWN : GT;
case UNLE: /* LEU - CF=1 | ZF=1 */
return TARGET_IEEE_FP ? UNKNOWN : GE;
default:
return swap_condition (code);
}
}
/* Return cost of comparison CODE using the best strategy for performance.
All following functions do use number of instructions as a cost metrics.
In future this should be tweaked to compute bytes for optimize_size and
take into account performance of various instructions on various CPUs. */
static int
ix86_fp_comparison_cost (enum rtx_code code)
{
int arith_cost;
/* The cost of code using bit-twiddling on %ah. */
switch (code)
{
case UNLE:
case UNLT:
case LTGT:
case GT:
case GE:
case UNORDERED:
case ORDERED:
case UNEQ:
arith_cost = 4;
break;
case LT:
case NE:
case EQ:
case UNGE:
arith_cost = TARGET_IEEE_FP ? 5 : 4;
break;
case LE:
case UNGT:
arith_cost = TARGET_IEEE_FP ? 6 : 4;
break;
default:
gcc_unreachable ();
}
switch (ix86_fp_comparison_strategy (code))
{
case IX86_FPCMP_COMI:
return arith_cost > 4 ? 3 : 2;
case IX86_FPCMP_SAHF:
return arith_cost > 4 ? 4 : 3;
default:
return arith_cost;
}
}
/* Swap, force into registers, or otherwise massage the two operands
to a fp comparison. The operands are updated in place; the new
comparison code is returned. */
static enum rtx_code
ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
{
bool unordered_compare = ix86_unordered_fp_compare (code);
rtx op0 = *pop0, op1 = *pop1;
machine_mode op_mode = GET_MODE (op0);
bool is_sse = SSE_FLOAT_MODE_SSEMATH_OR_HF_P (op_mode);
/* All of the unordered compare instructions only work on registers.
The same is true of the fcomi compare instructions. The XFmode
compare instructions require registers except when comparing
against zero or when converting operand 1 from fixed point to
floating point. */
if (!is_sse
&& (unordered_compare
|| (op_mode == XFmode
&& ! (standard_80387_constant_p (op0) == 1
|| standard_80387_constant_p (op1) == 1)
&& GET_CODE (op1) != FLOAT)
|| ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
{
op0 = force_reg (op_mode, op0);
op1 = force_reg (op_mode, op1);
}
else
{
/* %%% We only allow op1 in memory; op0 must be st(0). So swap
things around if they appear profitable, otherwise force op0
into a register. */
if (standard_80387_constant_p (op0) == 0
|| (MEM_P (op0)
&& ! (standard_80387_constant_p (op1) == 0
|| MEM_P (op1))))
{
enum rtx_code new_code = ix86_fp_swap_condition (code);
if (new_code != UNKNOWN)
{
std::swap (op0, op1);
code = new_code;
}
}
if (!REG_P (op0))
op0 = force_reg (op_mode, op0);
if (CONSTANT_P (op1))
{
int tmp = standard_80387_constant_p (op1);
if (tmp == 0)
op1 = validize_mem (force_const_mem (op_mode, op1));
else if (tmp == 1)
{
if (TARGET_CMOVE)
op1 = force_reg (op_mode, op1);
}
else
op1 = force_reg (op_mode, op1);
}
}
/* Try to rearrange the comparison to make it cheaper. */
if (ix86_fp_comparison_cost (code)
> ix86_fp_comparison_cost (swap_condition (code))
&& (REG_P (op1) || can_create_pseudo_p ()))
{
std::swap (op0, op1);
code = swap_condition (code);
if (!REG_P (op0))
op0 = force_reg (op_mode, op0);
}
*pop0 = op0;
*pop1 = op1;
return code;
}
/* Generate insn patterns to do a floating point compare of OPERANDS. */
static rtx
ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1)
{
bool unordered_compare = ix86_unordered_fp_compare (code);
machine_mode cmp_mode;
rtx tmp, scratch;
code = ix86_prepare_fp_compare_args (code, &op0, &op1);
tmp = gen_rtx_COMPARE (CCFPmode, op0, op1);
if (unordered_compare)
tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP);
/* Do fcomi/sahf based test when profitable. */
switch (ix86_fp_comparison_strategy (code))
{
case IX86_FPCMP_COMI:
cmp_mode = CCFPmode;
emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp));
break;
case IX86_FPCMP_SAHF:
cmp_mode = CCFPmode;
tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
scratch = gen_reg_rtx (HImode);
emit_insn (gen_rtx_SET (scratch, tmp));
emit_insn (gen_x86_sahf_1 (scratch));
break;
case IX86_FPCMP_ARITH:
cmp_mode = CCNOmode;
tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
scratch = gen_reg_rtx (HImode);
emit_insn (gen_rtx_SET (scratch, tmp));
/* In the unordered case, we have to check C2 for NaN's, which
doesn't happen to work out to anything nice combination-wise.
So do some bit twiddling on the value we've got in AH to come
up with an appropriate set of condition codes. */
switch (code)
{
case GT:
case UNGT:
if (code == GT || !TARGET_IEEE_FP)
{
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
code = EQ;
}
else
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
cmp_mode = CCmode;
code = GEU;
}
break;
case LT:
case UNLT:
if (code == LT && TARGET_IEEE_FP)
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
cmp_mode = CCmode;
code = EQ;
}
else
{
emit_insn (gen_testqi_ext_1_ccno (scratch, const1_rtx));
code = NE;
}
break;
case GE:
case UNGE:
if (code == GE || !TARGET_IEEE_FP)
{
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x05)));
code = EQ;
}
else
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_xorqi_ext_1_cc (scratch, scratch, const1_rtx));
code = NE;
}
break;
case LE:
case UNLE:
if (code == LE && TARGET_IEEE_FP)
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
cmp_mode = CCmode;
code = LTU;
}
else
{
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x45)));
code = NE;
}
break;
case EQ:
case UNEQ:
if (code == EQ && TARGET_IEEE_FP)
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
cmp_mode = CCmode;
code = EQ;
}
else
{
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
code = NE;
}
break;
case NE:
case LTGT:
if (code == NE && TARGET_IEEE_FP)
{
emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45)));
emit_insn (gen_xorqi_ext_1_cc (scratch, scratch,
GEN_INT (0x40)));
code = NE;
}
else
{
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x40)));
code = EQ;
}
break;
case UNORDERED:
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
code = NE;
break;
case ORDERED:
emit_insn (gen_testqi_ext_1_ccno (scratch, GEN_INT (0x04)));
code = EQ;
break;
default:
gcc_unreachable ();
}
break;
default:
gcc_unreachable();
}
/* Return the test that should be put into the flags user, i.e.
the bcc, scc, or cmov instruction. */
return gen_rtx_fmt_ee (code, VOIDmode,
gen_rtx_REG (cmp_mode, FLAGS_REG),
const0_rtx);
}
/* Generate insn patterns to do an integer compare of OPERANDS. */
static rtx
ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
{
machine_mode cmpmode;
rtx tmp, flags;
/* Swap operands to emit carry flag comparison. */
if ((code == GTU || code == LEU)
&& nonimmediate_operand (op1, VOIDmode))
{
std::swap (op0, op1);
code = swap_condition (code);
}
cmpmode = SELECT_CC_MODE (code, op0, op1);
flags = gen_rtx_REG (cmpmode, FLAGS_REG);
/* This is very simple, but making the interface the same as in the
FP case makes the rest of the code easier. */
tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
emit_insn (gen_rtx_SET (flags, tmp));
/* Return the test that should be put into the flags user, i.e.
the bcc, scc, or cmov instruction. */
return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
}
static rtx
ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
{
rtx ret;
if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
{
gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
ret = ix86_expand_fp_compare (code, op0, op1);
}
else
ret = ix86_expand_int_compare (code, op0, op1);
return ret;
}
void
ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
{
rtx ret;
gcc_assert (GET_MODE (dest) == QImode);
ret = ix86_expand_compare (code, op0, op1);
PUT_MODE (ret, QImode);
emit_insn (gen_rtx_SET (dest, ret));
}
/* Expand comparison setting or clearing carry flag. Return true when
successful and set pop for the operation. */
static bool
ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
{
machine_mode mode
= GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
/* Do not handle double-mode compares that go through special path. */
if (mode == (TARGET_64BIT ? TImode : DImode))
return false;
if (SCALAR_FLOAT_MODE_P (mode))
{
rtx compare_op;
rtx_insn *compare_seq;
gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
/* Shortcut: following common codes never translate
into carry flag compares. */
if (code == EQ || code == NE || code == UNEQ || code == LTGT
|| code == ORDERED || code == UNORDERED)
return false;
/* These comparisons require zero flag; swap operands so they won't. */
if ((code == GT || code == UNLE || code == LE || code == UNGT)
&& !TARGET_IEEE_FP)
{
std::swap (op0, op1);
code = swap_condition (code);
}
/* Try to expand the comparison and verify that we end up with
carry flag based comparison. This fails to be true only when
we decide to expand comparison using arithmetic that is not
too common scenario. */
start_sequence ();
compare_op = ix86_expand_fp_compare (code, op0, op1);
compare_seq = get_insns ();
end_sequence ();
if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode)
code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
else
code = GET_CODE (compare_op);
if (code != LTU && code != GEU)
return false;
emit_insn (compare_seq);
*pop = compare_op;
return true;
}
if (!INTEGRAL_MODE_P (mode))
return false;
switch (code)
{
case LTU:
case GEU:
break;
/* Convert a==0 into (unsigned)a<1. */
case EQ:
case NE:
if (op1 != const0_rtx)
return false;
op1 = const1_rtx;
code = (code == EQ ? LTU : GEU);
break;
/* Convert a>b into b<a or a>=b-1. */
case GTU:
case LEU:
if (CONST_INT_P (op1))
{
op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
/* Bail out on overflow. We still can swap operands but that
would force loading of the constant into register. */
if (op1 == const0_rtx
|| !x86_64_immediate_operand (op1, GET_MODE (op1)))
return false;
code = (code == GTU ? GEU : LTU);
}
else
{
std::swap (op0, op1);
code = (code == GTU ? LTU : GEU);
}
break;
/* Convert a>=0 into (unsigned)a<0x80000000. */
case LT:
case GE:
if (mode == DImode || op1 != const0_rtx)
return false;
op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
code = (code == LT ? GEU : LTU);
break;
case LE:
case GT:
if (mode == DImode || op1 != constm1_rtx)
return false;
op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
code = (code == LE ? GEU : LTU);
break;
default:
return false;
}
/* Swapping operands may cause constant to appear as first operand. */
if (!nonimmediate_operand (op0, VOIDmode))
{
if (!can_create_pseudo_p ())
return false;
op0 = force_reg (mode, op0);
}
*pop = ix86_expand_compare (code, op0, op1);
gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
return true;
}
/* Expand conditional increment or decrement using adb/sbb instructions.
The default case using setcc followed by the conditional move can be
done by generic code. */
bool
ix86_expand_int_addcc (rtx operands[])
{
enum rtx_code code = GET_CODE (operands[1]);
rtx flags;
rtx (*insn) (machine_mode, rtx, rtx, rtx, rtx, rtx);
rtx compare_op;
rtx val = const0_rtx;
bool fpcmp = false;
machine_mode mode;
rtx op0 = XEXP (operands[1], 0);
rtx op1 = XEXP (operands[1], 1);
if (operands[3] != const1_rtx
&& operands[3] != constm1_rtx)
return false;
if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
return false;
code = GET_CODE (compare_op);
flags = XEXP (compare_op, 0);
if (GET_MODE (flags) == CCFPmode)
{
fpcmp = true;
code = ix86_fp_compare_code_to_integer (code);
}
if (code != LTU)
{
val = constm1_rtx;
if (fpcmp)
PUT_CODE (compare_op,
reverse_condition_maybe_unordered
(GET_CODE (compare_op)));
else
PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
}
mode = GET_MODE (operands[0]);
/* Construct either adc or sbb insn. */
if ((code == LTU) == (operands[3] == constm1_rtx))
insn = gen_sub3_carry;
else
insn = gen_add3_carry;
emit_insn (insn (mode, operands[0], operands[2], val, flags, compare_op));
return true;
}
bool
ix86_expand_int_movcc (rtx operands[])
{
enum rtx_code code = GET_CODE (operands[1]), compare_code;
rtx_insn *compare_seq;
rtx compare_op;
machine_mode mode = GET_MODE (operands[0]);
bool sign_bit_compare_p = false;
rtx op0 = XEXP (operands[1], 0);
rtx op1 = XEXP (operands[1], 1);
if (GET_MODE (op0) == TImode
|| (GET_MODE (op0) == DImode
&& !TARGET_64BIT))
return false;
start_sequence ();
compare_op = ix86_expand_compare (code, op0, op1);
compare_seq = get_insns ();
end_sequence ();
compare_code = GET_CODE (compare_op);
if ((op1 == const0_rtx && (code == GE || code == LT))
|| (op1 == constm1_rtx && (code == GT || code == LE)))
sign_bit_compare_p = true;
/* Don't attempt mode expansion here -- if we had to expand 5 or 6
HImode insns, we'd be swallowed in word prefix ops. */
if ((mode != HImode || TARGET_FAST_PREFIX)
&& (mode != (TARGET_64BIT ? TImode : DImode))
&& CONST_INT_P (operands[2])
&& CONST_INT_P (operands[3]))
{
rtx out = operands[0];
HOST_WIDE_INT ct = INTVAL (operands[2]);
HOST_WIDE_INT cf = INTVAL (operands[3]);
HOST_WIDE_INT diff;
diff = ct - cf;
/* Sign bit compares are better done using shifts than we do by using
sbb. */
if (sign_bit_compare_p
|| ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
{
/* Detect overlap between destination and compare sources. */
rtx tmp = out;
if (!sign_bit_compare_p)
{
rtx flags;
bool fpcmp = false;
compare_code = GET_CODE (compare_op);
flags = XEXP (compare_op, 0);
if (GET_MODE (flags) == CCFPmode)
{
fpcmp = true;
compare_code
= ix86_fp_compare_code_to_integer (compare_code);
}
/* To simplify rest of code, restrict to the GEU case. */
if (compare_code == LTU)
{
std::swap (ct, cf);
compare_code = reverse_condition (compare_code);
code = reverse_condition (code);
}
else
{
if (fpcmp)
PUT_CODE (compare_op,
reverse_condition_maybe_unordered
(GET_CODE (compare_op)));
else
PUT_CODE (compare_op,
reverse_condition (GET_CODE (compare_op)));
}
diff = ct - cf;