| /* Expand the basic unary and binary arithmetic operations, for GNU compiler. |
| Copyright (C) 1987-2022 Free Software Foundation, Inc. |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it under |
| the terms of the GNU General Public License as published by the Free |
| Software Foundation; either version 3, or (at your option) any later |
| version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "target.h" |
| #include "rtl.h" |
| #include "tree.h" |
| #include "memmodel.h" |
| #include "predict.h" |
| #include "tm_p.h" |
| #include "optabs.h" |
| #include "expmed.h" |
| #include "emit-rtl.h" |
| #include "recog.h" |
| #include "diagnostic-core.h" |
| #include "rtx-vector-builder.h" |
| |
| /* Include insn-config.h before expr.h so that HAVE_conditional_move |
| is properly defined. */ |
| #include "stor-layout.h" |
| #include "except.h" |
| #include "dojump.h" |
| #include "explow.h" |
| #include "expr.h" |
| #include "optabs-tree.h" |
| #include "libfuncs.h" |
| #include "internal-fn.h" |
| #include "langhooks.h" |
| |
| static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *, |
| machine_mode *); |
| static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int); |
| static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool); |
| |
| static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode); |
| |
| /* Debug facility for use in GDB. */ |
| void debug_optab_libfuncs (void); |
| |
| /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to |
| the result of operation CODE applied to OP0 (and OP1 if it is a binary |
| operation). OP0_MODE is OP0's mode. |
| |
| If the last insn does not set TARGET, don't do anything, but return 1. |
| |
| If the last insn or a previous insn sets TARGET and TARGET is one of OP0 |
| or OP1, don't add the REG_EQUAL note but return 0. Our caller can then |
| try again, ensuring that TARGET is not one of the operands. */ |
| |
| static int |
| add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, |
| rtx op1, machine_mode op0_mode) |
| { |
| rtx_insn *last_insn; |
| rtx set; |
| rtx note; |
| |
| gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns)); |
| |
| if (GET_RTX_CLASS (code) != RTX_COMM_ARITH |
| && GET_RTX_CLASS (code) != RTX_BIN_ARITH |
| && GET_RTX_CLASS (code) != RTX_COMM_COMPARE |
| && GET_RTX_CLASS (code) != RTX_COMPARE |
| && GET_RTX_CLASS (code) != RTX_UNARY) |
| return 1; |
| |
| if (GET_CODE (target) == ZERO_EXTRACT) |
| return 1; |
| |
| for (last_insn = insns; |
| NEXT_INSN (last_insn) != NULL_RTX; |
| last_insn = NEXT_INSN (last_insn)) |
| ; |
| |
| /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing |
| a value changing in the insn, so the note would be invalid for CSE. */ |
| if (reg_overlap_mentioned_p (target, op0) |
| || (op1 && reg_overlap_mentioned_p (target, op1))) |
| { |
| if (MEM_P (target) |
| && (rtx_equal_p (target, op0) |
| || (op1 && rtx_equal_p (target, op1)))) |
| { |
| /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note |
| over expanding it as temp = MEM op X, MEM = temp. If the target |
| supports MEM = MEM op X instructions, it is sometimes too hard |
| to reconstruct that form later, especially if X is also a memory, |
| and due to multiple occurrences of addresses the address might |
| be forced into register unnecessarily. |
| Note that not emitting the REG_EQUIV note might inhibit |
| CSE in some cases. */ |
| set = single_set (last_insn); |
| if (set |
| && GET_CODE (SET_SRC (set)) == code |
| && MEM_P (SET_DEST (set)) |
| && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0)) |
| || (op1 && rtx_equal_p (SET_DEST (set), |
| XEXP (SET_SRC (set), 1))))) |
| return 1; |
| } |
| return 0; |
| } |
| |
| set = set_for_reg_notes (last_insn); |
| if (set == NULL_RTX) |
| return 1; |
| |
| if (! rtx_equal_p (SET_DEST (set), target) |
| /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */ |
| && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART |
| || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target))) |
| return 1; |
| |
| if (GET_RTX_CLASS (code) == RTX_UNARY) |
| switch (code) |
| { |
| case FFS: |
| case CLZ: |
| case CTZ: |
| case CLRSB: |
| case POPCOUNT: |
| case PARITY: |
| case BSWAP: |
| if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode) |
| { |
| note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0)); |
| if (GET_MODE_UNIT_SIZE (op0_mode) |
| > GET_MODE_UNIT_SIZE (GET_MODE (target))) |
| note = simplify_gen_unary (TRUNCATE, GET_MODE (target), |
| note, op0_mode); |
| else |
| note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target), |
| note, op0_mode); |
| break; |
| } |
| /* FALLTHRU */ |
| default: |
| note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0)); |
| break; |
| } |
| else |
| note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1)); |
| |
| set_unique_reg_note (last_insn, REG_EQUAL, note); |
| |
| return 1; |
| } |
| |
| /* Given two input operands, OP0 and OP1, determine what the correct from_mode |
| for a widening operation would be. In most cases this would be OP0, but if |
| that's a constant it'll be VOIDmode, which isn't useful. */ |
| |
| static machine_mode |
| widened_mode (machine_mode to_mode, rtx op0, rtx op1) |
| { |
| machine_mode m0 = GET_MODE (op0); |
| machine_mode m1 = GET_MODE (op1); |
| machine_mode result; |
| |
| if (m0 == VOIDmode && m1 == VOIDmode) |
| return to_mode; |
| else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1)) |
| result = m1; |
| else |
| result = m0; |
| |
| if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode)) |
| return to_mode; |
| |
| return result; |
| } |
| |
| /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP |
| says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need |
| not actually do a sign-extend or zero-extend, but can leave the |
| higher-order bits of the result rtx undefined, for example, in the case |
| of logical operations, but not right shifts. */ |
| |
| static rtx |
| widen_operand (rtx op, machine_mode mode, machine_mode oldmode, |
| int unsignedp, int no_extend) |
| { |
| rtx result; |
| scalar_int_mode int_mode; |
| |
| /* If we don't have to extend and this is a constant, return it. */ |
| if (no_extend && GET_MODE (op) == VOIDmode) |
| return op; |
| |
| /* If we must extend do so. If OP is a SUBREG for a promoted object, also |
| extend since it will be more efficient to do so unless the signedness of |
| a promoted object differs from our extension. */ |
| if (! no_extend |
| || !is_a <scalar_int_mode> (mode, &int_mode) |
| || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op) |
| && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp))) |
| return convert_modes (mode, oldmode, op, unsignedp); |
| |
| /* If MODE is no wider than a single word, we return a lowpart or paradoxical |
| SUBREG. */ |
| if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD) |
| return gen_lowpart (int_mode, force_reg (GET_MODE (op), op)); |
| |
| /* Otherwise, get an object of MODE, clobber it, and set the low-order |
| part to OP. */ |
| |
| result = gen_reg_rtx (int_mode); |
| emit_clobber (result); |
| emit_move_insn (gen_lowpart (GET_MODE (op), result), op); |
| return result; |
| } |
| |
| /* Expand vector widening operations. |
| |
| There are two different classes of operations handled here: |
| 1) Operations whose result is wider than all the arguments to the operation. |
| Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR |
| In this case OP0 and optionally OP1 would be initialized, |
| but WIDE_OP wouldn't (not relevant for this case). |
| 2) Operations whose result is of the same size as the last argument to the |
| operation, but wider than all the other arguments to the operation. |
| Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR. |
| In the case WIDE_OP, OP0 and optionally OP1 would be initialized. |
| |
| E.g, when called to expand the following operations, this is how |
| the arguments will be initialized: |
| nops OP0 OP1 WIDE_OP |
| widening-sum 2 oprnd0 - oprnd1 |
| widening-dot-product 3 oprnd0 oprnd1 oprnd2 |
| widening-mult 2 oprnd0 oprnd1 - |
| type-promotion (vec-unpack) 1 oprnd0 - - */ |
| |
| rtx |
| expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, |
| rtx target, int unsignedp) |
| { |
| class expand_operand eops[4]; |
| tree oprnd0, oprnd1, oprnd2; |
| machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode; |
| optab widen_pattern_optab; |
| enum insn_code icode; |
| int nops = TREE_CODE_LENGTH (ops->code); |
| int op; |
| bool sbool = false; |
| |
| oprnd0 = ops->op0; |
| oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE; |
| oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE; |
| |
| tmode0 = TYPE_MODE (TREE_TYPE (oprnd0)); |
| if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR |
| || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR) |
| /* The sign is from the result type rather than operand's type |
| for these ops. */ |
| widen_pattern_optab |
| = optab_for_tree_code (ops->code, ops->type, optab_default); |
| else if ((ops->code == VEC_UNPACK_HI_EXPR |
| || ops->code == VEC_UNPACK_LO_EXPR) |
| && VECTOR_BOOLEAN_TYPE_P (ops->type) |
| && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0)) |
| && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0)) |
| && SCALAR_INT_MODE_P (TYPE_MODE (ops->type))) |
| { |
| /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is |
| the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use |
| vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in |
| the pattern number of elements in the wider vector. */ |
| widen_pattern_optab |
| = (ops->code == VEC_UNPACK_HI_EXPR |
| ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab); |
| sbool = true; |
| } |
| else if (ops->code == DOT_PROD_EXPR) |
| { |
| enum optab_subtype subtype = optab_default; |
| signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0)); |
| signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1)); |
| if (sign1 == sign2) |
| ; |
| else if (sign1 == SIGNED && sign2 == UNSIGNED) |
| { |
| subtype = optab_vector_mixed_sign; |
| /* Same as optab_vector_mixed_sign but flip the operands. */ |
| std::swap (op0, op1); |
| } |
| else if (sign1 == UNSIGNED && sign2 == SIGNED) |
| subtype = optab_vector_mixed_sign; |
| else |
| gcc_unreachable (); |
| |
| widen_pattern_optab |
| = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype); |
| } |
| else |
| widen_pattern_optab |
| = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default); |
| if (ops->code == WIDEN_MULT_PLUS_EXPR |
| || ops->code == WIDEN_MULT_MINUS_EXPR) |
| icode = find_widening_optab_handler (widen_pattern_optab, |
| TYPE_MODE (TREE_TYPE (ops->op2)), |
| tmode0); |
| else |
| icode = optab_handler (widen_pattern_optab, tmode0); |
| gcc_assert (icode != CODE_FOR_nothing); |
| |
| if (nops >= 2) |
| tmode1 = TYPE_MODE (TREE_TYPE (oprnd1)); |
| else if (sbool) |
| { |
| nops = 2; |
| op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ()); |
| tmode1 = tmode0; |
| } |
| |
| /* The last operand is of a wider mode than the rest of the operands. */ |
| if (nops == 2) |
| wmode = tmode1; |
| else if (nops == 3) |
| { |
| gcc_assert (tmode1 == tmode0); |
| gcc_assert (op1); |
| wmode = TYPE_MODE (TREE_TYPE (oprnd2)); |
| } |
| |
| op = 0; |
| create_output_operand (&eops[op++], target, TYPE_MODE (ops->type)); |
| create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp); |
| if (op1) |
| create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp); |
| if (wide_op) |
| create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp); |
| expand_insn (icode, op, eops); |
| return eops[0].value; |
| } |
| |
| /* Generate code to perform an operation specified by TERNARY_OPTAB |
| on operands OP0, OP1 and OP2, with result having machine-mode MODE. |
| |
| UNSIGNEDP is for the case where we have to widen the operands |
| to perform the operation. It says to use zero-extension. |
| |
| If TARGET is nonzero, the value |
| is generated there, if it is convenient to do so. |
| In all cases an rtx is returned for the locus of the value; |
| this may or may not be TARGET. */ |
| |
| rtx |
| expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0, |
| rtx op1, rtx op2, rtx target, int unsignedp) |
| { |
| class expand_operand ops[4]; |
| enum insn_code icode = optab_handler (ternary_optab, mode); |
| |
| gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing); |
| |
| create_output_operand (&ops[0], target, mode); |
| create_convert_operand_from (&ops[1], op0, mode, unsignedp); |
| create_convert_operand_from (&ops[2], op1, mode, unsignedp); |
| create_convert_operand_from (&ops[3], op2, mode, unsignedp); |
| expand_insn (icode, 4, ops); |
| return ops[0].value; |
| } |
| |
| |
| /* Like expand_binop, but return a constant rtx if the result can be |
| calculated at compile time. The arguments and return value are |
| otherwise the same as for expand_binop. */ |
| |
| rtx |
| simplify_expand_binop (machine_mode mode, optab binoptab, |
| rtx op0, rtx op1, rtx target, int unsignedp, |
| enum optab_methods methods) |
| { |
| if (CONSTANT_P (op0) && CONSTANT_P (op1)) |
| { |
| rtx x = simplify_binary_operation (optab_to_code (binoptab), |
| mode, op0, op1); |
| if (x) |
| return x; |
| } |
| |
| return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods); |
| } |
| |
| /* Like simplify_expand_binop, but always put the result in TARGET. |
| Return true if the expansion succeeded. */ |
| |
| bool |
| force_expand_binop (machine_mode mode, optab binoptab, |
| rtx op0, rtx op1, rtx target, int unsignedp, |
| enum optab_methods methods) |
| { |
| rtx x = simplify_expand_binop (mode, binoptab, op0, op1, |
| target, unsignedp, methods); |
| if (x == 0) |
| return false; |
| if (x != target) |
| emit_move_insn (target, x); |
| return true; |
| } |
| |
| /* Create a new vector value in VMODE with all elements set to OP. The |
| mode of OP must be the element mode of VMODE. If OP is a constant, |
| then the return value will be a constant. */ |
| |
| rtx |
| expand_vector_broadcast (machine_mode vmode, rtx op) |
| { |
| int n; |
| rtvec vec; |
| |
| gcc_checking_assert (VECTOR_MODE_P (vmode)); |
| |
| if (valid_for_const_vector_p (vmode, op)) |
| return gen_const_vec_duplicate (vmode, op); |
| |
| insn_code icode = optab_handler (vec_duplicate_optab, vmode); |
| if (icode != CODE_FOR_nothing) |
| { |
| class expand_operand ops[2]; |
| create_output_operand (&ops[0], NULL_RTX, vmode); |
| create_input_operand (&ops[1], op, GET_MODE (op)); |
| expand_insn (icode, 2, ops); |
| return ops[0].value; |
| } |
| |
| if (!GET_MODE_NUNITS (vmode).is_constant (&n)) |
| return NULL; |
| |
| /* ??? If the target doesn't have a vec_init, then we have no easy way |
| of performing this operation. Most of this sort of generic support |
| is hidden away in the vector lowering support in gimple. */ |
| icode = convert_optab_handler (vec_init_optab, vmode, |
| GET_MODE_INNER (vmode)); |
| if (icode == CODE_FOR_nothing) |
| return NULL; |
| |
| vec = rtvec_alloc (n); |
| for (int i = 0; i < n; ++i) |
| RTVEC_ELT (vec, i) = op; |
| rtx ret = gen_reg_rtx (vmode); |
| emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec))); |
| |
| return ret; |
| } |
| |
| /* This subroutine of expand_doubleword_shift handles the cases in which |
| the effective shift value is >= BITS_PER_WORD. The arguments and return |
| value are the same as for the parent routine, except that SUPERWORD_OP1 |
| is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET. |
| INTO_TARGET may be null if the caller has decided to calculate it. */ |
| |
| static bool |
| expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1, |
| rtx outof_target, rtx into_target, |
| int unsignedp, enum optab_methods methods) |
| { |
| if (into_target != 0) |
| if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1, |
| into_target, unsignedp, methods)) |
| return false; |
| |
| if (outof_target != 0) |
| { |
| /* For a signed right shift, we must fill OUTOF_TARGET with copies |
| of the sign bit, otherwise we must fill it with zeros. */ |
| if (binoptab != ashr_optab) |
| emit_move_insn (outof_target, CONST0_RTX (word_mode)); |
| else |
| if (!force_expand_binop (word_mode, binoptab, outof_input, |
| gen_int_shift_amount (word_mode, |
| BITS_PER_WORD - 1), |
| outof_target, unsignedp, methods)) |
| return false; |
| } |
| return true; |
| } |
| |
| /* This subroutine of expand_doubleword_shift handles the cases in which |
| the effective shift value is < BITS_PER_WORD. The arguments and return |
| value are the same as for the parent routine. */ |
| |
| static bool |
| expand_subword_shift (scalar_int_mode op1_mode, optab binoptab, |
| rtx outof_input, rtx into_input, rtx op1, |
| rtx outof_target, rtx into_target, |
| int unsignedp, enum optab_methods methods, |
| unsigned HOST_WIDE_INT shift_mask) |
| { |
| optab reverse_unsigned_shift, unsigned_shift; |
| rtx tmp, carries; |
| |
| reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab); |
| unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab); |
| |
| /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT. |
| We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in |
| the opposite direction to BINOPTAB. */ |
| if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD) |
| { |
| carries = outof_input; |
| tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, |
| op1_mode), op1_mode); |
| tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, |
| 0, true, methods); |
| } |
| else |
| { |
| /* We must avoid shifting by BITS_PER_WORD bits since that is either |
| the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or |
| has unknown behavior. Do a single shift first, then shift by the |
| remainder. It's OK to use ~OP1 as the remainder if shift counts |
| are truncated to the mode size. */ |
| carries = expand_binop (word_mode, reverse_unsigned_shift, |
| outof_input, const1_rtx, 0, unsignedp, methods); |
| if (shift_mask == BITS_PER_WORD - 1) |
| { |
| tmp = immed_wide_int_const |
| (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode); |
| tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp, |
| 0, true, methods); |
| } |
| else |
| { |
| tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1, |
| op1_mode), op1_mode); |
| tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, |
| 0, true, methods); |
| } |
| } |
| if (tmp == 0 || carries == 0) |
| return false; |
| carries = expand_binop (word_mode, reverse_unsigned_shift, |
| carries, tmp, 0, unsignedp, methods); |
| if (carries == 0) |
| return false; |
| |
| /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT |
| so the result can go directly into INTO_TARGET if convenient. */ |
| tmp = expand_binop (word_mode, unsigned_shift, into_input, op1, |
| into_target, unsignedp, methods); |
| if (tmp == 0) |
| return false; |
| |
| /* Now OR in the bits carried over from OUTOF_INPUT. */ |
| if (!force_expand_binop (word_mode, ior_optab, tmp, carries, |
| into_target, unsignedp, methods)) |
| return false; |
| |
| /* Use a standard word_mode shift for the out-of half. */ |
| if (outof_target != 0) |
| if (!force_expand_binop (word_mode, binoptab, outof_input, op1, |
| outof_target, unsignedp, methods)) |
| return false; |
| |
| return true; |
| } |
| |
| |
| /* Try implementing expand_doubleword_shift using conditional moves. |
| The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true, |
| otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1 |
| are the shift counts to use in the former and latter case. All other |
| arguments are the same as the parent routine. */ |
| |
| static bool |
| expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab, |
| enum rtx_code cmp_code, rtx cmp1, rtx cmp2, |
| rtx outof_input, rtx into_input, |
| rtx subword_op1, rtx superword_op1, |
| rtx outof_target, rtx into_target, |
| int unsignedp, enum optab_methods methods, |
| unsigned HOST_WIDE_INT shift_mask) |
| { |
| rtx outof_superword, into_superword; |
| |
| /* Put the superword version of the output into OUTOF_SUPERWORD and |
| INTO_SUPERWORD. */ |
| outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0; |
| if (outof_target != 0 && subword_op1 == superword_op1) |
| { |
| /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in |
| OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */ |
| into_superword = outof_target; |
| if (!expand_superword_shift (binoptab, outof_input, superword_op1, |
| outof_superword, 0, unsignedp, methods)) |
| return false; |
| } |
| else |
| { |
| into_superword = gen_reg_rtx (word_mode); |
| if (!expand_superword_shift (binoptab, outof_input, superword_op1, |
| outof_superword, into_superword, |
| unsignedp, methods)) |
| return false; |
| } |
| |
| /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */ |
| if (!expand_subword_shift (op1_mode, binoptab, |
| outof_input, into_input, subword_op1, |
| outof_target, into_target, |
| unsignedp, methods, shift_mask)) |
| return false; |
| |
| /* Select between them. Do the INTO half first because INTO_SUPERWORD |
| might be the current value of OUTOF_TARGET. */ |
| if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode }, |
| into_target, into_superword, word_mode, false)) |
| return false; |
| |
| if (outof_target != 0) |
| if (!emit_conditional_move (outof_target, |
| { cmp_code, cmp1, cmp2, op1_mode }, |
| outof_target, outof_superword, |
| word_mode, false)) |
| return false; |
| |
| return true; |
| } |
| |
| /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts. |
| OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first |
| input operand; the shift moves bits in the direction OUTOF_INPUT-> |
| INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words |
| of the target. OP1 is the shift count and OP1_MODE is its mode. |
| If OP1 is constant, it will have been truncated as appropriate |
| and is known to be nonzero. |
| |
| If SHIFT_MASK is zero, the result of word shifts is undefined when the |
| shift count is outside the range [0, BITS_PER_WORD). This routine must |
| avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2). |
| |
| If SHIFT_MASK is nonzero, all word-mode shift counts are effectively |
| masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will |
| fill with zeros or sign bits as appropriate. |
| |
| If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize |
| a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1. |
| Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED. |
| In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2) |
| are undefined. |
| |
| BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function |
| may not use INTO_INPUT after modifying INTO_TARGET, and similarly for |
| OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent |
| function wants to calculate it itself. |
| |
| Return true if the shift could be successfully synthesized. */ |
| |
| static bool |
| expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab, |
| rtx outof_input, rtx into_input, rtx op1, |
| rtx outof_target, rtx into_target, |
| int unsignedp, enum optab_methods methods, |
| unsigned HOST_WIDE_INT shift_mask) |
| { |
| rtx superword_op1, tmp, cmp1, cmp2; |
| enum rtx_code cmp_code; |
| |
| /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will |
| fill the result with sign or zero bits as appropriate. If so, the value |
| of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call |
| this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT |
| and INTO_INPUT), then emit code to set up OUTOF_TARGET. |
| |
| This isn't worthwhile for constant shifts since the optimizers will |
| cope better with in-range shift counts. */ |
| if (shift_mask >= BITS_PER_WORD |
| && outof_target != 0 |
| && !CONSTANT_P (op1)) |
| { |
| if (!expand_doubleword_shift (op1_mode, binoptab, |
| outof_input, into_input, op1, |
| 0, into_target, |
| unsignedp, methods, shift_mask)) |
| return false; |
| if (!force_expand_binop (word_mode, binoptab, outof_input, op1, |
| outof_target, unsignedp, methods)) |
| return false; |
| return true; |
| } |
| |
| /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2) |
| is true when the effective shift value is less than BITS_PER_WORD. |
| Set SUPERWORD_OP1 to the shift count that should be used to shift |
| OUTOF_INPUT into INTO_TARGET when the condition is false. */ |
| tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode); |
| if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1) |
| { |
| /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1 |
| is a subword shift count. */ |
| cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp, |
| 0, true, methods); |
| cmp2 = CONST0_RTX (op1_mode); |
| cmp_code = EQ; |
| superword_op1 = op1; |
| } |
| else |
| { |
| /* Set CMP1 to OP1 - BITS_PER_WORD. */ |
| cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp, |
| 0, true, methods); |
| cmp2 = CONST0_RTX (op1_mode); |
| cmp_code = LT; |
| superword_op1 = cmp1; |
| } |
| if (cmp1 == 0) |
| return false; |
| |
| /* If we can compute the condition at compile time, pick the |
| appropriate subroutine. */ |
| tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2); |
| if (tmp != 0 && CONST_INT_P (tmp)) |
| { |
| if (tmp == const0_rtx) |
| return expand_superword_shift (binoptab, outof_input, superword_op1, |
| outof_target, into_target, |
| unsignedp, methods); |
| else |
| return expand_subword_shift (op1_mode, binoptab, |
| outof_input, into_input, op1, |
| outof_target, into_target, |
| unsignedp, methods, shift_mask); |
| } |
| |
| /* Try using conditional moves to generate straight-line code. */ |
| if (HAVE_conditional_move) |
| { |
| rtx_insn *start = get_last_insn (); |
| if (expand_doubleword_shift_condmove (op1_mode, binoptab, |
| cmp_code, cmp1, cmp2, |
| outof_input, into_input, |
| op1, superword_op1, |
| outof_target, into_target, |
| unsignedp, methods, shift_mask)) |
| return true; |
| delete_insns_since (start); |
| } |
| |
| /* As a last resort, use branches to select the correct alternative. */ |
| rtx_code_label *subword_label = gen_label_rtx (); |
| rtx_code_label *done_label = gen_label_rtx (); |
| |
| NO_DEFER_POP; |
| do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode, |
| 0, 0, subword_label, |
| profile_probability::uninitialized ()); |
| OK_DEFER_POP; |
| |
| if (!expand_superword_shift (binoptab, outof_input, superword_op1, |
| outof_target, into_target, |
| unsignedp, methods)) |
| return false; |
| |
| emit_jump_insn (targetm.gen_jump (done_label)); |
| emit_barrier (); |
| emit_label (subword_label); |
| |
| if (!expand_subword_shift (op1_mode, binoptab, |
| outof_input, into_input, op1, |
| outof_target, into_target, |
| unsignedp, methods, shift_mask)) |
| return false; |
| |
| emit_label (done_label); |
| return true; |
| } |
| |
| /* Subroutine of expand_binop. Perform a double word multiplication of |
| operands OP0 and OP1 both of mode MODE, which is exactly twice as wide |
| as the target's word_mode. This function return NULL_RTX if anything |
| goes wrong, in which case it may have already emitted instructions |
| which need to be deleted. |
| |
| If we want to multiply two two-word values and have normal and widening |
| multiplies of single-word values, we can do this with three smaller |
| multiplications. |
| |
| The multiplication proceeds as follows: |
| _______________________ |
| [__op0_high_|__op0_low__] |
| _______________________ |
| * [__op1_high_|__op1_low__] |
| _______________________________________________ |
| _______________________ |
| (1) [__op0_low__*__op1_low__] |
| _______________________ |
| (2a) [__op0_low__*__op1_high_] |
| _______________________ |
| (2b) [__op0_high_*__op1_low__] |
| _______________________ |
| (3) [__op0_high_*__op1_high_] |
| |
| |
| This gives a 4-word result. Since we are only interested in the |
| lower 2 words, partial result (3) and the upper words of (2a) and |
| (2b) don't need to be calculated. Hence (2a) and (2b) can be |
| calculated using non-widening multiplication. |
| |
| (1), however, needs to be calculated with an unsigned widening |
| multiplication. If this operation is not directly supported we |
| try using a signed widening multiplication and adjust the result. |
| This adjustment works as follows: |
| |
| If both operands are positive then no adjustment is needed. |
| |
| If the operands have different signs, for example op0_low < 0 and |
| op1_low >= 0, the instruction treats the most significant bit of |
| op0_low as a sign bit instead of a bit with significance |
| 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low |
| with 2**BITS_PER_WORD - op0_low, and two's complements the |
| result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to |
| the result. |
| |
| Similarly, if both operands are negative, we need to add |
| (op0_low + op1_low) * 2**BITS_PER_WORD. |
| |
| We use a trick to adjust quickly. We logically shift op0_low right |
| (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to |
| op0_high (op1_high) before it is used to calculate 2b (2a). If no |
| logical shift exists, we do an arithmetic right shift and subtract |
| the 0 or -1. */ |
| |
| static rtx |
| expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target, |
| bool umulp, enum optab_methods methods) |
| { |
| int low = (WORDS_BIG_ENDIAN ? 1 : 0); |
| int high = (WORDS_BIG_ENDIAN ? 0 : 1); |
| rtx wordm1 = (umulp ? NULL_RTX |
| : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1)); |
| rtx product, adjust, product_high, temp; |
| |
| rtx op0_high = operand_subword_force (op0, high, mode); |
| rtx op0_low = operand_subword_force (op0, low, mode); |
| rtx op1_high = operand_subword_force (op1, high, mode); |
| rtx op1_low = operand_subword_force (op1, low, mode); |
| |
| /* If we're using an unsigned multiply to directly compute the product |
| of the low-order words of the operands and perform any required |
| adjustments of the operands, we begin by trying two more multiplications |
| and then computing the appropriate sum. |
| |
| We have checked above that the required addition is provided. |
| Full-word addition will normally always succeed, especially if |
| it is provided at all, so we don't worry about its failure. The |
| multiplication may well fail, however, so we do handle that. */ |
| |
| if (!umulp) |
| { |
| /* ??? This could be done with emit_store_flag where available. */ |
| temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1, |
| NULL_RTX, 1, methods); |
| if (temp) |
| op0_high = expand_binop (word_mode, add_optab, op0_high, temp, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| else |
| { |
| temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1, |
| NULL_RTX, 0, methods); |
| if (!temp) |
| return NULL_RTX; |
| op0_high = expand_binop (word_mode, sub_optab, op0_high, temp, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| } |
| |
| if (!op0_high) |
| return NULL_RTX; |
| } |
| |
| adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (!adjust) |
| return NULL_RTX; |
| |
| /* OP0_HIGH should now be dead. */ |
| |
| if (!umulp) |
| { |
| /* ??? This could be done with emit_store_flag where available. */ |
| temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1, |
| NULL_RTX, 1, methods); |
| if (temp) |
| op1_high = expand_binop (word_mode, add_optab, op1_high, temp, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| else |
| { |
| temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1, |
| NULL_RTX, 0, methods); |
| if (!temp) |
| return NULL_RTX; |
| op1_high = expand_binop (word_mode, sub_optab, op1_high, temp, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| } |
| |
| if (!op1_high) |
| return NULL_RTX; |
| } |
| |
| temp = expand_binop (word_mode, smul_optab, op1_high, op0_low, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (!temp) |
| return NULL_RTX; |
| |
| /* OP1_HIGH should now be dead. */ |
| |
| adjust = expand_binop (word_mode, add_optab, adjust, temp, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| |
| if (target && !REG_P (target)) |
| target = NULL_RTX; |
| |
| /* *_widen_optab needs to determine operand mode, make sure at least |
| one operand has non-VOID mode. */ |
| if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode) |
| op0_low = force_reg (word_mode, op0_low); |
| |
| if (umulp) |
| product = expand_binop (mode, umul_widen_optab, op0_low, op1_low, |
| target, 1, OPTAB_DIRECT); |
| else |
| product = expand_binop (mode, smul_widen_optab, op0_low, op1_low, |
| target, 1, OPTAB_DIRECT); |
| |
| if (!product) |
| return NULL_RTX; |
| |
| product_high = operand_subword (product, high, 1, mode); |
| adjust = expand_binop (word_mode, add_optab, product_high, adjust, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| emit_move_insn (product_high, adjust); |
| return product; |
| } |
| |
| /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for |
| constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range |
| (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be |
| computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1)) |
| + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values |
| depends on the bit value, if 2, then carry from the addition needs to be |
| added too, i.e. like: |
| sum += __builtin_add_overflow (low, high, &sum) |
| |
| Optimize signed double-word OP0 % OP1 similarly, just apply some correction |
| factor to the sum before doing unsigned remainder, in the form of |
| sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const); |
| then perform unsigned |
| remainder = sum % OP1; |
| and finally |
| remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */ |
| |
| static rtx |
| expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp) |
| { |
| if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0) |
| return NULL_RTX; |
| |
| rtx_insn *last = get_last_insn (); |
| for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--) |
| { |
| wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD); |
| if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1)) |
| continue; |
| rtx sum = NULL_RTX, mask = NULL_RTX; |
| if (bit == BITS_PER_WORD) |
| { |
| /* For signed modulo we need to add correction to the sum |
| and that might again overflow. */ |
| if (!unsignedp) |
| continue; |
| if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing) |
| continue; |
| tree wtype = lang_hooks.types.type_for_mode (word_mode, 1); |
| if (wtype == NULL_TREE) |
| continue; |
| tree ctype = build_complex_type (wtype); |
| if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode)) |
| continue; |
| machine_mode cmode = TYPE_MODE (ctype); |
| rtx op00 = operand_subword_force (op0, 0, mode); |
| rtx op01 = operand_subword_force (op0, 1, mode); |
| rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode), |
| gen_reg_rtx (word_mode)); |
| tree lhs = make_tree (ctype, cres); |
| tree arg0 = make_tree (wtype, op00); |
| tree arg1 = make_tree (wtype, op01); |
| expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0, |
| arg1, true, true, true, false, NULL); |
| sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0), |
| XEXP (cres, 1), NULL_RTX, 1, |
| OPTAB_DIRECT); |
| if (sum == NULL_RTX) |
| return NULL_RTX; |
| } |
| else |
| { |
| /* Code below uses GEN_INT, so we need the masks to be representable |
| in HOST_WIDE_INTs. */ |
| if (bit >= HOST_BITS_PER_WIDE_INT) |
| continue; |
| /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might |
| overflow. Consider 64-bit -1ULL for word size 32, if we add |
| 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */ |
| if (bit == BITS_PER_WORD - 1) |
| continue; |
| |
| int count = (2 * BITS_PER_WORD + bit - 1) / bit; |
| rtx sum_corr = NULL_RTX; |
| |
| if (!unsignedp) |
| { |
| /* For signed modulo, compute it as unsigned modulo of |
| sum with a correction added to it if OP0 is negative, |
| such that the result can be computed as unsigned |
| remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */ |
| w = wi::min_value (2 * BITS_PER_WORD, SIGNED); |
| wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1)); |
| wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1)); |
| /* wmod2 == -wmod1. */ |
| wmod2 = wmod2 + (INTVAL (op1) - 1); |
| if (wi::ne_p (wmod1, wmod2)) |
| { |
| wide_int wcorr = wmod2 - wmod1; |
| if (wi::neg_p (w)) |
| wcorr = wcorr + INTVAL (op1); |
| /* Now verify if the count sums can't overflow, and punt |
| if they could. */ |
| w = wi::mask (bit, false, 2 * BITS_PER_WORD); |
| w = w * (count - 1); |
| w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit, |
| false, 2 * BITS_PER_WORD); |
| w = w + wcorr; |
| w = wi::lrshift (w, BITS_PER_WORD); |
| if (wi::ne_p (w, 0)) |
| continue; |
| |
| mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1, |
| mode); |
| mask = expand_simple_binop (word_mode, ASHIFTRT, mask, |
| GEN_INT (BITS_PER_WORD - 1), |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (mask == NULL_RTX) |
| return NULL_RTX; |
| sum_corr = immed_wide_int_const (wcorr, word_mode); |
| sum_corr = expand_simple_binop (word_mode, AND, mask, |
| sum_corr, NULL_RTX, 1, |
| OPTAB_DIRECT); |
| if (sum_corr == NULL_RTX) |
| return NULL_RTX; |
| } |
| } |
| |
| for (int i = 0; i < count; i++) |
| { |
| rtx v = op0; |
| if (i) |
| v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit), |
| NULL_RTX, 1, OPTAB_DIRECT); |
| if (v == NULL_RTX) |
| return NULL_RTX; |
| v = lowpart_subreg (word_mode, v, mode); |
| if (v == NULL_RTX) |
| return NULL_RTX; |
| if (i != count - 1) |
| v = expand_simple_binop (word_mode, AND, v, |
| GEN_INT ((HOST_WIDE_INT_1U << bit) |
| - 1), NULL_RTX, 1, |
| OPTAB_DIRECT); |
| if (v == NULL_RTX) |
| return NULL_RTX; |
| if (sum == NULL_RTX) |
| sum = v; |
| else |
| sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX, |
| 1, OPTAB_DIRECT); |
| if (sum == NULL_RTX) |
| return NULL_RTX; |
| } |
| if (sum_corr) |
| { |
| sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr, |
| NULL_RTX, 1, OPTAB_DIRECT); |
| if (sum == NULL_RTX) |
| return NULL_RTX; |
| } |
| } |
| rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, NULL, NULL, |
| sum, gen_int_mode (INTVAL (op1), |
| word_mode), |
| NULL_RTX, 1, OPTAB_DIRECT); |
| if (remainder == NULL_RTX) |
| return NULL_RTX; |
| |
| if (!unsignedp) |
| { |
| if (mask == NULL_RTX) |
| { |
| mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1, |
| mode); |
| mask = expand_simple_binop (word_mode, ASHIFTRT, mask, |
| GEN_INT (BITS_PER_WORD - 1), |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (mask == NULL_RTX) |
| return NULL_RTX; |
| } |
| mask = expand_simple_binop (word_mode, AND, mask, |
| gen_int_mode (1 - INTVAL (op1), |
| word_mode), |
| NULL_RTX, 1, OPTAB_DIRECT); |
| if (mask == NULL_RTX) |
| return NULL_RTX; |
| remainder = expand_simple_binop (word_mode, PLUS, remainder, |
| mask, NULL_RTX, 1, OPTAB_DIRECT); |
| if (remainder == NULL_RTX) |
| return NULL_RTX; |
| } |
| |
| remainder = convert_modes (mode, word_mode, remainder, unsignedp); |
| /* Punt if we need any library calls. */ |
| if (last) |
| last = NEXT_INSN (last); |
| else |
| last = get_insns (); |
| for (; last; last = NEXT_INSN (last)) |
| if (CALL_P (last)) |
| return NULL_RTX; |
| return remainder; |
| } |
| return NULL_RTX; |
| } |
| |
| /* Similarly to the above function, but compute both quotient and remainder. |
| Quotient can be computed from the remainder as: |
| rem = op0 % op1; // Handled using expand_doubleword_mod |
| quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo |
| // 2 * BITS_PER_WORD |
| |
| We can also handle cases where op1 is a multiple of power of two constant |
| and constant handled by expand_doubleword_mod. |
| op11 = 1 << __builtin_ctz (op1); |
| op12 = op1 / op11; |
| rem1 = op0 % op12; // Handled using expand_doubleword_mod |
| quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo |
| // 2 * BITS_PER_WORD |
| rem = (quot1 % op11) * op12 + rem1; |
| quot = quot1 / op11; */ |
| |
| rtx |
| expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem, |
| bool unsignedp) |
| { |
| *rem = NULL_RTX; |
| |
| /* Negative dividend should have been optimized into positive, |
| similarly modulo by 1 and modulo by power of two is optimized |
| differently too. */ |
| if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1))) |
| return NULL_RTX; |
| |
| rtx op11 = const1_rtx; |
| rtx op12 = op1; |
| if ((INTVAL (op1) & 1) == 0) |
| { |
| int bit = ctz_hwi (INTVAL (op1)); |
| op11 = GEN_INT (HOST_WIDE_INT_1 << bit); |
| op12 = GEN_INT (INTVAL (op1) >> bit); |
| } |
| |
| rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp); |
| if (rem1 == NULL_RTX) |
| return NULL_RTX; |
| |
| int prec = 2 * BITS_PER_WORD; |
| wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED); |
| wide_int b = wi::shifted_mask (prec, 1, false, prec + 1); |
| wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED); |
| rtx inv = immed_wide_int_const (m, mode); |
| |
| rtx_insn *last = get_last_insn (); |
| rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1, |
| NULL_RTX, unsignedp, OPTAB_DIRECT); |
| if (quot1 == NULL_RTX) |
| return NULL_RTX; |
| |
| quot1 = expand_simple_binop (mode, MULT, quot1, inv, |
| NULL_RTX, unsignedp, OPTAB_DIRECT); |
| if (quot1 == NULL_RTX) |
| return NULL_RTX; |
| |
| if (op11 != const1_rtx) |
| { |
| rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, NULL, NULL, quot1, |
| op11, NULL_RTX, unsignedp, OPTAB_DIRECT); |
| if (rem2 == NULL_RTX) |
| return NULL_RTX; |
| |
| rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX, |
| unsignedp, OPTAB_DIRECT); |
| if (rem2 == NULL_RTX) |
| return NULL_RTX; |
| |
| rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX, |
| unsignedp, OPTAB_DIRECT); |
| if (rem2 == NULL_RTX) |
| return NULL_RTX; |
| |
| rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, NULL, NULL, quot1, |
| op11, NULL_RTX, unsignedp, OPTAB_DIRECT); |
| if (quot2 == NULL_RTX) |
| return NULL_RTX; |
| |
| rem1 = rem2; |
| quot1 = quot2; |
| } |
| |
| /* Punt if we need any library calls. */ |
| if (last) |
| last = NEXT_INSN (last); |
| else |
| last = get_insns (); |
| for (; last; last = NEXT_INSN (last)) |
| if (CALL_P (last)) |
| return NULL_RTX; |
| |
| *rem = rem1; |
| return quot1; |
| } |
| |
| /* Wrapper around expand_binop which takes an rtx code to specify |
| the operation to perform, not an optab pointer. All other |
| arguments are the same. */ |
| rtx |
| expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0, |
| rtx op1, rtx target, int unsignedp, |
| enum optab_methods methods) |
| { |
| optab binop = code_to_optab (code); |
| gcc_assert (binop); |
| |
| return expand_binop (mode, binop, op0, op1, target, unsignedp, methods); |
| } |
| |
| /* Return whether OP0 and OP1 should be swapped when expanding a commutative |
| binop. Order them according to commutative_operand_precedence and, if |
| possible, try to put TARGET or a pseudo first. */ |
| static bool |
| swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1) |
| { |
| int op0_prec = commutative_operand_precedence (op0); |
| int op1_prec = commutative_operand_precedence (op1); |
| |
| if (op0_prec < op1_prec) |
| return true; |
| |
| if (op0_prec > op1_prec) |
| return false; |
| |
| /* With equal precedence, both orders are ok, but it is better if the |
| first operand is TARGET, or if both TARGET and OP0 are pseudos. */ |
| if (target == 0 || REG_P (target)) |
| return (REG_P (op1) && !REG_P (op0)) || target == op1; |
| else |
| return rtx_equal_p (op1, target); |
| } |
| |
| /* Return true if BINOPTAB implements a shift operation. */ |
| |
| static bool |
| shift_optab_p (optab binoptab) |
| { |
| switch (optab_to_code (binoptab)) |
| { |
| case ASHIFT: |
| case SS_ASHIFT: |
| case US_ASHIFT: |
| case ASHIFTRT: |
| case LSHIFTRT: |
| case ROTATE: |
| case ROTATERT: |
| return true; |
| |
| default: |
| return false; |
| } |
| } |
| |
| /* Return true if BINOPTAB implements a commutative binary operation. */ |
| |
| static bool |
| commutative_optab_p (optab binoptab) |
| { |
| return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH |
| || binoptab == smul_widen_optab |
| || binoptab == umul_widen_optab |
| || binoptab == smul_highpart_optab |
| || binoptab == umul_highpart_optab); |
| } |
| |
| /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're |
| optimizing, and if the operand is a constant that costs more than |
| 1 instruction, force the constant into a register and return that |
| register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */ |
| |
| static rtx |
| avoid_expensive_constant (machine_mode mode, optab binoptab, |
| int opn, rtx x, bool unsignedp) |
| { |
| bool speed = optimize_insn_for_speed_p (); |
| |
| if (mode != VOIDmode |
| && optimize |
| && CONSTANT_P (x) |
| && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed) |
| > set_src_cost (x, mode, speed))) |
| { |
| if (CONST_INT_P (x)) |
| { |
| HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode); |
| if (intval != INTVAL (x)) |
| x = GEN_INT (intval); |
| } |
| else |
| x = convert_modes (mode, VOIDmode, x, unsignedp); |
| x = force_reg (mode, x); |
| } |
| return x; |
| } |
| |
| /* Helper function for expand_binop: handle the case where there |
| is an insn ICODE that directly implements the indicated operation. |
| Returns null if this is not possible. */ |
| static rtx |
| expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab, |
| rtx op0, rtx op1, |
| rtx target, int unsignedp, enum optab_methods methods, |
| rtx_insn *last) |
| { |
| machine_mode xmode0 = insn_data[(int) icode].operand[1].mode; |
| machine_mode xmode1 = insn_data[(int) icode].operand[2].mode; |
| machine_mode mode0, mode1, tmp_mode; |
| class expand_operand ops[3]; |
| bool commutative_p; |
| rtx_insn *pat; |
| rtx xop0 = op0, xop1 = op1; |
| bool canonicalize_op1 = false; |
| |
| /* If it is a commutative operator and the modes would match |
| if we would swap the operands, we can save the conversions. */ |
| commutative_p = commutative_optab_p (binoptab); |
| if (commutative_p |
| && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1 |
| && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0) |
| std::swap (xop0, xop1); |
| |
| /* If we are optimizing, force expensive constants into a register. */ |
| xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp); |
| if (!shift_optab_p (binoptab)) |
| xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp); |
| else |
| /* Shifts and rotates often use a different mode for op1 from op0; |
| for VOIDmode constants we don't know the mode, so force it |
| to be canonicalized using convert_modes. */ |
| canonicalize_op1 = true; |
| |
| /* In case the insn wants input operands in modes different from |
| those of the actual operands, convert the operands. It would |
| seem that we don't need to convert CONST_INTs, but we do, so |
| that they're properly zero-extended, sign-extended or truncated |
| for their mode. */ |
| |
| mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode; |
| if (xmode0 != VOIDmode && xmode0 != mode0) |
| { |
| xop0 = convert_modes (xmode0, mode0, xop0, unsignedp); |
| mode0 = xmode0; |
| } |
| |
| mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1) |
| ? GET_MODE (xop1) : mode); |
| if (xmode1 != VOIDmode && xmode1 != mode1) |
| { |
| xop1 = convert_modes (xmode1, mode1, xop1, unsignedp); |
| mode1 = xmode1; |
| } |
| |
| /* If operation is commutative, |
| try to make the first operand a register. |
| Even better, try to make it the same as the target. |
| Also try to make the last operand a constant. */ |
| if (commutative_p |
| && swap_commutative_operands_with_target (target, xop0, xop1)) |
| std::swap (xop0, xop1); |
| |
| /* Now, if insn's predicates don't allow our operands, put them into |
| pseudo regs. */ |
| |
| if (binoptab == vec_pack_trunc_optab |
| || binoptab == vec_pack_usat_optab |
| || binoptab == vec_pack_ssat_optab |
| || binoptab == vec_pack_ufix_trunc_optab |
| || binoptab == vec_pack_sfix_trunc_optab |
| || binoptab == vec_packu_float_optab |
| || binoptab == vec_packs_float_optab) |
| { |
| /* The mode of the result is different then the mode of the |
| arguments. */ |
| tmp_mode = insn_data[(int) icode].operand[0].mode; |
| if (VECTOR_MODE_P (mode) |
| && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode))) |
| { |
| delete_insns_since (last); |
| return NULL_RTX; |
| } |
| } |
| else |
| tmp_mode = mode; |
| |
| create_output_operand (&ops[0], target, tmp_mode); |
| create_input_operand (&ops[1], xop0, mode0); |
| create_input_operand (&ops[2], xop1, mode1); |
| pat = maybe_gen_insn (icode, 3, ops); |
| if (pat) |
| { |
| /* If PAT is composed of more than one insn, try to add an appropriate |
| REG_EQUAL note to it. If we can't because TEMP conflicts with an |
| operand, call expand_binop again, this time without a target. */ |
| if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX |
| && ! add_equal_note (pat, ops[0].value, |
| optab_to_code (binoptab), |
| ops[1].value, ops[2].value, mode0)) |
| { |
| delete_insns_since (last); |
| return expand_binop (mode, binoptab, op0, op1, NULL_RTX, |
| unsignedp, methods); |
| } |
| |
| emit_insn (pat); |
| return ops[0].value; |
| } |
| delete_insns_since (last); |
| return NULL_RTX; |
| } |
| |
| /* Generate code to perform an operation specified by BINOPTAB |
| on operands OP0 and OP1, with result having machine-mode MODE. |
| |
| UNSIGNEDP is for the case where we have to widen the operands |
| to perform the operation. It says to use zero-extension. |
| |
| If TARGET is nonzero, the value |
| is generated there, if it is convenient to do so. |
| In all cases an rtx is returned for the locus of the value; |
| this may or may not be TARGET. */ |
| |
| rtx |
| expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1, |
| rtx target, int unsignedp, enum optab_methods methods) |
| { |
| enum optab_methods next_methods |
| = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN |
| ? OPTAB_WIDEN : methods); |
| enum mode_class mclass; |
| enum insn_code icode; |
| machine_mode wider_mode; |
| scalar_int_mode int_mode; |
| rtx libfunc; |
| rtx temp; |
| rtx_insn *entry_last = get_last_insn (); |
| rtx_insn *last; |
| |
| mclass = GET_MODE_CLASS (mode); |
| |
| /* If subtracting an integer constant, convert this into an addition of |
| the negated constant. */ |
| |
| if (binoptab == sub_optab && CONST_INT_P (op1)) |
| { |
| op1 = negate_rtx (mode, op1); |
| binoptab = add_optab; |
| } |
| /* For shifts, constant invalid op1 might be expanded from different |
| mode than MODE. As those are invalid, force them to a register |
| to avoid further problems during expansion. */ |
| else if (CONST_INT_P (op1) |
| && shift_optab_p (binoptab) |
| && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode))) |
| { |
| op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode)); |
| op1 = force_reg (GET_MODE_INNER (mode), op1); |
| } |
| |
| /* Record where to delete back to if we backtrack. */ |
| last = get_last_insn (); |
| |
| /* If we can do it with a three-operand insn, do so. */ |
| |
| if (methods != OPTAB_MUST_WIDEN) |
| { |
| if (convert_optab_p (binoptab)) |
| { |
| machine_mode from_mode = widened_mode (mode, op0, op1); |
| icode = find_widening_optab_handler (binoptab, mode, from_mode); |
| } |
| else |
| icode = optab_handler (binoptab, mode); |
| if (icode != CODE_FOR_nothing) |
| { |
| temp = expand_binop_directly (icode, mode, binoptab, op0, op1, |
| target, unsignedp, methods, last); |
| if (temp) |
| return temp; |
| } |
| } |
| |
| /* If we were trying to rotate, and that didn't work, try rotating |
| the other direction before falling back to shifts and bitwise-or. */ |
| if (((binoptab == rotl_optab |
| && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing) |
| || (binoptab == rotr_optab |
| && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing)) |
| && is_int_mode (mode, &int_mode)) |
| { |
| optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab); |
| rtx newop1; |
| unsigned int bits = GET_MODE_PRECISION (int_mode); |
| |
| if (CONST_INT_P (op1)) |
| newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1)); |
| else if (targetm.shift_truncation_mask (int_mode) == bits - 1) |
| newop1 = negate_rtx (GET_MODE (op1), op1); |
| else |
| newop1 = expand_binop (GET_MODE (op1), sub_optab, |
| gen_int_mode (bits, GET_MODE (op1)), op1, |
| NULL_RTX, unsignedp, OPTAB_DIRECT); |
| |
| temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1, |
| target, unsignedp, methods, last); |
| if (temp) |
| return temp; |
| } |
| |
| /* If this is a multiply, see if we can do a widening operation that |
| takes operands of this mode and makes a wider mode. */ |
| |
| if (binoptab == smul_optab |
| && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode) |
| && (convert_optab_handler ((unsignedp |
| ? umul_widen_optab |
| : smul_widen_optab), |
| wider_mode, mode) != CODE_FOR_nothing)) |
| { |
| /* *_widen_optab needs to determine operand mode, make sure at least |
| one operand has non-VOID mode. */ |
| if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode) |
| op0 = force_reg (mode, op0); |
| temp = expand_binop (wider_mode, |
| unsignedp ? umul_widen_optab : smul_widen_optab, |
| op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT); |
| |
| if (temp != 0) |
| { |
| if (GET_MODE_CLASS (mode) == MODE_INT |
| && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp))) |
| return gen_lowpart (mode, temp); |
| else |
| return convert_to_mode (mode, temp, unsignedp); |
| } |
| } |
| |
| /* If this is a vector shift by a scalar, see if we can do a vector |
| shift by a vector. If so, broadcast the scalar into a vector. */ |
| if (mclass == MODE_VECTOR_INT) |
| { |
| optab otheroptab = unknown_optab; |
| |
| if (binoptab == ashl_optab) |
| otheroptab = vashl_optab; |
| else if (binoptab == ashr_optab) |
| otheroptab = vashr_optab; |
| else if (binoptab == lshr_optab) |
| otheroptab = vlshr_optab; |
| else if (binoptab == rotl_optab) |
| otheroptab = vrotl_optab; |
| else if (binoptab == rotr_optab) |
| otheroptab = vrotr_optab; |
| |
| if (otheroptab |
| && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing) |
| { |
| /* The scalar may have been extended to be too wide. Truncate |
| it back to the proper size to fit in the broadcast vector. */ |
| scalar_mode inner_mode = GET_MODE_INNER (mode); |
| if (!CONST_INT_P (op1) |
| && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1))) |
| > GET_MODE_BITSIZE (inner_mode))) |
| op1 = force_reg (inner_mode, |
| simplify_gen_unary (TRUNCATE, inner_mode, op1, |
| GET_MODE (op1))); |
| rtx vop1 = expand_vector_broadcast (mode, op1); |
| if (vop1) |
| { |
| temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1, |
| target, unsignedp, methods, last); |
| if (temp) |
| return temp; |
| } |
| } |
| } |
| |
| /* Look for a wider mode of the same class for which we think we |
| can open-code the operation. Check for a widening multiply at the |
| wider mode as well. */ |
| |
| if (CLASS_HAS_WIDER_MODES_P (mclass) |
| && methods != OPTAB_DIRECT && methods != OPTAB_LIB) |
| FOR_EACH_WIDER_MODE (wider_mode, mode) |
| { |
| machine_mode next_mode; |
| if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing |
| || (binoptab == smul_optab |
| && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode) |
| && (find_widening_optab_handler ((unsignedp |
| ? umul_widen_optab |
| : smul_widen_optab), |
| next_mode, mode) |
| != CODE_FOR_nothing))) |
| { |
| rtx xop0 = op0, xop1 = op1; |
| int no_extend = 0; |
| |
| /* For certain integer operations, we need not actually extend |
| the narrow operands, as long as we will truncate |
| the results to the same narrowness. */ |
| |
| if ((binoptab == ior_optab || binoptab == and_optab |
| || binoptab == xor_optab |
| || binoptab == add_optab || binoptab == sub_optab |
| || binoptab == smul_optab || binoptab == ashl_optab) |
| && mclass == MODE_INT) |
| { |
| no_extend = 1; |
| xop0 = avoid_expensive_constant (mode, binoptab, 0, |
| xop0, unsignedp); |
| if (binoptab != ashl_optab) |
| xop1 = avoid_expensive_constant (mode, binoptab, 1, |
| xop1, unsignedp); |
| } |
| |
| xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend); |
| |
| /* The second operand of a shift must always be extended. */ |
| xop1 = widen_operand (xop1, wider_mode, mode, unsignedp, |
| no_extend && binoptab != ashl_optab); |
| |
| temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX, |
| unsignedp, OPTAB_DIRECT); |
| if (temp) |
| { |
| if (mclass != MODE_INT |
| || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) |
| { |
| if (target == 0) |
| target = gen_reg_rtx (mode); |
| convert_move (target, temp, 0); |
| return target; |
| } |
| else |
| return gen_lowpart (mode, temp); |
| } |
| else |
| delete_insns_since (last); |
| } |
| } |
| |
| /* If operation is commutative, |
| try to make the first operand a register. |
| Even better, try to make it the same as the target. |
| Also try to make the last operand a constant. */ |
| if (commutative_optab_p (binoptab) |
| && swap_commutative_operands_with_target (target, op0, op1)) |
| std::swap (op0, op1); |
| |
| /* These can be done a word at a time. */ |
| if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab) |
| && is_int_mode (mode, &int_mode) |
| && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD |
| && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) |
| { |
| int i; |
| rtx_insn *insns; |
| |
| /* If TARGET is the same as one of the operands, the REG_EQUAL note |
| won't be accurate, so use a new target. */ |
| if (target == 0 |
| || target == op0 |
| || target == op1 |
| || reg_overlap_mentioned_p (target, op0) |
| || reg_overlap_mentioned_p (target, op1) |
| || !valid_multiword_target_p (target)) |
| target = gen_reg_rtx (int_mode); |
| |
| start_sequence (); |
| |
| /* Do the actual arithmetic. */ |
| machine_mode op0_mode = GET_MODE (op0); |
| machine_mode op1_mode = GET_MODE (op1); |
| if (op0_mode == VOIDmode) |
| op0_mode = int_mode; |
| if (op1_mode == VOIDmode) |
| op1_mode = int_mode; |
| for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++) |
| { |
| rtx target_piece = operand_subword (target, i, 1, int_mode); |
| rtx x = expand_binop (word_mode, binoptab, |
| operand_subword_force (op0, i, op0_mode), |
| operand_subword_force (op1, i, op1_mode), |
| target_piece, unsignedp, next_methods); |
| |
| if (x == 0) |
| break; |
| |
| if (target_piece != x) |
| emit_move_insn (target_piece, x); |
| } |
| |
| insns = get_insns (); |
| end_sequence (); |
| |
| if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD) |
| { |
| emit_insn (insns); |
| return target; |
| } |
| } |
| |
| /* Synthesize double word shifts from single word shifts. */ |
| if ((binoptab == lshr_optab || binoptab == ashl_optab |
| || binoptab == ashr_optab) |
| && is_int_mode (mode, &int_mode) |
| && (CONST_INT_P (op1) || optimize_insn_for_speed_p ()) |
| && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD |
| && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode) |
| && optab_handler (binoptab, word_mode) != CODE_FOR_nothing |
| && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing |
| && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) |
| { |
| unsigned HOST_WIDE_INT shift_mask, double_shift_mask; |
| scalar_int_mode op1_mode; |
| |
| double_shift_mask = targetm.shift_truncation_mask (int_mode); |
| shift_mask = targetm.shift_truncation_mask (word_mode); |
| op1_mode = (GET_MODE (op1) != VOIDmode |
| ? as_a <scalar_int_mode> (GET_MODE (op1)) |
| : word_mode); |
| |
| /* Apply the truncation to constant shifts. */ |
| if (double_shift_mask > 0 && CONST_INT_P (op1)) |
| op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode); |
| |
| if (op1 == CONST0_RTX (op1_mode)) |
| return op0; |
| |
| /* Make sure that this is a combination that expand_doubleword_shift |
| can handle. See the comments there for details. */ |
| if (double_shift_mask == 0 |
| || (shift_mask == BITS_PER_WORD - 1 |
| && double_shift_mask == BITS_PER_WORD * 2 - 1)) |
| { |
| rtx_insn *insns; |
| rtx into_target, outof_target; |
| rtx into_input, outof_input; |
| int left_shift, outof_word; |
| |
| /* If TARGET is the same as one of the operands, the REG_EQUAL note |
| won't be accurate, so use a new target. */ |
| if (target == 0 |
| || target == op0 |
| || target == op1 |
| || reg_overlap_mentioned_p (target, op0) |
| || reg_overlap_mentioned_p (target, op1) |
| || !valid_multiword_target_p (target)) |
| target = gen_reg_rtx (int_mode); |
| |
| start_sequence (); |
| |
| /* OUTOF_* is the word we are shifting bits away from, and |
| INTO_* is the word that we are shifting bits towards, thus |
| they differ depending on the direction of the shift and |
| WORDS_BIG_ENDIAN. */ |
| |
| left_shift = binoptab == ashl_optab; |
| outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; |
| |
| outof_target = operand_subword (target, outof_word, 1, int_mode); |
| into_target = operand_subword (target, 1 - outof_word, 1, int_mode); |
| |
| outof_input = operand_subword_force (op0, outof_word, int_mode); |
| into_input = operand_subword_force (op0, 1 - outof_word, int_mode); |
| |
| if (expand_doubleword_shift (op1_mode, binoptab, |
| outof_input, into_input, op1, |
| outof_target, into_target, |
| unsignedp, next_methods, shift_mask)) |
| { |
| insns = get_insns (); |
| end_sequence (); |
| |
| emit_insn (insns); |
| return target; |
| } |
| end_sequence (); |
| } |
| } |
| |
| /* Synthesize double word rotates from single word shifts. */ |
| if ((binoptab == rotl_optab || binoptab == rotr_optab) |
| && is_int_mode (mode, &int_mode) |
| && CONST_INT_P (op1) |
| && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD |
| && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing |
| && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) |
| { |
| rtx_insn *insns; |
| rtx into_target, outof_target; |
| rtx into_input, outof_input; |
| rtx inter; |
| int shift_count, left_shift, outof_word; |
| |
| /* If TARGET is the same as one of the operands, the REG_EQUAL note |
| won't be accurate, so use a new target. Do this also if target is not |
| a REG, first because having a register instead may open optimization |
| opportunities, and second because if target and op0 happen to be MEMs |
| designating the same location, we would risk clobbering it too early |
| in the code sequence we generate below. */ |
| if (target == 0 |
| || target == op0 |
| || target == op1 |
| || !REG_P (target) |
| || reg_overlap_mentioned_p (target, op0) |
| || reg_overlap_mentioned_p (target, op1) |
| || !valid_multiword_target_p (target)) |
| target = gen_reg_rtx (int_mode); |
| |
| start_sequence (); |
| |
| shift_count = INTVAL (op1); |
| |
| /* OUTOF_* is the word we are shifting bits away from, and |
| INTO_* is the word that we are shifting bits towards, thus |
| they differ depending on the direction of the shift and |
| WORDS_BIG_ENDIAN. */ |
| |
| left_shift = (binoptab == rotl_optab); |
| outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; |
| |
| outof_target = operand_subword (target, outof_word, 1, int_mode); |
| into_target = operand_subword (target, 1 - outof_word, 1, int_mode); |
| |
| outof_input = operand_subword_force (op0, outof_word, int_mode); |
| into_input = operand_subword_force (op0, 1 - outof_word, int_mode); |
| |
| if (shift_count == BITS_PER_WORD) |
| { |
| /* This is just a word swap. */ |
| emit_move_insn (outof_target, into_input); |
| emit_move_insn (into_target, outof_input); |
| inter = const0_rtx; |
| } |
| else |
| { |
| rtx into_temp1, into_temp2, outof_temp1, outof_temp2; |
| HOST_WIDE_INT first_shift_count, second_shift_count; |
| optab reverse_unsigned_shift, unsigned_shift; |
| |
| reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD) |
| ? lshr_optab : ashl_optab); |
| |
| unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD) |
| ? ashl_optab : lshr_optab); |
| |
| if (shift_count > BITS_PER_WORD) |
| { |
| first_shift_count = shift_count - BITS_PER_WORD; |
| second_shift_count = 2 * BITS_PER_WORD - shift_count; |
| } |
| else |
| { |
| first_shift_count = BITS_PER_WORD - shift_count; |
| second_shift_count = shift_count; |
| } |
| rtx first_shift_count_rtx |
| = gen_int_shift_amount (word_mode, first_shift_count); |
| rtx second_shift_count_rtx |
| = gen_int_shift_amount (word_mode, second_shift_count); |
| |
| into_temp1 = expand_binop (word_mode, unsigned_shift, |
| outof_input, first_shift_count_rtx, |
| NULL_RTX, unsignedp, next_methods); |
| into_temp2 = expand_binop (word_mode, reverse_unsigned_shift, |
| into_input, second_shift_count_rtx, |
| NULL_RTX, unsignedp, next_methods); |
| |
| if (into_temp1 != 0 && into_temp2 != 0) |
| inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2, |
| into_target, unsignedp, next_methods); |
| else |
| inter = 0; |
| |
| if (inter != 0 && inter != into_target) |
| emit_move_insn (into_target, inter); |
| |
| outof_temp1 = expand_binop (word_mode, unsigned_shift, |
| into_input, first_shift_count_rtx, |
| NULL_RTX, unsignedp, next_methods); |
| outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift, |
| outof_input, second_shift_count_rtx, |
| NULL_RTX, unsignedp, next_methods); |
| |
| if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0) |
| inter = expand_binop (word_mode, ior_optab, |
| outof_temp1, outof_temp2, |
| outof_target, unsignedp, next_methods); |
| |
| if (inter != 0 && inter != outof_target) |
| emit_move_insn (outof_target, inter); |
| } |
| |
| insns = get_insns (); |
| end_sequence (); |
| |
| if (inter != 0) |
| { |
| emit_insn (insns); |
| return target; |
| } |
| } |
| |
| /* These can be done a word at a time by propagating carries. */ |
| if ((binoptab == add_optab || binoptab == sub_optab) |
| && is_int_mode (mode, &int_mode) |
| && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD |
| && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) |
| { |
| unsigned int i; |
| optab otheroptab = binoptab == add_optab ? sub_optab : add_optab; |
| const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; |
| rtx carry_in = NULL_RTX, carry_out = NULL_RTX; |
| rtx xop0, xop1, xtarget; |
| |
| /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG |
| value is one of those, use it. Otherwise, use 1 since it is the |
| one easiest to get. */ |
| #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1 |
| int normalizep = STORE_FLAG_VALUE; |
| #else |
| int normalizep = 1; |
| #endif |
| |
| /* Prepare the operands. */ |
| xop0 = force_reg (int_mode, op0); |
| xop1 = force_reg (int_mode, op1); |
| |
| xtarget = gen_reg_rtx (int_mode); |
| |
| if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target)) |
| target = xtarget; |
| |
| /* Indicate for flow that the entire target reg is being set. */ |
| if (REG_P (target)) |
| emit_clobber (xtarget); |
| |
| /* Do the actual arithmetic. */ |
| for (i = 0; i < nwords; i++) |
| { |
| int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i); |
| rtx target_piece = operand_subword (xtarget, index, 1, int_mode); |
| rtx op0_piece = operand_subword_force (xop0, index, int_mode); |
| rtx op1_piece = operand_subword_force (xop1, index, int_mode); |
| rtx x; |
| |
| /* Main add/subtract of the input operands. */ |
| x = expand_binop (word_mode, binoptab, |
| op0_piece, op1_piece, |
| target_piece, unsignedp, next_methods); |
| if (x == 0) |
| break; |
| |
| if (i + 1 < nwords) |
| { |
| /* Store carry from main add/subtract. */ |
| carry_out = gen_reg_rtx (word_mode); |
| carry_out = emit_store_flag_force (carry_out, |
| (binoptab == add_optab |
| ? LT : GT), |
| x, op0_piece, |
| word_mode, 1, normalizep); |
| } |
| |
| if (i > 0) |
| { |
| rtx newx; |
| |
| /* Add/subtract previous carry to main result. */ |
| newx = expand_binop (word_mode, |
| normalizep == 1 ? binoptab : otheroptab, |
| x, carry_in, |
| NULL_RTX, 1, next_methods); |
| |
| if (i + 1 < nwords) |
| { |
| /* Get out carry from adding/subtracting carry in. */ |
| rtx carry_tmp = gen_reg_rtx (word_mode); |
| carry_tmp = emit_store_flag_force (carry_tmp, |
| (binoptab == add_optab |
| ? LT : GT), |
| newx, x, |
| word_mode, 1, normalizep); |
| |
| /* Logical-ior the two poss. carry together. */ |
| carry_out = expand_binop (word_mode, ior_optab, |
| carry_out, carry_tmp, |
| carry_out, 0, next_methods); |
| if (carry_out == 0) |
| break; |
| } |
| emit_move_insn (target_piece, newx); |
| } |
| else |
| { |
| if (x != target_piece) |
| emit_move_insn (target_piece, x); |
| } |
| |
| carry_in = carry_out; |
| } |
| |
| if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD) |
| { |
| if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing |
| || ! rtx_equal_p (target, xtarget)) |
| { |
| rtx_insn *temp = emit_move_insn (target, xtarget); |
| |
| set_dst_reg_note (temp, REG_EQUAL, |
| gen_rtx_fmt_ee (optab_to_code (binoptab), |
| int_mode, copy_rtx (xop0), |
| copy_rtx (xop1)), |
| target); |
| } |
| else |
| target = xtarget; |
| |
| return target; |
| } |
| |
| else |
| delete_insns_since (last); |
| } |
| |
| /* Attempt to synthesize double word multiplies using a sequence of word |
| mode multiplications. We first attempt to generate a sequence using a |
| more efficient unsigned widening multiply, and if that fails we then |
| try using a signed widening multiply. */ |
| |
| if (binoptab == smul_optab |
| && is_int_mode (mode, &int_mode) |
| && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD |
| && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing |
| && optab_handler (add_optab, word_mode) != CODE_FOR_nothing) |
| { |
| rtx product = NULL_RTX; |
| if (convert_optab_handler (umul_widen_optab, int_mode, word_mode) |
| != CODE_FOR_nothing) |
| { |
| product = expand_doubleword_mult (int_mode, op0, op1, target, |
| true, methods); |
| if (!product) |
| delete_insns_since (last); |
| } |
| |
| if (product == NULL_RTX |
| && (convert_optab_handler (smul_widen_optab, int_mode, word_mode) |
| != CODE_FOR_nothing)) |
| { |
| product = expand_doubleword_mult (int_mode, op0, op1, target, |
| false, methods); |
| if (!product) |
| delete_insns_since (last); |
| } |
| |
| if (product != NULL_RTX) |
| { |
| if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing) |
| { |
| rtx_insn *move = emit_move_insn (target ? target : product, |
| product); |
| set_dst_reg_note (move, |
| REG_EQUAL, |
| gen_rtx_fmt_ee (MULT, int_mode, |
| copy_rtx (op0), |
| copy_rtx (op1)), |
| target ? target : product); |
| } |
| return product; |
| } |
| } |
| |
| /* Attempt to synthetize double word modulo by constant divisor. */ |
| if ((binoptab == umod_optab |
| || binoptab == smod_optab |
| || binoptab == udiv_optab |
| || binoptab == sdiv_optab) |
| && optimize |
| && CONST_INT_P (op1) |
| && is_int_mode (mode, &int_mode) |
| && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD |
| && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab) |
| ? udivmod_optab : sdivmod_optab, |
| int_mode) == CODE_FOR_nothing |
| && optab_handler (and_optab, word_mode) != CODE_FOR_nothing |
| && optab_handler (add_optab, word_mode) != CODE_FOR_nothing |
| && optimize_insn_for_speed_p ()) |
| { |
| rtx res = NULL_RTX; |
| if ((binoptab == umod_optab || binoptab == smod_optab) |
| && (INTVAL (op1) & 1) == 0) |
| res = expand_doubleword_mod (int_mode, op0, op1, |
| binoptab == umod_optab); |
| else |
| { |
| rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res, |
| binoptab == umod_optab |
| || binoptab == udiv_optab); |
| if (quot == NULL_RTX) |
| res = NULL_RTX; |
| else if (binoptab == udiv_optab || binoptab == sdiv_optab) |
| res = quot; |
| } |
| if (res != NULL_RTX) |
| { |
| if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing) |
| { |
| rtx_insn *move = emit_move_insn (target ? target : res, |
| res); |
| set_dst_reg_note (move, REG_EQUAL, |
| gen_rtx_fmt_ee (optab_to_code (binoptab), |
| int_mode, copy_rtx (op0), op1), |
| target ? target : res); |
| } |
| return res; |
| } |
| else |
| delete_insns_since (last); |
| } |
| |
| /* It can't be open-coded in this mode. |
| Use a library call if one is available and caller says that's ok. */ |
| |
| libfunc = optab_libfunc (binoptab, mode); |
| if (libfunc |
| && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN)) |
| { |
| rtx_insn *insns; |
| rtx op1x = op1; |
| machine_mode op1_mode = mode; |
| rtx value; |
| |
| start_sequence (); |
| |
| if (shift_optab_p (binoptab)) |
| { |
| op1_mode = targetm.libgcc_shift_count_mode (); |
| /* Specify unsigned here, |
| since negative shift counts are meaningless. */ |
| op1x = convert_to_mode (op1_mode, op1, 1); |
| } |
| |
| if (GET_MODE (op0) != VOIDmode |
| && GET_MODE (op0) != mode) |
| op0 = convert_to_mode (mode, op0, unsignedp); |
| |
| /* Pass 1 for NO_QUEUE so we don't lose any increments |
| if the libcall is cse'd or moved. */ |
| value = emit_library_call_value (libfunc, |
| NULL_RTX, LCT_CONST, mode, |
| op0, mode, op1x, op1_mode); |
| |
| insns = get_insns (); |
| end_sequence (); |
| |
| bool trapv = trapv_binoptab_p (binoptab); |
| target = gen_reg_rtx (mode); |
| emit_libcall_block_1 (insns, target, value, |
| trapv ? NULL_RTX |
| : gen_rtx_fmt_ee (optab_to_code (binoptab), |
| mode, op0, op1), trapv); |
| |
| return target; |
| } |
| |
| delete_insns_since (last); |
| |
| /* It can't be done in this mode. Can we do it in a wider mode? */ |
| |
| if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN |
| || methods == OPTAB_MUST_WIDEN)) |
| { |
| /* Caller says, don't even try. */ |
| delete_insns_since (entry_last); |
| return 0; |
| } |
| |
| /* Compute the value of METHODS to pass to recursive calls. |
| Don't allow widening to be tried recursively. */ |
| |
| methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT); |
| |
| /* Look for a wider mode of the same class for which it appears we can do |
| the operation. */ |
| |
| if (CLASS_HAS_WIDER_MODES_P (mclass)) |
| { |
| /* This code doesn't make sense for conversion optabs, since we |
| wouldn't then want to extend the operands to be the same size |
| as the result. */ |
| gcc_assert (!convert_optab_p (binoptab)); |
| FOR_EACH_WIDER_MODE (wider_mode, mode) |
| { |
| if (optab_handler (binoptab, wider_mode) |
| || (methods == OPTAB_LIB |
| && optab_libfunc (binoptab, wider_mode))) |
| { |
| rtx xop0 = op0, xop1 = op1; |
| int no_extend = 0; |
| |
| /* For certain integer operations, we need not actually extend |
| the narrow operands, as long as we will truncate |
| the results to the same narrowness. */ |
| |
| if ((binoptab == ior_optab || binoptab == and_optab |
| || binoptab == xor_optab |
| || binoptab == add_optab || binoptab == sub_optab |
| || binoptab == smul_optab || binoptab == ashl_optab) |
| && mclass == MODE_INT) |
| no_extend = 1; |
| |
| xop0 = widen_operand (xop0, wider_mode, mode, |
| unsignedp, no_extend); |
| |
| /* The second operand of a shift must always be extended. */ |
| xop1 = widen_operand (xop1, wider_mode, mode, unsignedp, |
| no_extend && binoptab != ashl_optab); |
| |
| temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX, |
| unsignedp, methods); |
| if (temp) |
| { |
| if (mclass != MODE_INT |
| || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) |
| { |
| if (target == 0) |
| target = gen_reg_rtx (mode); |
| convert_move (target, temp, 0); |
| return target; |
| } |
| else |
| return gen_lowpart (mode, temp); |
| } |
| else |
| delete_insns_since (last); |
| } |
| } |
| } |
| |
| delete_insns_since (entry_last); |
| return 0; |
| } |
| |
| /* Expand a binary operator which has both signed and unsigned forms. |
| UOPTAB is the optab for unsigned operations, and SOPTAB is for |
| signed operations. |
| |
| If we widen unsigned operands, we may use a signed wider operation instead |
| of an unsigned wider operation, since the result would be the same. */ |
| |
| rtx |
| sign_expand_binop (machine_mode mode, optab uoptab, optab soptab, |
| rtx op0, rtx op1, rtx target, int unsignedp, |
| enum optab_methods methods) |
| { |
| rtx temp; |
| optab direct_optab = unsignedp ? uoptab : soptab; |
| bool save_enable; |
| |
| /* Do it without widening, if possible. */ |
| temp = expand_binop (mode, direct_optab, op0, op1, target, |
| unsignedp, OPTAB_DIRECT); |
| if (temp || methods == OPTAB_DIRECT) |
| return temp; |
| |
| /* Try widening to a signed int. Disable any direct use of any |
| signed insn in the current mode. */ |
| save_enable = swap_optab_enable (soptab, mode, false); |
| |
| temp = expand_binop (mode, soptab, op0, op1, target, |
| unsignedp, OPTAB_WIDEN); |
| |
| /* For unsigned operands, try widening to an unsigned int. */ |
| if (!temp && unsignedp) |
| temp = expand_binop (mode, uoptab, op0, op1, target, |
| unsignedp, OPTAB_WIDEN); |
| if (temp || methods == OPTAB_WIDEN) |
| goto egress; |
| |
| /* Use the right width libcall if that exists. */ |
| temp = expand_binop (mode, direct_optab, op0, op1, target, |
| unsignedp, OPTAB_LIB); |
| if (temp || methods == OPTAB_LIB) |
| goto egress; |
| |
| /* Must widen and use a libcall, use either signed or unsigned. */ |
| temp = expand_binop (mode, soptab, op0, op1, target, |
| unsignedp, methods); |
| if (!temp && unsignedp) |
| temp = expand_binop (mode, uoptab, op0, op1, target, |
| unsignedp, methods); |
| |
| egress: |
| /* Undo the fiddling above. */ |
| if (save_enable) |
| swap_optab_enable (soptab, mode, true); |
| return temp; |
| } |
| |
| /* Generate code to perform an operation specified by UNOPPTAB |
| on operand OP0, with two results to TARG0 and TARG1. |
| We assume that the order of the operands for the instruction |
| is TARG0, TARG1, OP0. |
| |
| Either TARG0 or TARG1 may be zero, but what that means is that |
| the result is not actually wanted. We will generate it into |
| a dummy pseudo-reg and discard it. They may not both be zero. |
| |
| Returns 1 if this operation can be performed; 0 if not. */ |
| |
| int |
| expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1, |
| int unsignedp) |
| { |
| machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); |
| enum mode_class mclass; |
| machine_mode wider_mode; |
| rtx_insn *entry_last = get_last_insn (); |
| rtx_insn *last; |
| |
| mclass = GET_MODE_CLASS (mode); |
| |
| if (!targ0) |
| targ0 = gen_reg_rtx (mode); |
| if (!targ1) |
| targ1 = gen_reg_rtx (mode); |
| |
| /* Record where to go back to if we fail. */ |
| last = get_last_insn (); |
| |
| if (optab_handler (unoptab, mode) != CODE_FOR_nothing) |
| { |
| class expand_operand ops[3]; |
| enum insn_code icode = optab_handler (unoptab, mode); |
| |
| create_fixed_operand (&ops[0], targ0); |
| create_fixed_operand (&ops[1], targ1); |
| create_convert_operand_from (&ops[2], op0, mode, unsignedp); |
| if (maybe_expand_insn (icode, 3, ops)) |
| return 1; |
| } |
| |
| /* It can't be done in this mode. Can we do it in a wider mode? */ |
| |
| if (CLASS_HAS_WIDER_MODES_P (mclass)) |
| { |
| FOR_EACH_WIDER_MODE (wider_mode, mode) |
| { |
| if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) |
| { |
| rtx t0 = gen_reg_rtx (wider_mode); |
| rtx t1 = gen_reg_rtx (wider_mode); |
| rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp); |
| |
| if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp)) |
| { |
| convert_move (targ0, t0, unsignedp); |
| convert_move (targ1, t1, unsignedp); |
| return 1; |
| } |
| else |
| delete_insns_since (last); |
| } |
| } |
| } |
| |
| delete_insns_since (entry_last); |
| return 0; |
| } |
| |
| /* Generate code to perform an operation specified by BINOPTAB |
| on operands OP0 and OP1, with two results to TARG1 and TARG2. |
| We assume that the order of the operands for the instruction |
| is TARG0, OP0, OP1, TARG1, which would fit a pattern like |
| [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))]. |
| |
| Either TARG0 or TARG1 may be zero, but what that means is that |
| the result is not actually wanted. We will generate it into |
| a dummy pseudo-reg and discard it. They may not both be zero. |
| |
| Returns 1 if this operation can be performed; 0 if not. */ |
| |
| int |
| expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1, |
| int unsignedp) |
| { |
| machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); |
| enum mode_class mclass; |
| machine_mode wider_mode; |
| rtx_insn *entry_last = get_last_insn (); |
| rtx_insn *last; |
| |
| mclass = GET_MODE_CLASS (mode); |
| |
| if (!targ0) |
| targ0 = gen_reg_rtx (mode); |
| if (!targ1) |
| targ1 = gen_reg_rtx (mode); |
| |
| /* Record where to go back to if we fail. */ |
| last = get_last_insn (); |
| |
| if (optab_handler (binoptab, mode) != CODE_FOR_nothing) |
| { |
| class expand_operand ops[4]; |
| enum insn_code icode = optab_handler (binoptab, mode); |
| machine_mode mode0 = insn_data[icode].operand[1].mode; |
| machine_mode mode1 = insn_data[icode].operand[2].mode; |
| rtx xop0 = op0, xop1 = op1; |
| |
| /* If we are optimizing, force expensive constants into a register. */ |
| xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp); |
| xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp); |
| |
| create_fixed_operand (&ops[0], targ0); |
| create_convert_operand_from (&ops[1], xop0, mode, unsignedp); |
| create_convert_operand_from (&ops[2], xop1, mode, unsignedp); |
| create_fixed_operand (&ops[3], targ1); |
| if (maybe_expand_insn (icode, 4, ops)) |
| return 1; |
| delete_insns_since (last); |
| } |
| |
| /* It can't be done in this mode. Can we do it in a wider mode? */ |
| |
| if (CLASS_HAS_WIDER_MODES_P (mclass)) |
| { |
| FOR_EACH_WIDER_MODE (wider_mode, mode) |
| { |
| if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing) |
| { |
| rtx t0 = gen_reg_rtx (wider_mode); |
| rtx t1 = gen_reg_rtx (wider_mode); |
| rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp); |
| rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp); |
| |
| if (expand_twoval_binop (binoptab, cop0, cop1, |
| t0, t1, unsignedp)) |
| { |
| convert_move (targ0, t0, unsignedp); |
| convert_move (targ1, t1, unsignedp); |
| return 1; |
| } |
| else |
| delete_insns_since (last); |
| } |
| } |
| } |
| |
| delete_insns_since (entry_last); |
| return 0; |
| } |
| |
| /* Expand the two-valued library call indicated by BINOPTAB, but |
| preserve only one of the values. If TARG0 is non-NULL, the first |
| value is placed into TARG0; otherwise the second value is placed |
| into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The |
| value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1). |
| This routine assumes that the value returned by the library call is |
| as if the return value was of an integral mode twice as wide as the |
| mode of OP0. Returns 1 if the call was successful. */ |
| |
| bool |
| expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1, |
| rtx targ0, rtx targ1, enum rtx_code code) |
| { |
| machine_mode mode; |
| machine_mode libval_mode; |
| rtx libval; |
| rtx_insn *insns; |
| rtx libfunc; |
| |
| /* Exactly one of TARG0 or TARG1 should be non-NULL. */ |
| gcc_assert (!targ0 != !targ1); |
| |
| mode = GET_MODE (op0); |
| libfunc = optab_libfunc (binoptab, mode); |
| if (!libfunc) |
| return false; |
| |
| /* The value returned by the library function will have twice as |
| many bits as the nominal MODE. */ |
| libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode)); |
| start_sequence (); |
| libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, |
| libval_mode, |
| op0, mode, |
| op1, mode); |
| /* Get the part of VAL containing the value that we want. */ |
| libval = simplify_gen_subreg (mode, libval, libval_mode, |
| targ0 ? 0 : GET_MODE_SIZE (mode)); |
| insns = get_insns (); |
| end_sequence (); |
| /* Move the into the desired location. */ |
| emit_libcall_block (insns, targ0 ? targ0 : targ1, libval, |
| gen_rtx_fmt_ee (code, mode, op0, op1)); |
| |
| return true; |
| } |
| |
| |
| /* Wrapper around expand_unop which takes an rtx code to specify |
| the operation to perform, not an optab pointer. All other |
| arguments are the same. */ |
| rtx |
| expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0, |
| rtx target, int unsignedp) |
| { |
| optab unop = code_to_optab (code); |
| gcc_assert (unop); |
| |
| return expand_unop (mode, unop, op0, target, unsignedp); |
| } |
| |
| /* Try calculating |
| (clz:narrow x) |
| as |
| (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)). |
| |
| A similar operation can be used for clrsb. UNOPTAB says which operation |
| we are trying to expand. */ |
| static rtx |
| widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab) |
| { |
| opt_scalar_int_mode wider_mode_iter; |
| FOR_EACH_WIDER_MODE (wider_mode_iter, mode) |
| { |
| scalar_int_mode wider_mode = wider_mode_iter.require (); |
| if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) |
| { |
| rtx xop0, temp; |
| rtx_insn *last; |
| |
| last = get_last_insn (); |
| |
| if (target == 0) |
| target = gen_reg_rtx (mode); |
| xop0 = widen_operand (op0, wider_mode, mode, |
| unoptab != clrsb_optab, false); |
| temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, |
| unoptab != clrsb_optab); |
| if (temp != 0) |
| temp = expand_binop |
| (wider_mode, sub_optab, temp, |
| gen_int_mode (GET_MODE_PRECISION (wider_mode) |
| - GET_MODE_PRECISION (mode), |
| wider_mode), |
| target, true, OPTAB_DIRECT); |
| if (temp == 0) |
| delete_insns_since (last); |
| |
| return temp; |
| } |
| } |
| return 0; |
| } |
| |
| /* Attempt to emit (clrsb:mode op0) as |
| (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1)))) |
| (const_int -1)) |
| if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec, |
| or as |
| (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1)) |
| (ashr:mode op0 (const_int prec-1))) |
| (const_int 1))) |
| otherwise. */ |
| |
| static rtx |
| expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| if (optimize_insn_for_size_p () |
| || optab_handler (clz_optab, mode) == CODE_FOR_nothing) |
| return NULL_RTX; |
| |
| start_sequence (); |
| HOST_WIDE_INT val = 0; |
| if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2 |
| || val != GET_MODE_PRECISION (mode)) |
| val = 0; |
| else |
| val = 1; |
| |
| rtx temp2 = op0; |
| if (!val) |
| { |
| temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (!temp2) |
| { |
| fail: |
| end_sequence (); |
| return NULL_RTX; |
| } |
| } |
| |
| rtx temp = expand_binop (mode, ashr_optab, op0, |
| GEN_INT (GET_MODE_PRECISION (mode) - 1), |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| |
| temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0, |
| OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| |
| if (!val) |
| { |
| temp = expand_binop (mode, ior_optab, temp, const1_rtx, |
| NULL_RTX, 0, OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| } |
| temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target, |
| true); |
| if (!temp) |
| goto fail; |
| if (val) |
| { |
| temp = expand_binop (mode, add_optab, temp, constm1_rtx, |
| target, 0, OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| } |
| |
| rtx_insn *seq = get_insns (); |
| end_sequence (); |
| |
| add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode); |
| emit_insn (seq); |
| return temp; |
| } |
| |
| /* Try calculating clz of a double-word quantity as two clz's of word-sized |
| quantities, choosing which based on whether the high word is nonzero. */ |
| static rtx |
| expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| rtx xop0 = force_reg (mode, op0); |
| rtx subhi = gen_highpart (word_mode, xop0); |
| rtx sublo = gen_lowpart (word_mode, xop0); |
| rtx_code_label *hi0_label = gen_label_rtx (); |
| rtx_code_label *after_label = gen_label_rtx (); |
| rtx_insn *seq; |
| rtx temp, result; |
| |
| /* If we were not given a target, use a word_mode register, not a |
| 'mode' register. The result will fit, and nobody is expecting |
| anything bigger (the return type of __builtin_clz* is int). */ |
| if (!target) |
| target = gen_reg_rtx (word_mode); |
| |
| /* In any case, write to a word_mode scratch in both branches of the |
| conditional, so we can ensure there is a single move insn setting |
| 'target' to tag a REG_EQUAL note on. */ |
| result = gen_reg_rtx (word_mode); |
| |
| start_sequence (); |
| |
| /* If the high word is not equal to zero, |
| then clz of the full value is clz of the high word. */ |
| emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0, |
| word_mode, true, hi0_label); |
| |
| temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true); |
| if (!temp) |
| goto fail; |
| |
| if (temp != result) |
| convert_move (result, temp, true); |
| |
| emit_jump_insn (targetm.gen_jump (after_label)); |
| emit_barrier (); |
| |
| /* Else clz of the full value is clz of the low word plus the number |
| of bits in the high word. */ |
| emit_label (hi0_label); |
| |
| temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true); |
| if (!temp) |
| goto fail; |
| temp = expand_binop (word_mode, add_optab, temp, |
| gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode), |
| result, true, OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| if (temp != result) |
| convert_move (result, temp, true); |
| |
| emit_label (after_label); |
| convert_move (target, result, true); |
| |
| seq = get_insns (); |
| end_sequence (); |
| |
| add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode); |
| emit_insn (seq); |
| return target; |
| |
| fail: |
| end_sequence (); |
| return 0; |
| } |
| |
| /* Try calculating popcount of a double-word quantity as two popcount's of |
| word-sized quantities and summing up the results. */ |
| static rtx |
| expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| rtx t0, t1, t; |
| rtx_insn *seq; |
| |
| start_sequence (); |
| |
| t0 = expand_unop_direct (word_mode, popcount_optab, |
| operand_subword_force (op0, 0, mode), NULL_RTX, |
| true); |
| t1 = expand_unop_direct (word_mode, popcount_optab, |
| operand_subword_force (op0, 1, mode), NULL_RTX, |
| true); |
| if (!t0 || !t1) |
| { |
| end_sequence (); |
| return NULL_RTX; |
| } |
| |
| /* If we were not given a target, use a word_mode register, not a |
| 'mode' register. The result will fit, and nobody is expecting |
| anything bigger (the return type of __builtin_popcount* is int). */ |
| if (!target) |
| target = gen_reg_rtx (word_mode); |
| |
| t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT); |
| |
| seq = get_insns (); |
| end_sequence (); |
| |
| add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode); |
| emit_insn (seq); |
| return t; |
| } |
| |
| /* Try calculating |
| (parity:wide x) |
| as |
| (parity:narrow (low (x) ^ high (x))) */ |
| static rtx |
| expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| rtx t = expand_binop (word_mode, xor_optab, |
| operand_subword_force (op0, 0, mode), |
| operand_subword_force (op0, 1, mode), |
| NULL_RTX, 0, OPTAB_DIRECT); |
| return expand_unop (word_mode, parity_optab, t, target, true); |
| } |
| |
| /* Try calculating |
| (bswap:narrow x) |
| as |
| (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */ |
| static rtx |
| widen_bswap (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| rtx x; |
| rtx_insn *last; |
| opt_scalar_int_mode wider_mode_iter; |
| |
| FOR_EACH_WIDER_MODE (wider_mode_iter, mode) |
| if (optab_handler (bswap_optab, wider_mode_iter.require ()) |
| != CODE_FOR_nothing) |
| break; |
| |
| if (!wider_mode_iter.exists ()) |
| return NULL_RTX; |
| |
| scalar_int_mode wider_mode = wider_mode_iter.require (); |
| last = get_last_insn (); |
| |
| x = widen_operand (op0, wider_mode, mode, true, true); |
| x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true); |
| |
| gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode) |
| && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)); |
| if (x != 0) |
| x = expand_shift (RSHIFT_EXPR, wider_mode, x, |
| GET_MODE_BITSIZE (wider_mode) |
| - GET_MODE_BITSIZE (mode), |
| NULL_RTX, true); |
| |
| if (x != 0) |
| { |
| if (target == 0) |
| target = gen_reg_rtx (mode); |
| emit_move_insn (target, gen_lowpart (mode, x)); |
| } |
| else |
| delete_insns_since (last); |
| |
| return target; |
| } |
| |
| /* Try calculating bswap as two bswaps of two word-sized operands. */ |
| |
| static rtx |
| expand_doubleword_bswap (machine_mode mode, rtx op, rtx target) |
| { |
| rtx t0, t1; |
| |
| t1 = expand_unop (word_mode, bswap_optab, |
| operand_subword_force (op, 0, mode), NULL_RTX, true); |
| t0 = expand_unop (word_mode, bswap_optab, |
| operand_subword_force (op, 1, mode), NULL_RTX, true); |
| |
| if (target == 0 || !valid_multiword_target_p (target)) |
| target = gen_reg_rtx (mode); |
| if (REG_P (target)) |
| emit_clobber (target); |
| emit_move_insn (operand_subword (target, 0, 1, mode), t0); |
| emit_move_insn (operand_subword (target, 1, 1, mode), t1); |
| |
| return target; |
| } |
| |
| /* Try calculating (parity x) as (and (popcount x) 1), where |
| popcount can also be done in a wider mode. */ |
| static rtx |
| expand_parity (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| enum mode_class mclass = GET_MODE_CLASS (mode); |
| opt_scalar_int_mode wider_mode_iter; |
| FOR_EACH_MODE_FROM (wider_mode_iter, mode) |
| { |
| scalar_int_mode wider_mode = wider_mode_iter.require (); |
| if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing) |
| { |
| rtx xop0, temp; |
| rtx_insn *last; |
| |
| last = get_last_insn (); |
| |
| if (target == 0 || GET_MODE (target) != wider_mode) |
| target = gen_reg_rtx (wider_mode); |
| |
| xop0 = widen_operand (op0, wider_mode, mode, true, false); |
| temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX, |
| true); |
| if (temp != 0) |
| temp = expand_binop (wider_mode, and_optab, temp, const1_rtx, |
| target, true, OPTAB_DIRECT); |
| |
| if (temp) |
| { |
| if (mclass != MODE_INT |
| || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) |
| return convert_to_mode (mode, temp, 0); |
| else |
| return gen_lowpart (mode, temp); |
| } |
| else |
| delete_insns_since (last); |
| } |
| } |
| return 0; |
| } |
| |
| /* Try calculating ctz(x) as K - clz(x & -x) , |
| where K is GET_MODE_PRECISION(mode) - 1. |
| |
| Both __builtin_ctz and __builtin_clz are undefined at zero, so we |
| don't have to worry about what the hardware does in that case. (If |
| the clz instruction produces the usual value at 0, which is K, the |
| result of this code sequence will be -1; expand_ffs, below, relies |
| on this. It might be nice to have it be K instead, for consistency |
| with the (very few) processors that provide a ctz with a defined |
| value, but that would take one more instruction, and it would be |
| less convenient for expand_ffs anyway. */ |
| |
| static rtx |
| expand_ctz (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| rtx_insn *seq; |
| rtx temp; |
| |
| if (optab_handler (clz_optab, mode) == CODE_FOR_nothing) |
| return 0; |
| |
| start_sequence (); |
| |
| temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true); |
| if (temp) |
| temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX, |
| true, OPTAB_DIRECT); |
| if (temp) |
| temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true); |
| if (temp) |
| temp = expand_binop (mode, sub_optab, |
| gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode), |
| temp, target, |
| true, OPTAB_DIRECT); |
| if (temp == 0) |
| { |
| end_sequence (); |
| return 0; |
| } |
| |
| seq = get_insns (); |
| end_sequence (); |
| |
| add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode); |
| emit_insn (seq); |
| return temp; |
| } |
| |
| |
| /* Try calculating ffs(x) using ctz(x) if we have that instruction, or |
| else with the sequence used by expand_clz. |
| |
| The ffs builtin promises to return zero for a zero value and ctz/clz |
| may have an undefined value in that case. If they do not give us a |
| convenient value, we have to generate a test and branch. */ |
| static rtx |
| expand_ffs (scalar_int_mode mode, rtx op0, rtx target) |
| { |
| HOST_WIDE_INT val = 0; |
| bool defined_at_zero = false; |
| rtx temp; |
| rtx_insn *seq; |
| |
| if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing) |
| { |
| start_sequence (); |
| |
| temp = expand_unop_direct (mode, ctz_optab, op0, 0, true); |
| if (!temp) |
| goto fail; |
| |
| defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2); |
| } |
| else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing) |
| { |
| start_sequence (); |
| temp = expand_ctz (mode, op0, 0); |
| if (!temp) |
| goto fail; |
| |
| if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2) |
| { |
| defined_at_zero = true; |
| val = (GET_MODE_PRECISION (mode) - 1) - val; |
| } |
| } |
| else |
| return 0; |
| |
| if (defined_at_zero && val == -1) |
| /* No correction needed at zero. */; |
| else |
| { |
| /* We don't try to do anything clever with the situation found |
| on some processors (eg Alpha) where ctz(0:mode) == |
| bitsize(mode). If someone can think of a way to send N to -1 |
| and leave alone all values in the range 0..N-1 (where N is a |
| power of two), cheaper than this test-and-branch, please add it. |
| |
| The test-and-branch is done after the operation itself, in case |
| the operation sets condition codes that can be recycled for this. |
| (This is true on i386, for instance.) */ |
| |
| rtx_code_label *nonzero_label = gen_label_rtx (); |
| emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0, |
| mode, true, nonzero_label); |
| |
| convert_move (temp, GEN_INT (-1), false); |
| emit_label (nonzero_label); |
| } |
| |
| /* temp now has a value in the range -1..bitsize-1. ffs is supposed |
| to produce a value in the range 0..bitsize. */ |
| temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode), |
| target, false, OPTAB_DIRECT); |
| if (!temp) |
| goto fail; |
| |
| seq = get_insns (); |
| end_sequence (); |
| |
| add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode); |
| emit_insn (seq); |
| return temp; |
| |
| fail: |
| end_sequence (); |
| return 0; |
| } |
| |
| /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain |
| conditions, VAL may already be a SUBREG against which we cannot generate |
| a further SUBREG. In this case, we expect forcing the value into a |
| register will work around the situation. */ |
| |
| static rtx |
| lowpart_subreg_maybe_copy (machine_mode omode, rtx val, |
| machine_mode imode) |
| { |
| rtx ret; |
| ret = lowpart_subreg (omode, val, imode); |
| if (ret == NULL) |
| { |
| val = force_reg (imode, val); |
| ret = lowpart_subreg (omode, val, imode); |
| gcc_assert (ret != NULL); |
| } |
| return ret; |
| } |
| |
| /* Expand a floating point absolute value or negation operation via a |
| logical operation on the sign bit. */ |
| |
| static rtx |
| expand_absneg_bit (enum rtx_code code, scalar_float_mode mode, |
| rtx op0, rtx target) |
| { |
| const struct real_format *fmt; |
| int bitpos, word, nwords, i; |
| scalar_int_mode imode; |
| rtx temp; |
| rtx_insn *insns; |
| |
| /* The format has to have a simple sign bit. */ |
| fmt = REAL_MODE_FORMAT (mode); |
| if (fmt == NULL) |
| return NULL_RTX; |
| |
| bitpos = fmt->signbit_rw; |
| if (bitpos < 0) |
| return NULL_RTX; |
| |
| /* Don't create negative zeros if the format doesn't support them. */ |
| if (code == NEG && !fmt->has_signed_zero) |
| return NULL_RTX; |
| |
| if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) |
| { |
| if (!int_mode_for_mode (mode).exists (&imode)) |
| return NULL_RTX; |
| word = 0; |
| nwords = 1; |
| } |
| else |
| { |
| imode = word_mode; |
| |
| if (FLOAT_WORDS_BIG_ENDIAN) |
| word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD; |
| else |
|