gcc/match.pd

/* Match-and-simplify patterns for shared GENERIC and GIMPLE folding.
   This file is consumed by genmatch which produces gimple-match.cc
   and generic-match.cc from it.

   Copyright (C) 2014-2026 Free Software Foundation, Inc.
   Contributed by Richard Biener <rguenther@suse.de>
   and Prathamesh Kulkarni  <bilbotheelffriend@gmail.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */


/* Generic tree predicates we inherit.  */
(define_predicates
   integer_onep integer_zerop integer_all_onesp integer_minus_onep
   integer_each_onep integer_truep integer_nonzerop
   real_zerop real_onep real_minus_onep
   zerop
   initializer_each_zero_or_onep
   CONSTANT_CLASS_P
   poly_int_tree_p
   tree_expr_nonnegative_p
   tree_expr_nonzero_p
   integer_valued_real_p
   integer_pow2p
   uniform_integer_cst_p
   HONOR_NANS
   uniform_vector_p
   expand_vec_cmp_expr_p
   bitmask_inv_cst_vector_p)

/* Operator lists.  */
(define_operator_list tcc_comparison
  lt   le   eq ne ge   gt   unordered ordered   unlt unle ungt unge uneq ltgt)
(define_operator_list inverted_tcc_comparison
  ge   gt   ne eq lt   le   ordered   unordered ge   gt   le   lt   ltgt uneq)
(define_operator_list inverted_tcc_comparison_with_nans
  unge ungt ne eq unlt unle ordered   unordered ge   gt   le   lt   ltgt uneq)
(define_operator_list swapped_tcc_comparison
  gt   ge   eq ne le   lt   unordered ordered   ungt unge unlt unle uneq ltgt)
(define_operator_list simple_comparison         lt   le   eq ne ge   gt)
(define_operator_list swapped_simple_comparison gt   ge   eq ne le   lt)

#include "cfn-operators.pd"

/* Define operand lists for math rounding functions {,i,l,ll}FN,
   where the versions prefixed with "i" return an int, those prefixed with
   "l" return a long and those prefixed with "ll" return a long long.

   Also define operand lists:

     X<FN>F for all float functions, in the order i, l, ll
     X<FN> for all double functions, in the same order
     X<FN>L for all long double functions, in the same order.  */
#define DEFINE_INT_AND_FLOAT_ROUND_FN(FN) \
  (define_operator_list X##FN##F BUILT_IN_I##FN##F \
				 BUILT_IN_L##FN##F \
				 BUILT_IN_LL##FN##F) \
  (define_operator_list X##FN BUILT_IN_I##FN \
			      BUILT_IN_L##FN \
			      BUILT_IN_LL##FN) \
  (define_operator_list X##FN##L BUILT_IN_I##FN##L \
				 BUILT_IN_L##FN##L \
				 BUILT_IN_LL##FN##L)

DEFINE_INT_AND_FLOAT_ROUND_FN (FLOOR)
DEFINE_INT_AND_FLOAT_ROUND_FN (CEIL)
DEFINE_INT_AND_FLOAT_ROUND_FN (ROUND)
DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)

/* Unary operations and their associated IFN_COND_* function.  */
(define_operator_list UNCOND_UNARY
  negate bit_not)
(define_operator_list COND_UNARY
  IFN_COND_NEG IFN_COND_NOT)
(define_operator_list COND_LEN_UNARY
  IFN_COND_LEN_NEG IFN_COND_LEN_NOT)

/* Binary operations and their associated IFN_COND_* function.  */
(define_operator_list UNCOND_BINARY
  plus minus
  mult trunc_div trunc_mod rdiv
  min max
  IFN_FMIN IFN_FMAX IFN_COPYSIGN
  bit_and bit_ior bit_xor
  lshift rshift)
(define_operator_list COND_BINARY
  IFN_COND_ADD IFN_COND_SUB
  IFN_COND_MUL IFN_COND_DIV IFN_COND_MOD IFN_COND_RDIV
  IFN_COND_MIN IFN_COND_MAX
  IFN_COND_FMIN IFN_COND_FMAX IFN_COND_COPYSIGN
  IFN_COND_AND IFN_COND_IOR IFN_COND_XOR
  IFN_COND_SHL IFN_COND_SHR)
(define_operator_list COND_LEN_BINARY
  IFN_COND_LEN_ADD IFN_COND_LEN_SUB
  IFN_COND_LEN_MUL IFN_COND_LEN_DIV
  IFN_COND_LEN_MOD IFN_COND_LEN_RDIV
  IFN_COND_LEN_MIN IFN_COND_LEN_MAX
  IFN_COND_LEN_FMIN IFN_COND_LEN_FMAX IFN_COND_LEN_COPYSIGN
  IFN_COND_LEN_AND IFN_COND_LEN_IOR IFN_COND_LEN_XOR
  IFN_COND_LEN_SHL IFN_COND_LEN_SHR)

/* Same for ternary operations.  */
(define_operator_list UNCOND_TERNARY
  IFN_FMA IFN_FMS IFN_FNMA IFN_FNMS)
(define_operator_list COND_TERNARY
  IFN_COND_FMA IFN_COND_FMS IFN_COND_FNMA IFN_COND_FNMS)
(define_operator_list COND_LEN_TERNARY
  IFN_COND_LEN_FMA IFN_COND_LEN_FMS IFN_COND_LEN_FNMA IFN_COND_LEN_FNMS)

/* __atomic_fetch_or_*, __atomic_fetch_xor_*, __atomic_xor_fetch_*  */
(define_operator_list ATOMIC_FETCH_OR_XOR_N
  BUILT_IN_ATOMIC_FETCH_OR_1 BUILT_IN_ATOMIC_FETCH_OR_2
  BUILT_IN_ATOMIC_FETCH_OR_4 BUILT_IN_ATOMIC_FETCH_OR_8
  BUILT_IN_ATOMIC_FETCH_OR_16
  BUILT_IN_ATOMIC_FETCH_XOR_1 BUILT_IN_ATOMIC_FETCH_XOR_2
  BUILT_IN_ATOMIC_FETCH_XOR_4 BUILT_IN_ATOMIC_FETCH_XOR_8
  BUILT_IN_ATOMIC_FETCH_XOR_16
  BUILT_IN_ATOMIC_XOR_FETCH_1 BUILT_IN_ATOMIC_XOR_FETCH_2
  BUILT_IN_ATOMIC_XOR_FETCH_4 BUILT_IN_ATOMIC_XOR_FETCH_8
  BUILT_IN_ATOMIC_XOR_FETCH_16)
/* __sync_fetch_and_or_*, __sync_fetch_and_xor_*, __sync_xor_and_fetch_*  */
(define_operator_list SYNC_FETCH_OR_XOR_N
  BUILT_IN_SYNC_FETCH_AND_OR_1 BUILT_IN_SYNC_FETCH_AND_OR_2
  BUILT_IN_SYNC_FETCH_AND_OR_4 BUILT_IN_SYNC_FETCH_AND_OR_8
  BUILT_IN_SYNC_FETCH_AND_OR_16
  BUILT_IN_SYNC_FETCH_AND_XOR_1 BUILT_IN_SYNC_FETCH_AND_XOR_2
  BUILT_IN_SYNC_FETCH_AND_XOR_4 BUILT_IN_SYNC_FETCH_AND_XOR_8
  BUILT_IN_SYNC_FETCH_AND_XOR_16
  BUILT_IN_SYNC_XOR_AND_FETCH_1 BUILT_IN_SYNC_XOR_AND_FETCH_2
  BUILT_IN_SYNC_XOR_AND_FETCH_4 BUILT_IN_SYNC_XOR_AND_FETCH_8
  BUILT_IN_SYNC_XOR_AND_FETCH_16)
/* __atomic_fetch_and_*.  */
(define_operator_list ATOMIC_FETCH_AND_N
  BUILT_IN_ATOMIC_FETCH_AND_1 BUILT_IN_ATOMIC_FETCH_AND_2
  BUILT_IN_ATOMIC_FETCH_AND_4 BUILT_IN_ATOMIC_FETCH_AND_8
  BUILT_IN_ATOMIC_FETCH_AND_16)
/* __sync_fetch_and_and_*.  */
(define_operator_list SYNC_FETCH_AND_AND_N
  BUILT_IN_SYNC_FETCH_AND_AND_1 BUILT_IN_SYNC_FETCH_AND_AND_2
  BUILT_IN_SYNC_FETCH_AND_AND_4 BUILT_IN_SYNC_FETCH_AND_AND_8
  BUILT_IN_SYNC_FETCH_AND_AND_16)

/* With nop_convert? combine convert? and view_convert? in one pattern
   plus conditionalize on tree_nop_conversion_p conversions.  */
(match (nop_convert @0)
 (convert @0)
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))))
(match (nop_convert @0)
 (view_convert @0)
 (if (vector_nop_conversion_p (type, TREE_TYPE (@0)))))

/* These are used by gimple_bitwise_inverted_equal_p to simplify
   detection of BIT_NOT and comparisons. */
(match (bit_not_with_nop @0)
 (bit_not @0))
(match (bit_not_with_nop @0)
 (convert (bit_not @0))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))))
(match (bit_xor_cst @0 @1)
 (bit_xor @0 uniform_integer_cst_p@1))
(match (maybe_cmp @0)
 (tcc_comparison@0 @1 @2))
(match (maybe_cmp @0)
 (convert (tcc_comparison@0 @1 @2))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))))

/* `a ^ b` is another form of `a != b` when the type
    is a 1bit precision integer.  */
(match (maybe_cmp @0)
 (bit_xor@0 @1 @2)
 (if (INTEGRAL_TYPE_P (type)
      && TYPE_PRECISION (type) == 1)))
/* maybe_bit_not is used to match what
   is acceptable for bitwise_inverted_equal_p. */
(match (maybe_bit_not @0)
 (bit_not_with_nop@0 @1))
(match (maybe_bit_not @0)
 (INTEGER_CST@0))
(match (maybe_bit_not @0)
 (maybe_cmp@0 @1))
(match (maybe_bit_not @0)
 (bit_xor_cst@0 @1 @2))

#if GIMPLE
(match (maybe_truncate @0)
 (convert @0)
 (if (INTEGRAL_TYPE_P (type)
      && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@0)))))
#endif

/* Transform likes of (char) ABS_EXPR <(int) x> into (char) ABSU_EXPR <x>
   ABSU_EXPR returns unsigned absolute value of the operand and the operand
   of the ABSU_EXPR will have the corresponding signed type.  */
(simplify (abs (convert @0))
 (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !TYPE_UNSIGNED (TREE_TYPE (@0))
      && element_precision (type) > element_precision (TREE_TYPE (@0))
      && (!VECTOR_TYPE_P (type)
	  || target_supports_op_p (TREE_TYPE (@0), ABSU_EXPR, optab_vector)))
  (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
   (convert (absu:utype @0)))))

/* (trunc)abs (extend x) --> abs (x) for non-integral types.  */
(simplify
 (convert (abs (convert@1 @0)))
  (if (!ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !HONOR_SNANS (@1)
       && types_match (type, TREE_TYPE (@0))
       && element_precision (type) < element_precision (TREE_TYPE (@1))
       && (!VECTOR_TYPE_P (type)
	   || target_supports_op_p (type, ABS_EXPR, optab_default)))
   (abs @0)))

#if GIMPLE
/* Optimize (X + (X >> (prec - 1))) ^ (X >> (prec - 1)) into abs (X).  */
(simplify
 (bit_xor:c (plus:c @0 (rshift@2 @0 INTEGER_CST@1)) @2)
 (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !TYPE_UNSIGNED (TREE_TYPE (@0))
      && wi::to_widest (@1) == element_precision (TREE_TYPE (@0)) - 1)
  (abs @0)))
#endif

/* (X - (X < 0)) ^ -(X < 0) -> abs (X) */
(simplify
 (bit_xor:c (minus @0 (convert@1 (lt @0 integer_zerop)))
	    (negate @1))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (abs @0)))

/* (X ^ -(X < 0)) + (X < 0) -> abs (X) */
(simplify
 (plus:c (bit_xor:c @0 (negate (convert@1 (lt @0 integer_zerop)))) @1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !TYPE_UNSIGNED (TREE_TYPE (@0))
      && !TYPE_SATURATING (TREE_TYPE (@0))
      && (GIMPLE || !TREE_SIDE_EFFECTS (@0)))
  (abs @0)))

/* Following match patterns are used by the match_spaceship function to detect
   all possible spaceship combinations.  */
#if GIMPLE
(if (INTEGRAL_TYPE_P (type))
 (for ltne (lt ne)
      gtne (gt ne)
  (match (spaceship @0 @1)
   (cond^ (le @0 @1) (negate (convert? (ltne @0 @1))) integer_onep))
  (match (spaceship @0 @1)
    (cond^ (le @1 @0) (convert? (ltne @1 @0)) integer_minus_onep))
  (match (spaceship @0 @1)
    (cond^ (ge @0 @1) (convert? (gtne @0 @1)) integer_minus_onep))
  (match (spaceship @0 @1)
    (cond^ (ge @1 @0) (negate (convert? (gtne @1 @0))) integer_onep))))
#endif

/* Simplifications of operations with one constant operand and
   simplifications to constants or single values.  */

(for op (plus pointer_plus minus bit_ior bit_xor)
  (simplify
    (op @0 integer_zerop)
    (non_lvalue @0)))

/* 0 +p index -> (type)index */
(simplify
 (pointer_plus integer_zerop @1)
 (non_lvalue (convert @1)))

/* ptr - 0 -> (type)ptr */
(simplify
 (pointer_diff @0 integer_zerop)
 (convert @0))

/* See if ARG1 is zero and X + ARG1 reduces to X.
   Likewise if the operands are reversed.  */
(simplify
 (plus:c @0 real_zerop@1)
 (if (fold_real_zero_addition_p (type, @0, @1, 0))
  (non_lvalue @0)))

/* See if ARG1 is zero and X - ARG1 reduces to X.  */
(simplify
 (minus @0 real_zerop@1)
 (if (fold_real_zero_addition_p (type, @0, @1, 1))
  (non_lvalue @0)))

/* Even if the fold_real_zero_addition_p can't simplify X + 0.0
   into X, we can optimize (X + 0.0) + 0.0 or (X + 0.0) - 0.0
   or (X - 0.0) + 0.0 into X + 0.0 and (X - 0.0) - 0.0 into X - 0.0
   if not -frounding-math.  For sNaNs the first operation would raise
   exceptions but turn the result into qNan, so the second operation
   would not raise it.   */
(for inner_op (plus minus)
 (for outer_op (plus minus)
  (simplify
   (outer_op (inner_op@3 @0 REAL_CST@1) REAL_CST@2)
    (if (real_zerop (@1)
	 && real_zerop (@2)
	 && !HONOR_SIGN_DEPENDENT_ROUNDING (type))
     (with { bool inner_plus = ((inner_op == PLUS_EXPR)
				^ REAL_VALUE_MINUS_ZERO (TREE_REAL_CST (@1)));
	     bool outer_plus
	       = ((outer_op == PLUS_EXPR)
		  ^ REAL_VALUE_MINUS_ZERO (TREE_REAL_CST (@2))); }
      (if (outer_plus && !inner_plus)
       (outer_op @0 @2)
       @3))))))

/* Simplify x - x.
   This is unsafe for certain floats even in non-IEEE formats.
   In IEEE, it is unsafe because it does wrong for NaNs.
   PR middle-end/98420: x - x may be -0.0 with FE_DOWNWARD.
   Also note that operand_equal_p is always false if an operand
   is volatile.  */
(simplify
 (minus @0 @0)
 (if (!FLOAT_TYPE_P (type)
      || (!tree_expr_maybe_nan_p (@0)
	  && !tree_expr_maybe_infinite_p (@0)
	  && (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
	      || !HONOR_SIGNED_ZEROS (type))))
  { build_zero_cst (type); }))
(simplify
 (pointer_diff @@0 @0)
 { build_zero_cst (type); })

(simplify
 (mult @0 integer_zerop@1)
 @1)

#if GIMPLE
/* When multiplying a value by a boolean involving the value, we may
   be able to simplify further.
     a * ((a || b) != 0) -> a
     a * ((a || b) == 0) -> 0

   There are also bit-and cases which don't show up in practice yet.
     a * ((a && b) != 0) -> a * b
     a * ((a && b) == 0) -> b != 0 ? a : b */
(for neeq (ne eq)
 (simplify
  (mult:c (convert? (neeq (bit_ior:c @0 @1) integer_zerop@2)) @0)
  (if (neeq == EQ_EXPR)
   { build_zero_cst (type); }
   @0)))
#endif

/* -x == x -> x == 0 */
(for cmp (eq ne)
 (simplify
  (cmp:c @0 (negate @0))
   (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
        && !TYPE_OVERFLOW_WRAPS (TREE_TYPE(@0)))
    (cmp @0 { build_zero_cst (TREE_TYPE(@0)); }))))

/* Maybe fold x * 0 to 0.  The expressions aren't the same
   when x is NaN, since x * 0 is also NaN.  Nor are they the
   same in modes with signed zeros, since multiplying a
   negative value by 0 gives -0, not +0.  Nor when x is +-Inf,
   since x * 0 is NaN.  */
(simplify
 (mult @0 real_zerop@1)
 (if (!tree_expr_maybe_nan_p (@0)
      && (!HONOR_NANS (type) || !tree_expr_maybe_infinite_p (@0))
      && (!HONOR_SIGNED_ZEROS (type) || tree_expr_nonnegative_p (@0)))
  @1))

/* In IEEE floating point, x*1 is not equivalent to x for snans.
   Likewise for complex arithmetic with signed zeros.  */
(simplify
 (mult @0 real_onep)
 (if (!tree_expr_maybe_signaling_nan_p (@0)
      && (!HONOR_SIGNED_ZEROS (type)
          || !COMPLEX_FLOAT_TYPE_P (type)))
  (non_lvalue @0)))

/* Transform x * -1.0 into -x.  */
(simplify
 (mult @0 real_minus_onep)
  (if (!tree_expr_maybe_signaling_nan_p (@0)
       && (!HONOR_SIGNED_ZEROS (type)
           || !COMPLEX_FLOAT_TYPE_P (type)))
   (negate @0)))

/* Transform x * { 0 or 1, 0 or 1, ... } into x & { 0 or -1, 0 or -1, ...},
   unless the target has native support for the former but not the latter.  */
(simplify
 (mult @0 VECTOR_CST@1)
 (if (initializer_each_zero_or_onep (@1)
      && !HONOR_SNANS (type)
      && !HONOR_SIGNED_ZEROS (type))
  (with { tree itype = FLOAT_TYPE_P (type) ? unsigned_type_for (type) : type; }
   (if (itype
	&& (!VECTOR_MODE_P (TYPE_MODE (type))
	    || (VECTOR_MODE_P (TYPE_MODE (itype))
		&& optab_handler (and_optab,
				  TYPE_MODE (itype)) != CODE_FOR_nothing)))
    (view_convert (bit_and:itype (view_convert @0)
				 (ne @1 { build_zero_cst (type); })))))))

/* In SWAR (SIMD within a register) code a signed comparison of packed data
   can be constructed with a particular combination of shift, bitwise and,
   and multiplication by constants.  If that code is vectorized we can
   convert this pattern into a more efficient vector comparison.  */
(simplify
 (mult (bit_and (rshift @0 uniform_integer_cst_p@1)
	    uniform_integer_cst_p@2)
    uniform_integer_cst_p@3)
 (with {
   tree rshift_cst = uniform_integer_cst_p (@1);
   tree bit_and_cst = uniform_integer_cst_p (@2);
   tree mult_cst = uniform_integer_cst_p (@3);
  }
  /* Make sure we're working with vectors and uniform vector constants.  */
  (if (VECTOR_TYPE_P (type)
       && tree_fits_uhwi_p (rshift_cst)
       && tree_fits_uhwi_p (mult_cst)
       && tree_fits_uhwi_p (bit_and_cst))
   /* Compute what constants would be needed for this to represent a packed
      comparison based on the shift amount denoted by RSHIFT_CST.  */
   (with {
     HOST_WIDE_INT vec_elem_bits = vector_element_bits (type);
     poly_int64 vec_nelts = TYPE_VECTOR_SUBPARTS (type);
     poly_int64 vec_bits = vec_elem_bits * vec_nelts;
     unsigned HOST_WIDE_INT cmp_bits_i, bit_and_i, mult_i;
     unsigned HOST_WIDE_INT target_mult_i, target_bit_and_i;
     cmp_bits_i = tree_to_uhwi (rshift_cst) + 1;
     mult_i = tree_to_uhwi (mult_cst);
     target_mult_i = (HOST_WIDE_INT_1U << cmp_bits_i) - 1;
     bit_and_i = tree_to_uhwi (bit_and_cst);
     target_bit_and_i = 0;

     /* The bit pattern in BIT_AND_I should be a mask for the least
	significant bit of each packed element that is CMP_BITS wide.  */
     for (unsigned i = 0; i < vec_elem_bits / cmp_bits_i; i++)
       target_bit_and_i = (target_bit_and_i << cmp_bits_i) | 1U;
    }
    (if ((exact_log2 (cmp_bits_i)) >= 0
	 && cmp_bits_i < HOST_BITS_PER_WIDE_INT
	 && multiple_p (vec_bits, cmp_bits_i)
	 && vec_elem_bits <= HOST_BITS_PER_WIDE_INT
	 && target_mult_i == mult_i
	 && target_bit_and_i == bit_and_i)
     /* Compute the vector shape for the comparison and check if the target is
	able to expand the comparison with that type.  */
     (with {
       /* We're doing a signed comparison.  */
       tree cmp_type = build_nonstandard_integer_type (cmp_bits_i, 0);
       poly_int64 vector_type_nelts = exact_div (vec_bits, cmp_bits_i);
       tree vec_cmp_type = build_vector_type (cmp_type, vector_type_nelts);
       tree vec_truth_type = truth_type_for (vec_cmp_type);
       tree zeros = build_zero_cst (vec_cmp_type);
       tree ones = build_all_ones_cst (vec_cmp_type);
      }
      (if (expand_vec_cmp_expr_p (vec_cmp_type, vec_truth_type, LT_EXPR)
	   && expand_vec_cond_expr_p (vec_cmp_type, vec_truth_type))
       (view_convert:type (vec_cond (lt:vec_truth_type
				     (view_convert:vec_cmp_type @0)
				     { zeros; })
			   { ones; } { zeros; })))))))))

#if GIMPLE
/* (x & lowhalf_mask) * (y & lowhalf_mask)
   -> VEC_WIDEN_MULT_EVEN (VIEW_CONVERT (x), VIEW_CONVERT (y)), or
   VEC_WIDEN_MULT_ODD on big-endian targets where the low halves are the
   odd lanes.  */
(simplify
 (mult (bit_and @0 VECTOR_CST@2)
       (bit_and @1 @2))
 (if (uniform_vector_p (@2)
      /* Single-element results are excluded since targets disagree
	 on the mode of a vector(1) widening multiply result (V2DI -> TI
	 on s390 but V2DI -> V1TI on rs6000).  */
      && known_gt (TYPE_VECTOR_SUBPARTS (type), 1U))
  (with
   {
     auto elem = wi::to_wide (uniform_vector_p (@2));
     unsigned int outer_prec = TYPE_PRECISION (TREE_TYPE (type));
     unsigned int inner_prec = outer_prec / 2;
     /* The half-width lanes are zero-extended, so the multiply is
	unsigned even for a signed TYPE.  */
     tree inner_scalar = build_nonstandard_integer_type (inner_prec, 1);
     tree inner_type
       = build_vector_type (inner_scalar, TYPE_VECTOR_SUBPARTS (type) * 2);
     optab op = (BYTES_BIG_ENDIAN
		 ? vec_widen_umult_odd_optab : vec_widen_umult_even_optab);
   }
   (if (elem == wi::mask (inner_prec, false, outer_prec)
	/* Without a mode of exactly INNER_PREC bits (e.g. for QImode
	   elements) the half-width vector would be wider than TYPE.  */
	&& type_has_mode_precision_p (inner_scalar)
	&& GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_VECTOR_INT
	&& optab_handler (op, TYPE_MODE (inner_type)) != CODE_FOR_nothing)
    (if (BYTES_BIG_ENDIAN)
     (vec_widen_mult_odd (view_convert:inner_type @0)
			 (view_convert:inner_type @1))
     (vec_widen_mult_even (view_convert:inner_type @0)
			  (view_convert:inner_type @1)))))))
#endif

(for cmp (gt ge lt le)
     outp (convert convert negate negate)
     outn (negate negate convert convert)
 /* Transform X * (X > 0.0 ? 1.0 : -1.0) into abs(X). */
 /* Transform X * (X >= 0.0 ? 1.0 : -1.0) into abs(X). */
 /* Transform X * (X < 0.0 ? 1.0 : -1.0) into -abs(X). */
 /* Transform X * (X <= 0.0 ? 1.0 : -1.0) into -abs(X). */
 (simplify
  (mult:c @0 (cond (cmp @0 real_zerop) real_onep@1 real_minus_onep))
  (if (!tree_expr_maybe_nan_p (@0) && !HONOR_SIGNED_ZEROS (type))
   (outp (abs @0))))
 /* Transform X * (X > 0.0 ? -1.0 : 1.0) into -abs(X). */
 /* Transform X * (X >= 0.0 ? -1.0 : 1.0) into -abs(X). */
 /* Transform X * (X < 0.0 ? -1.0 : 1.0) into abs(X). */
 /* Transform X * (X <= 0.0 ? -1.0 : 1.0) into abs(X). */
 (simplify
  (mult:c @0 (cond (cmp @0 real_zerop) real_minus_onep real_onep@1))
  (if (!tree_expr_maybe_nan_p (@0) && !HONOR_SIGNED_ZEROS (type))
   (outn (abs @0)))))

/* Transform X * copysign (1.0, X) into abs(X). */
(simplify
 (mult:c @0 (COPYSIGN_ALL real_onep @0))
 (if (!tree_expr_maybe_nan_p (@0) && !HONOR_SIGNED_ZEROS (type))
  (abs @0)))

/* Transform X * copysign (1.0, -X) into -abs(X). */
(simplify
 (mult:c @0 (COPYSIGN_ALL real_onep (negate @0)))
 (if (!tree_expr_maybe_nan_p (@0) && !HONOR_SIGNED_ZEROS (type))
  (negate (abs @0))))

/* Transform copysign (CST, X) into copysign (ABS(CST), X). */
(simplify
 (COPYSIGN_ALL REAL_CST@0 @1)
 (if (REAL_VALUE_NEGATIVE (TREE_REAL_CST (@0)))
  (COPYSIGN_ALL (negate @0) @1)))

/* Transform c ? x * copysign (1, y) : z to c ? x ^ signs(y) : z.
   tree-ssa-math-opts.cc does the corresponding optimization for
   unconditional multiplications (via xorsign).  */
(simplify
 (IFN_COND_MUL:c @0 @1 (IFN_COPYSIGN real_onep @2) @3)
 (with { tree signs = sign_mask_for (type); }
  (if (signs)
   (with { tree inttype = TREE_TYPE (signs); }
    (view_convert:type
     (IFN_COND_XOR:inttype @0
      (view_convert:inttype @1)
      (bit_and (view_convert:inttype @2) { signs; })
      (view_convert:inttype @3)))))))

/* (x >= 0 ? x : 0) + (x <= 0 ? -x : 0) -> abs x.  */
(simplify
 (plus:c (max @0 integer_zerop) (max (negate @0) integer_zerop))
 (if (ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_UNDEFINED (type))
  (abs @0)))

#if GIMPLE
/* x + uMax(x, 1) -> uMax(x+x, 1)
   if x+x is known not to wrap/overflow.  */
(simplify
 (plus:c @0 (max:s@2 @0 integer_onep@1))
 (if (TYPE_UNSIGNED (type))
  (with { int_range_max vr0; }
   (if (gimple_match_range_of_expr (vr0, @0, @2)
	&& range_op_handler (PLUS_EXPR).overflow_free_p (vr0, vr0))
    (max (plus @0 @0) @1)))))
/* minmax (a - c, b) + c -> minmax (a, b + c) if a-c and b+c don't overflow.  */
(for minmax (min max)
 (simplify
  (plus:c @0 (minmax:cs@3 (minus @1 @0) @2))
  (with { int_range_max vr0, vr1, vr2; }
   (if (TYPE_UNSIGNED (type)
	&& gimple_match_range_of_expr (vr0, @0)
	&& gimple_match_range_of_expr (vr1, @1, @3)
	&& range_op_handler (MINUS_EXPR).overflow_free_p (vr1, vr0)
	&& gimple_match_range_of_expr (vr2, @2, @3)
	&& range_op_handler (PLUS_EXPR).overflow_free_p (vr0, vr2))
    (minmax @1 (plus @2 @0))))))

#endif

/* X * 1, X / 1 -> X.  */
(for op (mult trunc_div ceil_div floor_div round_div exact_div)
  (simplify
    (op @0 integer_onep)
    (non_lvalue @0)))

/* (A / (1 << B)) -> (A >> B).
   Only for unsigned A.  For signed A, this would not preserve rounding
   toward zero.
   For example: (-1 / ( 1 << B)) !=  -1 >> B.
   Also handle widening conversions, like:
   (A / (unsigned long long) (1U << B)) -> (A >> B)
   or
   (A / (unsigned long long) (1 << B)) -> (A >> B).
   If the left shift is signed, it can be done only if the upper bits
   of A starting from shift's type sign bit are zero, as
   (unsigned long long) (1 << 31) is -2147483648ULL, not 2147483648ULL,
   so it is valid only if A >> 31 is zero.  */
(for div (trunc_div exact_div)
 (simplify
  (div (convert?@0 @3) (convert2? (lshift integer_onep@1 @2)))
  (if ((TYPE_UNSIGNED (type) || tree_expr_nonnegative_p (@0))
       && (!VECTOR_TYPE_P (type)
	   || target_supports_op_p (type, RSHIFT_EXPR, optab_vector)
	   || target_supports_op_p (type, RSHIFT_EXPR, optab_scalar))
       && (useless_type_conversion_p (type, TREE_TYPE (@1))
	   || (element_precision (type) >= element_precision (TREE_TYPE (@1))
	       && (TYPE_UNSIGNED (TREE_TYPE (@1))
		   || (element_precision (type)
		       == element_precision (TREE_TYPE (@1)))
		   || (INTEGRAL_TYPE_P (type)
		       && (tree_nonzero_bits (@0)
			   & wi::mask (element_precision (TREE_TYPE (@1)) - 1,
				       true,
				       element_precision (type))) == 0)))))
    (if (!VECTOR_TYPE_P (type)
	 && useless_type_conversion_p (TREE_TYPE (@3), TREE_TYPE (@1))
	 && element_precision (TREE_TYPE (@3)) < element_precision (type))
     (convert (rshift @3 @2))
     (rshift @0 @2)))))

/* Preserve explicit divisions by 0: the C++ front-end wants to detect
   undefined behavior in constexpr evaluation, and assuming that the division
   traps enables better optimizations than these anyway.  */
(for div (trunc_div ceil_div floor_div round_div exact_div)
 /* 0 / X is always zero.  */
 (simplify
  (div integer_zerop@0 @1)
  /* But not for 0 / 0 so that we can get the proper warnings and errors.  */
  (if (!integer_zerop (@1))
   @0))
 /* X / -1 is -X.  */
 (simplify
  (div @0 integer_minus_onep@1)
  (if (!TYPE_UNSIGNED (type))
   (negate @0)))
 /* X / bool_range_Y is X.  */
 (simplify
  (div @0 SSA_NAME@1)
  (if (INTEGRAL_TYPE_P (type)
       && ssa_name_has_boolean_range (@1)
       && !flag_non_call_exceptions)
   @0))
 /* X / X is one.  */
 (simplify
  (div @0 @0)
  /* But not for 0 / 0 so that we can get the proper warnings and errors.
     And not for _Fract types where we can't build 1.  */
  (if (!ALL_FRACT_MODE_P (TYPE_MODE (type))
       && !integer_zerop (@0)
       && (!flag_non_call_exceptions || tree_expr_nonzero_p (@0)))
   { build_one_cst (type); }))
 /* X / abs (X) is X < 0 ? -1 : 1.  */
 (simplify
   (div:C @0 (abs @0))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& !integer_zerop (@0)
	&& (!flag_non_call_exceptions || tree_expr_nonzero_p (@0)))
    (cond (lt @0 { build_zero_cst (type); })
          { build_minus_one_cst (type); } { build_one_cst (type); })))
 /* X / -X is -1.  */
 (simplify
   (div:C @0 (negate @0))
   (if ((INTEGRAL_TYPE_P (type) || VECTOR_INTEGER_TYPE_P (type))
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& !integer_zerop (@0)
	&& (!flag_non_call_exceptions || tree_expr_nonzero_p (@0)))
    { build_minus_one_cst (type); })))

/* For unsigned integral types, FLOOR_DIV_EXPR is the same as
   TRUNC_DIV_EXPR.  Rewrite into the latter in this case.  Similarly
   for MOD instead of DIV.  */
(for floor_divmod (floor_div floor_mod)
     trunc_divmod (trunc_div trunc_mod)
 (simplify
  (floor_divmod @0 @1)
  (if ((INTEGRAL_TYPE_P (type) || VECTOR_INTEGER_TYPE_P (type))
       && TYPE_UNSIGNED (type))
   (trunc_divmod @0 @1))))

/* 1 / X -> X == 1 for positive integer X.
   1 / X -> X >= -1 && X <= 1 ? X : 0 for when X could be negative.
   But not for 1 / 0 so that we can get proper warnings and errors,
   and not for 1-bit integers as they are edge cases better handled
   elsewhere.  Delay the conversion of the signed division until late
   because `1 / X` is simpler to handle than the resulting COND_EXPR.  */
(simplify
 (trunc_div integer_onep@0 @1)
 (if (INTEGRAL_TYPE_P (type)
      && TYPE_PRECISION (type) > 1
      && !integer_zerop (@1)
      && (!flag_non_call_exceptions || tree_expr_nonzero_p (@1)))
  (if (TYPE_UNSIGNED (type)
       || tree_expr_nonnegative_p (@1))
   (convert (eq:boolean_type_node @1 @0))
   (if (fold_before_rtl_expansion_p ())
    (with { tree utype = unsigned_type_for (type); }
     (cond (le (plus (convert:utype @1) { build_one_cst (utype); })
		{ build_int_cst (utype, 2); })
      @1 { build_zero_cst (type); }))))))

/* Combine two successive divisions.  Note that combining ceil_div
   and floor_div is trickier and combining round_div even more so.  */
(for div (trunc_div exact_div)
 (simplify
  (div (div@3 @0 INTEGER_CST@1) INTEGER_CST@2)
  (with {
    wi::overflow_type overflow;
    wide_int mul = wi::mul (wi::to_wide (@1), wi::to_wide (@2),
			    TYPE_SIGN (type), &overflow);
   }
   (if (div == EXACT_DIV_EXPR
	|| optimize_successive_divisions_p (@2, @3))
    (if (!overflow)
     (div @0 { wide_int_to_tree (type, mul); })
     (if (TYPE_UNSIGNED (type)
	  || mul != wi::min_value (TYPE_PRECISION (type), SIGNED))
      { build_zero_cst (type); }))))))

/* Combine successive multiplications.  Similar to above, but handling
   overflow is different.  */
(simplify
 (mult (mult @0 INTEGER_CST@1) INTEGER_CST@2)
 (with {
   wi::overflow_type overflow;
   wide_int mul = wi::mul (wi::to_wide (@1), wi::to_wide (@2),
			   TYPE_SIGN (type), &overflow);
  }
  /* Skip folding on overflow: the only special case is @1 * @2 == -INT_MIN,
     otherwise undefined overflow implies that @0 must be zero.  */
  (if (!overflow || TYPE_OVERFLOW_WRAPS (type))
   (mult @0 { wide_int_to_tree (type, mul); }))))

/* Similar to above, but there could be an extra add/sub between
   successive multuiplications.  */
(simplify
 (mult (plus:s@5 (mult:s@4 @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
 (with {
   bool overflowed = true;
   wi::overflow_type ovf1, ovf2;
   wide_int mul = wi::mul (wi::to_wide (@1), wi::to_wide (@3),
			   TYPE_SIGN (type), &ovf1);
   wide_int add = wi::mul (wi::to_wide (@2), wi::to_wide (@3),
			   TYPE_SIGN (type), &ovf2);
  if (TYPE_OVERFLOW_UNDEFINED (type))
    {
#if GIMPLE
      int_range_max vr0;
      if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE
	  && gimple_match_range_of_expr (vr0, @4, @5)
	  && !vr0.varying_p ())
	{
	  wide_int wmin0 = vr0.lower_bound ();
	  wide_int wmax0 = vr0.upper_bound ();
	  wmin0 = wi::mul (wmin0, wi::to_wide (@3), TYPE_SIGN (type), &ovf1);
	  wmax0 = wi::mul (wmax0, wi::to_wide (@3), TYPE_SIGN (type), &ovf2);
	  if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE)
	    {
	      wi::add (wmin0, add, TYPE_SIGN (type), &ovf1);
	      wi::add (wmax0, add, TYPE_SIGN (type), &ovf2);
	      if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE)
		overflowed = false;
	    }
	}
#endif
    }
  else
   overflowed = false;
 }
  /* Skip folding on overflow.  */
  (if (!overflowed)
   (plus (mult @0 { wide_int_to_tree (type, mul); })
	 { wide_int_to_tree (type, add); }))))

/* Similar to above, but a multiplication between successive additions.  */
(simplify
 (plus (mult:s (plus:s@4 @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
 (with {
   bool overflowed = true;
   wi::overflow_type ovf1;
   wi::overflow_type ovf2;
   wide_int mul = wi::mul (wi::to_wide (@1), wi::to_wide (@2),
			   TYPE_SIGN (type), &ovf1);
   wide_int add = wi::add (mul, wi::to_wide (@3),
			   TYPE_SIGN (type), &ovf2);
  if (TYPE_OVERFLOW_UNDEFINED (type))
    {
#if GIMPLE
      int_range_max vr0;
      if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE
	  && gimple_match_range_of_expr (vr0, @0, @4)
	  && !vr0.varying_p ())
	{
	  wide_int wmin0 = vr0.lower_bound ();
	  wide_int wmax0 = vr0.upper_bound ();
	  wmin0 = wi::mul (wmin0, wi::to_wide (@2), TYPE_SIGN (type), &ovf1);
	  wmax0 = wi::mul (wmax0, wi::to_wide (@2), TYPE_SIGN (type), &ovf2);
	  if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE)
	    {
	      wi::add (wmin0, mul, TYPE_SIGN (type), &ovf1);
	      wi::add (wmax0, mul, TYPE_SIGN (type), &ovf2);
	      if (ovf1 == wi::OVF_NONE && ovf2 == wi::OVF_NONE)
		overflowed = false;
	    }
	}
#endif
    }
  else
   overflowed = false;
 }
  /* Skip folding on overflow.  */
  (if (!overflowed)
   (plus (mult @0 @2) { wide_int_to_tree (type, add); }))))

/* Optimize A / A to 1.0 if we don't care about
   NaNs or Infinities.  */
(simplify
 (rdiv @0 @0)
 (if (FLOAT_TYPE_P (type)
      && ! HONOR_NANS (type)
      && ! HONOR_INFINITIES (type))
  { build_one_cst (type); }))

/* Optimize -A / A to -1.0 if we don't care about
   NaNs or Infinities.  */
(simplify
 (rdiv:C @0 (negate @0))
 (if (FLOAT_TYPE_P (type)
      && ! HONOR_NANS (type)
      && ! HONOR_INFINITIES (type))
  { build_minus_one_cst (type); }))

/* PR71078: x / abs(x) -> copysign (1.0, x) */
(simplify
 (rdiv:C (convert? @0) (convert? (abs @0)))
  (if (SCALAR_FLOAT_TYPE_P (type)
       && ! HONOR_NANS (type)
       && ! HONOR_INFINITIES (type))
   (switch
    (if (types_match (type, float_type_node))
     (BUILT_IN_COPYSIGNF { build_one_cst (type); } (convert @0)))
    (if (types_match (type, double_type_node))
     (BUILT_IN_COPYSIGN { build_one_cst (type); } (convert @0)))
    (if (types_match (type, long_double_type_node))
     (BUILT_IN_COPYSIGNL { build_one_cst (type); } (convert @0))))))

/* In IEEE floating point, x/1 is not equivalent to x for snans.  */
(simplify
 (rdiv @0 real_onep)
 (if (!tree_expr_maybe_signaling_nan_p (@0))
  (non_lvalue @0)))

/* In IEEE floating point, x/-1 is not equivalent to -x for snans.  */
(simplify
 (rdiv @0 real_minus_onep)
 (if (!tree_expr_maybe_signaling_nan_p (@0))
  (negate @0)))

(if (flag_reciprocal_math)
 /* Convert (A/B)/C to A/(B*C). */
 (simplify
  (rdiv (rdiv:s @0 @1) @2)
  (rdiv @0 (mult @1 @2)))

 /* Convert (A/B) * (C/D) to (A*C) / (B*D).  Two divisions become one
    multiply and one division, and a division costs several multiplies on
    every target.

    The two products are new places for the exponent to leave the range, so
    the rule needs more than the rounding licence: with B and D both large
    B * D is an infinity and the quotient becomes inf / inf, and with both
    small it is a zero and the quotient becomes 0 / 0.  Either way a finite
    result turns into a NaN, so the rule is restricted to the case where
    infinities and NaNs are excluded, as the cancellation rules above are.

    Both quotients have to be dead outside the product, otherwise a division
    would be added rather than removed, and a shared reciprocal is better
    left alone for the multiplications to reuse.  That is a hard requirement
    rather than a :s marker, because :s only forbids emitting new statements
    and both products can fold away to nothing, as they do for X * X where X
    is one reciprocal square root.  Requiring two singly used quotients also
    excludes that case, since a value feeding both operands of the product
    has two uses.  */
 (simplify
  (mult (rdiv@4 @0 @1) (rdiv@5 @2 @3))
  (if (flag_associative_math
       && !HONOR_NANS (type) && !HONOR_INFINITIES (type)
       && !flag_trapping_math
       && (TREE_CODE (type) != COMPLEX_TYPE
	   || !HONOR_SIGNED_ZEROS (type))
       && single_use (@4) && single_use (@5))
   (rdiv (mult @0 @2) (mult @1 @3))))

 /* Canonicalize x / (C1 * y) to (x * C2) / y.  */
 (simplify
  (rdiv @0 (mult:s @1 REAL_CST@2))
  (with
   { tree tem = const_binop (RDIV_EXPR, type, build_one_cst (type), @2); }
   (if (tem)
    (rdiv (mult @0 { tem; } ) @1))))

 /* Convert A/(B/C) to (A/B)*C  */
 (simplify
  (rdiv @0 (rdiv:s @1 @2))
   (mult (rdiv @0 @1) @2)))

/* Simplify x / (- y) to -x / y.  */
(simplify
 (rdiv @0 (negate @1))
 (rdiv (negate @0) @1))

/* convert semantics of round(x) function to floor(x+0.5).  */
/* (x-floor(x)) < (ceil(x)-x) ? floor(x) : ceil(x) --> floor(x+0.5).  */
/* (x-floor(x)) >= (ceil(x)-x) ? ceil(x) : floor(x) --> floor(x+0.5).  */
/* (ceil(x)-x) > (x-floor(x)) ? floor(x) : ceil(x) --> floor(x+0.5).  */
/* (ceil(x)-x) <= (x-floor(x)) ? ceil(x) : floor(x) --> floor(x+0.5).  */
(for op (lt lt lt lt ge ge ge ge)
     bt (BUILT_IN_FLOORF BUILT_IN_FLOOR BUILT_IN_FLOORL IFN_FLOOR
	 BUILT_IN_CEILF BUILT_IN_CEIL BUILT_IN_CEILL IFN_CEIL)
     bf (BUILT_IN_CEILF BUILT_IN_CEIL BUILT_IN_CEILL IFN_CEIL
	 BUILT_IN_FLOORF BUILT_IN_FLOOR BUILT_IN_FLOORL IFN_FLOOR)
     floor (BUILT_IN_FLOORF BUILT_IN_FLOOR BUILT_IN_FLOORL IFN_FLOOR
	    BUILT_IN_FLOORF BUILT_IN_FLOOR BUILT_IN_FLOORL IFN_FLOOR)
     ceil (BUILT_IN_CEILF BUILT_IN_CEIL BUILT_IN_CEILL IFN_CEIL
	   BUILT_IN_CEILF BUILT_IN_CEIL BUILT_IN_CEILL IFN_CEIL)
 (simplify
  (cond (op:c (minus:s @0 (floor @0))
	      (minus:s (ceil @0) @0))
	(bt @0) (bf @0))
  (if (!HONOR_SIGNED_ZEROS (type) && !HONOR_SIGN_DEPENDENT_ROUNDING (type))
   (floor (plus @0 { build_real (type, dconsthalf); })))))

(if (flag_unsafe_math_optimizations)
 /* Simplify (C / x op 0.0) to x op 0.0 for C != 0, C != Inf/Nan.
    Since C / x may underflow to zero, do this only for unsafe math.  */
 (for op (lt le gt ge)
      neg_op (gt ge lt le)
  (simplify
   (op (rdiv REAL_CST@0 @1) real_zerop@2)
   (if (!HONOR_SIGNED_ZEROS (@1) && !HONOR_INFINITIES (@1))
    (switch
     (if (real_less (&dconst0, TREE_REAL_CST_PTR (@0)))
      (op @1 @2))
     /* For C < 0, use the inverted operator.  */
     (if (real_less (TREE_REAL_CST_PTR (@0), &dconst0))
      (neg_op @1 @2)))))))

/* Optimize (X & (-A)) / A where A is a power of 2, to X >> log2(A) */
(for div (trunc_div ceil_div floor_div round_div exact_div)
 (simplify
  (div (convert? (bit_and @0 INTEGER_CST@1)) INTEGER_CST@2)
  (if (integer_pow2p (@2)
       && tree_int_cst_sgn (@2) > 0
       && tree_nop_conversion_p (type, TREE_TYPE (@0))
       && wi::to_wide (@2) + wi::to_wide (@1) == 0)
   (rshift (convert @0)
	   { build_int_cst (integer_type_node,
			    wi::exact_log2 (wi::to_wide (@2))); }))))

/* If ARG1 is a constant, we can convert this to a multiply by the
   reciprocal.  This does not have the same rounding properties,
   so only do this if -freciprocal-math.  We can actually
   always safely do it if ARG1 is a power of two, but it's hard to
   tell if it is or not in a portable manner.  */
(for cst (REAL_CST COMPLEX_CST VECTOR_CST)
 (simplify
  (rdiv @0 cst@1)
  (if (optimize)
   (if (flag_reciprocal_math
	&& !real_zerop (@1))
    (with
     { tree tem = const_binop (RDIV_EXPR, type, build_one_cst (type), @1); }
     (if (tem)
      (mult @0 { tem; } )))
    (if (cst != COMPLEX_CST)
     (with { tree inverse = exact_inverse (type, @1); }
      (if (inverse)
       (mult @0 { inverse; } ))))))))

(for mod (ceil_mod floor_mod round_mod trunc_mod)
 /* 0 % X is always zero.  */
 (simplify
  (mod integer_zerop@0 @1)
  /* But not for 0 % 0 so that we can get the proper warnings and errors.  */
  (if (!integer_zerop (@1))
   @0))
 /* X % 1 is always zero.  */
 (simplify
  (mod @0 integer_onep)
  { build_zero_cst (type); })
 /* X % -1 is zero.  */
 (simplify
  (mod @0 integer_minus_onep@1)
  (if (!TYPE_UNSIGNED (type))
   { build_zero_cst (type); }))
 /* X % X is zero.  */
 (simplify
  (mod @0 @0)
  /* But not for 0 % 0 so that we can get the proper warnings and errors.  */
  (if (!integer_zerop (@0))
   { build_zero_cst (type); }))
 /* (X % Y) % Y is just X % Y.  */
 (simplify
  (mod (mod@2 @0 @1) @1)
  @2)
 /* From extract_muldiv_1: (X * C1) % C2 is zero if C1 is a multiple of C2.  */
 (simplify
  (mod (mult @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (ANY_INTEGRAL_TYPE_P (type)
       && TYPE_OVERFLOW_UNDEFINED (type)
       && wi::multiple_of_p (wi::to_wide (@1), wi::to_wide (@2),
			     TYPE_SIGN (type)))
   { build_zero_cst (type); }))
 /* For (X % C) == 0, if X is signed and C is power of 2, use unsigned
    modulo and comparison, since it is simpler and equivalent.  */
 (for cmp (eq ne)
  (simplify
   (cmp (mod @0 integer_pow2p@2) integer_zerop@1)
   (if (!TYPE_UNSIGNED (TREE_TYPE (@0)))
    (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
     (cmp (mod (convert:utype @0) (convert:utype @2)) (convert:utype @1)))))))

/* (X % C1) % C2 is X % C2 when C2 divides C1.  Truncating remainder keeps
   the sign of X and the magnitude modulo C1, so reducing modulo C2 gives
   the same result as reducing X directly.  */
(simplify
 (trunc_mod (trunc_mod @0 INTEGER_CST@1) INTEGER_CST@2)
 (if (INTEGRAL_TYPE_P (type)
      && ((!TYPE_OVERFLOW_TRAPS (type)
	   && !TYPE_OVERFLOW_SANITIZED (type))
	  || !integer_minus_onep (@1))
      && !integer_zerop (@1)
      && !integer_zerop (@2)
      && wi::multiple_of_p (wi::to_widest (@1), wi::to_widest (@2), SIGNED))
  (if (!TYPE_UNSIGNED (type) && integer_minus_onep (@2))
   { build_zero_cst (type); }
   (trunc_mod @0 @2))))

/* X % -C is the same as X % C.  */
(simplify
 (trunc_mod @0 INTEGER_CST@1)
  (if (TYPE_SIGN (type) == SIGNED
       && !TREE_OVERFLOW (@1)
       && wi::neg_p (wi::to_wide (@1))
       && !TYPE_OVERFLOW_TRAPS (type)
       /* Avoid this transformation if C is INT_MIN, i.e. C == -C.  */
       && !sign_bit_p (@1, @1))
   (trunc_mod @0 (negate @1))))

/* X % -Y is the same as X % Y.  */
(simplify
 (trunc_mod @0 (convert? (negate@2 @1)))
 (if (INTEGRAL_TYPE_P (type)
      && !TYPE_UNSIGNED (type)
      && !TYPE_OVERFLOW_TRAPS (type)
      && tree_nop_conversion_p (type, TREE_TYPE (@1))
      /* Avoid this transformation if X might be INT_MIN or
	 Y might be -1, because we would then change valid
	 INT_MIN % -(-1) into invalid INT_MIN % -1.  */
      && (expr_not_equal_to (@0, wi::to_wide (TYPE_MIN_VALUE (type)))
	  || expr_not_equal_to (@1, wi::minus_one (TYPE_PRECISION
							(TREE_TYPE (@1))),
				gimple_match_ctx (@2))))
  (trunc_mod @0 (convert @1))))

/* X - (X / Y) * Y is the same as X % Y.  */
(simplify
 (minus (convert1? @0) (convert2? (mult:c (trunc_div @@0 @@1) @1)))
 (if (INTEGRAL_TYPE_P (type)
      || (VECTOR_INTEGER_TYPE_P (type)
	  && ((optimize_vectors_before_lowering_p ()
	       && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
	      || target_supports_op_p (type, TRUNC_MOD_EXPR,
				       optab_vector))))
  (convert (trunc_mod @0 @1))))

/* x * (1 + y / x) - y -> x - y % x */
(simplify
 (minus (mult:cs @0 (plus:s (trunc_div:s @1 @0) integer_onep)) @1)
 (if (INTEGRAL_TYPE_P (type))
  (minus @0 (trunc_mod @1 @0))))

/* Optimize TRUNC_MOD_EXPR by a power of two into a BIT_AND_EXPR,
   i.e. "X % C" into "X & (C - 1)", if X and C are positive.
   Also optimize A % (C << N)  where C is a power of 2,
   to A & ((C << N) - 1).
   Also optimize "A shift (B % C)", if C is a power of 2, to
   "A shift (B & (C - 1))".  SHIFT operation include "<<" and ">>"
   and assume (B % C) is nonnegative as shifts negative values would
   be UB.  */
(match (power_of_two_cand @1)
 INTEGER_CST@1)
(match (power_of_two_cand @1)
 (lshift INTEGER_CST@1 @2))
(for mod (trunc_mod floor_mod)
 (for shift (lshift rshift)
  (simplify
   (shift @0 (mod @1 (power_of_two_cand@2 @3)))
   (if (integer_pow2p (@3) && tree_int_cst_sgn (@3) > 0)
    (shift @0 (bit_and @1 (minus @2 { build_int_cst (TREE_TYPE (@2),
						      1); }))))))
 (simplify
  (mod @0 (convert? (power_of_two_cand@1 @2)))
  (if ((TYPE_UNSIGNED (type) || tree_expr_nonnegative_p (@0))
       /* Allow any integral conversions of the divisor, except
	  conversion from narrower signed to wider unsigned type
	  where if @1 would be negative power of two, the divisor
	  would not be a power of two.  */
       && INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@1))
	   || TYPE_UNSIGNED (TREE_TYPE (@1))
	   || !TYPE_UNSIGNED (type))
       && integer_pow2p (@2) && tree_int_cst_sgn (@2) > 0)
   (with { tree utype = TREE_TYPE (@1);
	   if (!TYPE_OVERFLOW_WRAPS (utype))
	     utype = unsigned_type_for (utype); }
    (bit_and @0 (convert (minus (convert:utype @1)
				{ build_one_cst (utype); })))))))

/* Simplify (unsigned t * 2)/2 -> unsigned t & 0x7FFFFFFF.  */
(for div (trunc_div exact_div)
 (simplify
  (div (mult @0 integer_pow2p@1) @1)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) && TYPE_UNSIGNED (TREE_TYPE (@0)))
   (bit_and @0 { wide_int_to_tree
		 (type, wi::mask (TYPE_PRECISION (type)
				  - wi::exact_log2 (wi::to_wide (@1)),
				  false, TYPE_PRECISION (type))); }))))

/* Simplify (unsigned t / 2) * 2 -> unsigned t & ~1.  */
(simplify
 (mult (trunc_div @0 integer_pow2p@1) @1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) && TYPE_UNSIGNED (TREE_TYPE (@0)))
  (bit_and @0 (negate @1))))

/* Canonicalize unsigned t / 4 to t >> 2.  */
#if GIMPLE
(for div (trunc_div floor_div exact_div)
 (simplify
  (div @0 integer_pow2p@1)
  (if (INTEGRAL_TYPE_P (type)
       && (TYPE_UNSIGNED (type) || tree_expr_nonnegative_p (@0)))
   (rshift @0 { build_int_cst (integer_type_node,
			       wi::exact_log2 (wi::to_wide (@1))); }))))
#endif

(for div (trunc_div ceil_div floor_div round_div exact_div)
 /* Simplify (t * u) / u -> t.  */
 (simplify
  (div (mult:c@2 @0 @1) @1)
  (if (ANY_INTEGRAL_TYPE_P (type))
   (if (TYPE_OVERFLOW_UNDEFINED (type) && !TYPE_OVERFLOW_SANITIZED (type))
    @0
#if GIMPLE
    (with {int_range_max vr0, vr1;}
     (if (INTEGRAL_TYPE_P (type)
	  && gimple_match_range_of_expr (vr0, @0, @2)
	  && gimple_match_range_of_expr (vr1, @1, @2)
	  && range_op_handler (MULT_EXPR).overflow_free_p (vr0, vr1))
      @0))
#endif
   )))
#if GIMPLE
 /* Simplify (t * u) / v -> t * (u / v) if u is multiple of v.  */
 (simplify
  (div (mult@3 @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (type)
       && wi::multiple_of_p (wi::to_widest (@1), wi::to_widest (@2), SIGNED))
   (if (TYPE_OVERFLOW_UNDEFINED (type) && !TYPE_OVERFLOW_SANITIZED (type))
    (mult @0 (div! @1 @2))
    (with {int_range_max vr0, vr1;}
     (if (gimple_match_range_of_expr (vr0, @0, @3)
	  && gimple_match_range_of_expr (vr1, @1)
	  && range_op_handler (MULT_EXPR).overflow_free_p (vr0, vr1))
      (mult @0 (div! @1 @2))))
   )))
#endif
 /* Simplify (t * u) / (t * v) -> (u / v) if u is multiple of v.  */
 (simplify
  (div (mult@3 @0 INTEGER_CST@1) (mult @0 INTEGER_CST@2))
  (if (INTEGRAL_TYPE_P (type)
       && wi::multiple_of_p (wi::to_widest (@1), wi::to_widest (@2), SIGNED))
   (if (TYPE_OVERFLOW_UNDEFINED (type) && !TYPE_OVERFLOW_SANITIZED (type))
    (div @1 @2)
#if GIMPLE
    (with {int_range_max vr0, vr1, vr2;}
     (if (gimple_match_range_of_expr (vr0, @0, @3)
	  && gimple_match_range_of_expr (vr1, @1)
	  && gimple_match_range_of_expr (vr2, @2)
	  && range_op_handler (MULT_EXPR).overflow_free_p (vr0, vr1)
	  && range_op_handler (MULT_EXPR).overflow_free_p (vr0, vr2))
      (div @1 @2)))
#endif
   ))))

/* Simplify (t * u) % u -> 0.  The product is an exact multiple of u, so
   the remainder is zero under every rounding convention as long as the
   multiplication does not overflow.  Mirrors (t * u) / u -> t above.  */
(for mod (trunc_mod floor_mod ceil_mod round_mod)
 (simplify
  (mod (mult:c@2 @0 @1) @1)
  (if (ANY_INTEGRAL_TYPE_P (type))
   (if (TYPE_OVERFLOW_UNDEFINED (type) && !TYPE_OVERFLOW_SANITIZED (type))
    { build_zero_cst (type); }
#if GIMPLE
    (with {int_range_max vr0, vr1;}
     (if (INTEGRAL_TYPE_P (type)
	  && gimple_match_range_of_expr (vr0, @0, @2)
	  && gimple_match_range_of_expr (vr1, @1, @2)
	  && range_op_handler (MULT_EXPR).overflow_free_p (vr0, vr1))
      { build_zero_cst (type); }))
#endif
   ))))

#if GIMPLE
(for div (trunc_div exact_div)
 /* Simplify (X + M*N) / N -> X / N + M.  */
 (simplify
  (div (plus:c@4 @0 (mult:c@3 @1 @2)) @2)
  (with {int_range_max vr0, vr1, vr2, vr3, vr4;}
  (if (INTEGRAL_TYPE_P (type)
       && gimple_match_range_of_expr (vr1, @1, @3)
       && gimple_match_range_of_expr (vr2, @2, @3)
       /* "N*M" doesn't overflow.  */
       && range_op_handler (MULT_EXPR).overflow_free_p (vr1, vr2)
       && gimple_match_range_of_expr (vr0, @0, @4)
       && gimple_match_range_of_expr (vr3, @3, @4)
       /* "X+(N*M)" doesn't overflow.  */
       && range_op_handler (PLUS_EXPR).overflow_free_p (vr0, vr3)
       && gimple_match_range_of_expr (vr4, @4)
       /* "X+N*M" is not with opposite sign as "X".  */
       && (TYPE_UNSIGNED (type)
	   || (vr0.nonnegative_p () && vr4.nonnegative_p ())
	   || (vr0.nonpositive_p () && vr4.nonpositive_p ()
	       /* X = INT_MIN and N = -1 causes overflow.  */
	       && (!vr0.contains_p (wi::min_value (TYPE_PRECISION (type), SIGNED))
	           || !vr2.contains_p (wi::minus_one (TYPE_PRECISION (type)))))))
  (plus (div @0 @2) @1))))

 /* Simplify (X - M*N) / N -> X / N - M.  */
 (simplify
  (div (minus@4 @0 (mult:c@3 @1 @2)) @2)
  (with {int_range_max vr0, vr1, vr2, vr3, vr4;}
  (if (INTEGRAL_TYPE_P (type)
       && gimple_match_range_of_expr (vr1, @1, @3)
       && gimple_match_range_of_expr (vr2, @2, @3)
       /* "N * M" doesn't overflow.  */
       && range_op_handler (MULT_EXPR).overflow_free_p (vr1, vr2)
       && gimple_match_range_of_expr (vr0, @0, @4)
       && gimple_match_range_of_expr (vr3, @3, @4)
       /* "X - (N*M)" doesn't overflow.  */
       && range_op_handler (MINUS_EXPR).overflow_free_p (vr0, vr3)
       && gimple_match_range_of_expr (vr4, @4)
       /* "X-N*M" is not with opposite sign as "X".  */
       && (TYPE_UNSIGNED (type)
	   || (vr0.nonnegative_p () && vr4.nonnegative_p ())
	   || (vr0.nonpositive_p () && vr4.nonpositive_p ()
	       /* X = INT_MIN and N = -1 causes overflow.  */
	       && (!vr0.contains_p (wi::min_value (TYPE_PRECISION (type), SIGNED))
	           || !vr2.contains_p (wi::minus_one (TYPE_PRECISION (type)))))))
  (minus (div @0 @2) @1)))))

/* Simplify
   (X + C) / N -> X / N + C / N where C is multiple of N.
   (X + C) >> N -> X >> N + C>>N if low N bits of C is 0.  */
(for op (trunc_div exact_div rshift)
 (simplify
  (op (plus@3 @0 INTEGER_CST@1) INTEGER_CST@2)
   (with
    {
      wide_int c = wi::to_wide (@1);
      wide_int n = wi::to_wide (@2);
      bool shift = op == RSHIFT_EXPR;
#define plus_op1(v) (shift ? wi::rshift (v, n, TYPE_SIGN (type)) \
			   : wi::div_trunc (v, n, TYPE_SIGN (type)))
#define exact_mod(v) (shift ? wi::ctz (v) >= n.to_shwi () \
			    : wi::multiple_of_p (v, n, TYPE_SIGN (type)))
      int_range_max vr0, vr1, vr3;
    }
    (if (INTEGRAL_TYPE_P (type)
	 && gimple_match_range_of_expr (vr0, @0, @3))
     (if (exact_mod (c)
	  && gimple_match_range_of_expr (vr1, @1)
	  /* "X+C" doesn't overflow.  */
	  && range_op_handler (PLUS_EXPR).overflow_free_p (vr0, vr1)
	  && gimple_match_range_of_expr (vr3, @3)
	  && !vr3.undefined_p ()
	  /* "X+C" and "X" are not of opposite sign.  */
	  && (TYPE_UNSIGNED (type)
	      || (vr0.nonnegative_p () && vr3.nonnegative_p ())
	      || (vr0.nonpositive_p () && vr3.nonpositive_p ())))
       (plus (op @0 @2) { wide_int_to_tree (type, plus_op1 (c)); })
       (if (!vr0.undefined_p () && TYPE_UNSIGNED (type) && c.sign_mask () < 0
	    && exact_mod (-c)
	    /* unsigned "X-(-C)" doesn't underflow.  */
	    && wi::geu_p (vr0.lower_bound (), -c))
	 (plus (op @0 @2) { wide_int_to_tree (type, -plus_op1 (-c)); })))))))
#undef plus_op1
#undef exact_mod
#endif

/* (nop_outer_cast)-(inner_cast)var -> -(outer_cast)(var)
   If var is smaller in precision this is always safe for both doing
   the negative in signed or unsigned as the value for undefined will not
   show up.  Else it is safe if the negation is done in an unsigned type.
   Note the outer cast cannot be a boolean type as the only valid values
   are 0,-1/1 (depending on the signedness of the boolean) and the negative
   is there to get the correct value.
   Also handle the case where we know that we won't cause a signed
   integer overflow at the point of the negation.  */
(simplify
 (convert (negate:s@1 (convert:s@ic @0)))
 (if (INTEGRAL_TYPE_P (type)
      && tree_nop_conversion_p (type, TREE_TYPE (@1))
      && (TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (@0))
	  || TYPE_OVERFLOW_WRAPS (type)
	  || expr_not_equal_to (@ic, wi::min_value (signed_type_for (type)),
				gimple_match_ctx (@1)))
      && TREE_CODE (type) != BOOLEAN_TYPE)
    (negate (convert @0))))

(for op (negate abs)
 /* Simplify cos(-x) and cos(|x|) -> cos(x).  Similarly for cosh.  */
 (for coss (COS COSH)
  (simplify
   (coss (op @0))
    (coss @0)))
 /* Simplify pow(-x, y) and pow(|x|,y) -> pow(x,y) if y is an even integer.  */
 (for pows (POW)
  (simplify
   (pows (op @0) REAL_CST@1)
   (with { HOST_WIDE_INT n; }
    (if (real_isinteger (&TREE_REAL_CST (@1), &n) && (n & 1) == 0)
     (pows @0 @1)))))
 /* Likewise for powi.  */
 (for pows (POWI)
  (simplify
   (pows (op @0) INTEGER_CST@1)
   (if ((wi::to_wide (@1) & 1) == 0)
    (pows @0 @1))))
 /* Strip negate and abs from both operands of hypot.  */
 (for hypots (HYPOT)
  (simplify
   (hypots (op @0) @1)
   (hypots @0 @1))
  (simplify
   (hypots @0 (op @1))
   (hypots @0 @1)))
 /* copysign(-x, y) and copysign(abs(x), y) -> copysign(x, y).  */
 (for copysigns (COPYSIGN_ALL)
  (simplify
   (copysigns (op @0) @1)
   (copysigns @0 @1))))

/* abs(x)*abs(x) -> x*x.  Should be valid for all types.  */
(simplify
 (mult (abs@1 @0) @1)
 (mult @0 @0))

/* Convert absu(x)*absu(x) -> x*x.  */
(simplify
 (mult (absu@1 @0) @1)
 (mult (convert@2 @0) @2))

/* abs(x) * abs(y) -> abs(x * y).  */
(simplify
 (mult (abs:s @0) (abs:s @1))
 (if ((FLOAT_TYPE_P (type)
       && !HONOR_SIGN_DEPENDENT_ROUNDING (type))
      || (ANY_INTEGRAL_TYPE_P (type)
	  && TYPE_OVERFLOW_UNDEFINED (type)
	  && !TYPE_OVERFLOW_SANITIZED (type)))
  (abs (mult @0 @1))))

#if GIMPLE
/* Simplify SAD(x, x, acc) -> acc since the absolute difference is zero.  */
(simplify
 (sad @0 @0 @1)
 @1)
#endif

/* cos(copysign(x, y)) -> cos(x).  Similarly for cosh.  */
(for coss (COS COSH)
 (for copysigns (COPYSIGN)
  (simplify
   (coss (copysigns @0 @1))
    (coss @0))))

/* pow(copysign(x, y), z) -> pow(x, z) if z is an even integer.  */
(for pows (POW)
 (for copysigns (COPYSIGN)
  (simplify
   (pows (copysigns @0 @2) REAL_CST@1)
   (with { HOST_WIDE_INT n; }
    (if (real_isinteger (&TREE_REAL_CST (@1), &n) && (n & 1) == 0)
     (pows @0 @1))))))
/* Likewise for powi.  */
(for pows (POWI)
 (for copysigns (COPYSIGN)
  (simplify
   (pows (copysigns @0 @2) INTEGER_CST@1)
   (if ((wi::to_wide (@1) & 1) == 0)
    (pows @0 @1)))))

(for hypots (HYPOT)
 (for copysigns (COPYSIGN)
  /* hypot(copysign(x, y), z) -> hypot(x, z).  */
  (simplify
   (hypots (copysigns @0 @1) @2)
   (hypots @0 @2))
  /* hypot(x, copysign(y, z)) -> hypot(x, y).  */
  (simplify
   (hypots @0 (copysigns @1 @2))
   (hypots @0 @1))))

/* copysign(x, CST) -> abs (x).  If the target does not
   support the copysign optab then canonicalize
   copysign(x, -CST) -> fneg (abs (x)).   */
(for copysigns (COPYSIGN_ALL)
 (simplify
  (copysigns @0 REAL_CST@1)
  (if (!REAL_VALUE_NEGATIVE (TREE_REAL_CST (@1)))
   (abs @0)
#if GIMPLE
   (if (!direct_internal_fn_supported_p (IFN_COPYSIGN, type,
					 OPTIMIZE_FOR_BOTH))
    (negate (abs @0)))
#endif
   )))

#if GIMPLE
/* Transform fneg (fabs (X)) -> copysign (X, -1) as the canonical
   representation if the target supports the copysign optab.  */
(simplify
 (negate (abs @0))
 (if (direct_internal_fn_supported_p (IFN_COPYSIGN, type,
				      OPTIMIZE_FOR_BOTH))
   (IFN_COPYSIGN @0 { build_minus_one_cst (type); })))
#endif
/* copysign(copysign(x, y), z) -> copysign(x, z).  */
(for copysigns (COPYSIGN_ALL)
 (simplify
  (copysigns (copysigns @0 @1) @2)
  (copysigns @0 @2)))

/* copysign(x,y)*copysign(x,y) -> x*x.  */
(for copysigns (COPYSIGN_ALL)
 (simplify
  (mult (copysigns@2 @0 @1) @2)
  (mult @0 @0)))

/* ccos(-x) -> ccos(x).  Similarly for ccosh.  */
(for ccoss (CCOS CCOSH)
 (simplify
  (ccoss (negate @0))
   (ccoss @0)))

/* cabs(-x) and cos(conj(x)) -> cabs(x).  */
(for ops (conj negate)
 (for cabss (CABS)
  (simplify
   (cabss (ops @0))
   (cabss @0))))

/* Fold (a * (1 << b)) into (a << b)  */
(simplify
 (mult:c @0 (convert? (lshift integer_onep@1 @2)))
  (if (! FLOAT_TYPE_P (type)
       && tree_nop_conversion_p (type, TREE_TYPE (@1)))
   (lshift @0 @2)))

/* Fold a * !a into 0.  */
(simplify
 (mult:c @0 (convert? (eq @0 integer_zerop@2)))
  @2)
(simplify
 (mult:c @0 (vec_cond (eq @0 integer_zerop) @1 integer_zerop@2))
  @2)
(simplify
 (mult:c @0 (vec_cond (ne @0 integer_zerop) integer_zerop@2 @1))
  @2)

(for shift (lshift rshift)
 /* Shifts by precision or greater result in zero.  */
 (simplify
  (shift @0 uniform_integer_cst_p@1)
  (if ((GIMPLE || !sanitize_flags_p (SANITIZE_SHIFT_EXPONENT))
       /* Leave arithmetic right shifts of possibly negative values alone.  */
       && (TYPE_UNSIGNED (type)
           || shift == LSHIFT_EXPR
	   || tree_expr_nonnegative_p (@0))
       /* Use a signed compare to leave negative shift counts alone.  */
       && wi::ges_p (wi::to_wide (uniform_integer_cst_p (@1)),
		     element_precision (type)))
   { build_zero_cst (type); }))
 /* Shifts by constants distribute over several binary operations:
    (X (<<, >>) C) & (Y (<<, >>) C) -> (X & Y) (<<, >>) C.
    (X (<<, >>) C) | (Y (<<, >>) C) -> (X | Y) (<<, >>) C.
    (X (<<, >>) C) ^ (Y (<<, >>) C) -> (X ^ Y) (<<, >>) C.  */
 (for op (bit_and bit_ior bit_xor)
  (simplify
   (op (shift:s @0 @1) (shift:s @2 @1))
   (if (INTEGRAL_TYPE_P (type))
    (shift (op @0 @2) @1)))))

/* (X << C) (+,-) (Y << C) -> (X (+,-) Y) << C.  */
(for op (plus minus)
  (simplify
    (op (lshift:s @0 @1) (lshift:s @2 @1))
    (if (INTEGRAL_TYPE_P (type)
	 && TYPE_OVERFLOW_WRAPS (type)
	 && !TYPE_SATURATING (type))
      (lshift (op @0 @2) @1))))

/* (y << x) == x -> false and (y << x) != x -> true when y != 0.  */
(for cmp (eq ne)
 (simplify
  (cmp:c (nop_convert1? (lshift @0 @1)) (convert2? @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && tree_expr_nonzero_p (@0))
      { constant_boolean_node (cmp != EQ_EXPR, type); })))

#if GIMPLE
/* (y << x) {<,<=} x -> false and (y << x) {>,>=} x -> true when y != 0
   and (y << x) >> x == y and for signed comparison (y << x) >= 0.  */
(for cmp (gt ge lt le)
 (simplify
  (cmp:c (nop_convert1?@3 (lshift@2 @0 @1)) (convert2? @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (with { bool ok = false;
	   int_range_max vr0, vr1;
	   if (gimple_match_range_of_expr (vr0, @0, @2)
	       && !vr0.varying_p ()
	       && gimple_match_range_of_expr (vr1, @1, @2)
	       && !vr1.varying_p ()
	       && !vr0.contains_p (wi::zero (TYPE_PRECISION (TREE_TYPE (@0)))))
	     {
	       unsigned lz = wi::clz (vr0.get_nonzero_bits ());
	       if (!wi::neg_p (vr1.upper_bound (), TYPE_SIGN (TREE_TYPE (@1)))
		   && wi::ltu_p (vr1.upper_bound (),
				 wi::uhwi (lz + TYPE_UNSIGNED (TREE_TYPE (@3)),
					   TYPE_PRECISION (TREE_TYPE (@1)))))
		 ok = true;
	     } }
    (if (ok)
     { constant_boolean_node (cmp == GT_EXPR || cmp == GE_EXPR, type); })))))
#endif

/* Fold (1 << (C - x)) where C = precision(type) - 1
   into ((1 << C) >> x). */
(simplify
 (lshift integer_onep@0 (minus@1 INTEGER_CST@2 @3))
  (if (INTEGRAL_TYPE_P (type)
       && wi::eq_p (wi::to_wide (@2), TYPE_PRECISION (type) - 1)
       && single_use (@1))
   (if (TYPE_UNSIGNED (type))
     (rshift (lshift @0 @2) @3)
   (with
    { tree utype = unsigned_type_for (type); }
    (convert (rshift (lshift (convert:utype @0) @2) @3))))))

/* Fold (((X >> C1) & C2) << C3) into (X >>/<< |C1 - C3|) & (C2' << C3),
   where C2' is obtained from C2 by clearing all but the rightmost C1 bits.  */
(simplify
 (lshift (bit_and (rshift @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
  (if (TYPE_UNSIGNED (TREE_TYPE (@0))
       && type_has_mode_precision_p (TREE_TYPE (@0)))
   (with { int prec = TYPE_PRECISION (TREE_TYPE (@0)); }
    (if (wi::ltu_p (wi::to_wide (@1), prec)
	 && wi::ltu_p (wi::to_wide (@3), prec))
     (with
      {
	/* Clear the first PREC - @1 bits of the mask, then shift it.  */
	wide_int mask_lz = wi::mask (prec - tree_to_uhwi (@1), false, prec);
	tree mask = fold_build2 (BIT_AND_EXPR, TREE_TYPE (@0),
			@2, wide_int_to_tree (TREE_TYPE (@0), mask_lz));
	tree mask_shifted = fold_build2 (LSHIFT_EXPR, TREE_TYPE (@0),
					 mask, @3);
      }
      (if (wi::to_widest (@1) <= wi::to_widest (@3))
       (bit_and (lshift @0 (minus @3 @1)) { mask_shifted; })
       (bit_and (rshift @0 (minus @1 @3)) { mask_shifted; })))))))

/* Fold ((type)(a<0)) << SIGNBITOFA into ((type)a) & signbit. */
(simplify
 (lshift (convert (lt @0 integer_zerop@1)) INTEGER_CST@2)
 (if (TYPE_SIGN (TREE_TYPE (@0)) == SIGNED
      && wi::eq_p (wi::to_wide (@2), TYPE_PRECISION (TREE_TYPE (@0)) - 1))
  (with { wide_int wone = wi::one (TYPE_PRECISION (type)); }
   (bit_and (convert @0)
            { wide_int_to_tree (type,
				wi::lshift (wone, wi::to_wide (@2))); }))))

/* Fold (-x >> C) into -(x > 0) where C = precision(type) - 1.  */
(for cst (INTEGER_CST VECTOR_CST)
 (simplify
  (rshift (negate:s @0) cst@1)
   (if (!TYPE_UNSIGNED (type)
        && TYPE_OVERFLOW_UNDEFINED (type))
    (with { tree stype = TREE_TYPE (@1);
	    tree bt = truth_type_for (type);
	    tree zeros = build_zero_cst (type);
	    tree cst = NULL_TREE; }
     (switch
      /* Handle scalar case.  */
      (if (INTEGRAL_TYPE_P (type)
	   /* If we apply the rule to the scalar type before vectorization
	      we will enforce the result of the comparison being a bool
	      which will require an extra AND on the result that will be
	      indistinguishable from when the user did actually want 0
	      or 1 as the result so it can't be removed.  */
	   && canonicalize_math_after_vectorization_p ()
	   && wi::eq_p (wi::to_wide (@1), TYPE_PRECISION (type) - 1))
       (negate (convert (gt @0 { zeros; }))))
      /* Handle vector case.  */
      (if (VECTOR_INTEGER_TYPE_P (type)
	   /* First check whether the target has the same mode for vector
	      comparison results as it's operands do.  */
	   && TYPE_MODE (bt) == TYPE_MODE (type)
	   /* Then check to see if the target is able to expand the comparison
	      with the given type later on, otherwise we may ICE.  */
	   && expand_vec_cmp_expr_p (type, bt, GT_EXPR)
	   && (cst = uniform_integer_cst_p (@1)) != NULL
	   && wi::eq_p (wi::to_wide (cst), element_precision (type) - 1))
       (view_convert (gt:bt @0 { zeros; }))))))))

/* Fold (C1/X)*C2 into (C1*C2)/X.  */
(simplify
 (mult (rdiv@3 REAL_CST@0 @1) REAL_CST@2)
  (if (flag_associative_math
       && single_use (@3))
   (with
    { tree tem = const_binop (MULT_EXPR, type, @0, @2); }
    (if (tem)
     (rdiv { tem; } @1)))))

/* Simplify ~X & X as zero.  */
(simplify
 (bit_and:c (convert? @0) (convert? (maybe_bit_not @1)))
 (with { bool wascmp; }
  (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1))
       && bitwise_inverted_equal_p (@0, @1, wascmp))
   { wascmp ? constant_boolean_node (false, type) : build_zero_cst (type); })))

/* zero_extend (X) & sign_extend (X) -> zero_extend (X) and
   zero_extend (X) | sign_extend (X) -> sign_extend (X)  */
(for bitop (bit_and bit_ior)
 (simplify
  (bitop:c (convert @0) (convert (nop_convert@1 @0)))
  (if (INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_UNSIGNED (TREE_TYPE (@0)) != TYPE_UNSIGNED (TREE_TYPE (@1))
       && TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (TREE_TYPE (@1))
       && TYPE_PRECISION (TREE_TYPE (@0)) <= TYPE_PRECISION (type))
   (if ((bitop == BIT_AND_EXPR) == TYPE_UNSIGNED (TREE_TYPE (@0)))
    (convert @0)
    (convert @1)))))

/* PR71636: Transform x & ((1U << b) - 1) -> x & ~(~0U << b);  */
(simplify
  (bit_and:c @0 (plus:s (lshift:s integer_onep @1) integer_minus_onep))
  (if (TYPE_UNSIGNED (type))
    (bit_and @0 (bit_not (lshift { build_all_ones_cst (type); } @1)))))

/* PR112533: Canonicalize boolean comparisons of masked pow2 bits into
   xor-mask tests.
   ((x & c) == 0) == ((y & c) == 0) -> (((x ^ y) & c) == 0)
   ((x & c) == 0) != ((y & c) == 0) -> (((x ^ y) & c) != 0)
   and equivalent xor forms.  */
(for masked_ocmp (eq ne bit_xor)
 (for icmp0 (eq eq ne ne)
      icmp1 (eq ne eq ne)
  (simplify
   (masked_ocmp
    (icmp0:s (bit_and:s @0 integer_pow2p@1) integer_zerop@3)
    (icmp1:s (bit_and:s @2 @1) @3))
   (if (types_match (@0, @2))
    (with {
      bool innersame = icmp0 == icmp1;
    }
     (if (innersame == (masked_ocmp == EQ_EXPR))
      (eq (bit_and (bit_xor @0 @2) @1) @3)
      (ne (bit_and (bit_xor @0 @2) @1) @3)))))))

(for bitop (bit_and bit_ior)
     cmp (eq ne)
 /* PR35691: Transform
    (x == 0 & y == 0) -> (x | typeof(x)(y)) == 0.
    (x != 0 | y != 0) -> (x | typeof(x)(y)) != 0.  */
 (simplify
  (bitop (cmp@3 @0 integer_zerop@2) (cmp@4 @1 integer_zerop))
  (switch
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& INTEGRAL_TYPE_P (TREE_TYPE (@1))
	&& TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (TREE_TYPE (@1)))
    (cmp (bit_ior @0 (convert @1)) @2))
   (if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (@0))
	&& VECTOR_INTEGER_TYPE_P (TREE_TYPE (@1))
	&& vector_nop_conversion_p (TREE_TYPE (@0), TREE_TYPE (@1))
	&& single_use (@3) && single_use (@4))
    (cmp (bit_ior @0 (view_convert @1)) @2))))
 /* Transform:
    (x == -1 & y == -1) -> (x & typeof(x)(y)) == -1.
    (x != -1 | y != -1) -> (x & typeof(x)(y)) != -1.  */
 (simplify
  (bitop (cmp@3 @0 integer_all_onesp@2)
	 (cmp@4 @1 integer_all_onesp))
  (switch
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& INTEGRAL_TYPE_P (TREE_TYPE (@1))
	&& TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (TREE_TYPE (@1)))
    (cmp (bit_and @0 (convert @1)) @2))
   (if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (@0))
	&& VECTOR_INTEGER_TYPE_P (TREE_TYPE (@1))
	&& vector_nop_conversion_p (TREE_TYPE (@0), TREE_TYPE (@1))
	&& single_use (@3) && single_use (@4))
    (cmp (bit_and @0 (view_convert @1)) @2)))))

/* Combine two single-bit tests of the same value X into one masked
   compare.  For distinct single bits C1, C2 and M = C1 | C2:
     (X & C1) != 0 & (X & C2) != 0  ->  (X & M) == M
     (X & C1) == 0 | (X & C2) == 0  ->  (X & M) != M
     (X & C1) != 0 & (X & C2) == 0  ->  (X & M) == C1
     (X & C1) != 0 | (X & C2) == 0  ->  (X & M) != C2
   The == 0 & == 0 and != 0 | != 0 cases are handled by the more general
   rule above.  Restricted to single-bit masks: for a multi-bit mask
   (X & C) != 0 is not a single-bit test.  */
(for bitop (bit_and bit_ior)
     cmp (ne eq)
     rcmp (eq ne)
 (simplify
  (bitop (cmp:s (bit_and:s @0 INTEGER_CST@1) integer_zerop)
	 (cmp:s (bit_and:s @0 INTEGER_CST@2) integer_zerop))
  (if (integer_pow2p (@1) && integer_pow2p (@2)
       && wi::to_wide (@1) != wi::to_wide (@2))
   (with { tree m = wide_int_to_tree (TREE_TYPE (@0),
				      wi::to_wide (@1) | wi::to_wide (@2)); }
    (rcmp (bit_and @0 { m; }) { m; }))))
 (simplify
  (bitop:c (ne:s (bit_and:s @0 INTEGER_CST@1) integer_zerop)
	   (eq:s (bit_and:s @0 INTEGER_CST@2) integer_zerop))
  (if (integer_pow2p (@1) && integer_pow2p (@2)
       && wi::to_wide (@1) != wi::to_wide (@2))
   (with { tree m = wide_int_to_tree (TREE_TYPE (@0),
				      wi::to_wide (@1) | wi::to_wide (@2)); }
    (rcmp (bit_and @0 { m; })
	  { bitop == BIT_AND_EXPR ? @1 : @2; })))))

/* Fold (A & ~B) - (A & B) into (A ^ B) - B.  */
(simplify
 (minus (bit_and:cs @0 (bit_not @1)) (bit_and:cs @0 @1))
  (minus (bit_xor @0 @1) @1))
(simplify
 (minus (bit_and:s @0 INTEGER_CST@2) (bit_and:s @0 INTEGER_CST@1))
 (if (~wi::to_wide (@2) == wi::to_wide (@1))
  (minus (bit_xor @0 @1) @1)))

/* Fold (A & B) - (A & ~B) into B - (A ^ B).  */
(simplify
 (minus (bit_and:cs @0 @1) (bit_and:cs @0 (bit_not @1)))
  (minus @1 (bit_xor @0 @1)))

/* Simplify (X & ~Y) |^+ (~X & Y) -> X ^ Y.  */
(for op (bit_ior bit_xor plus)
 (simplify
  (op (bit_and:c @0 @2) (bit_and:c @3 @1))
  (with { bool wascmp0, wascmp1; }
   (if (bitwise_inverted_equal_p (@2, @1, wascmp0)
        && bitwise_inverted_equal_p (@0, @3, wascmp1)
	&& ((!wascmp0 && !wascmp1)
	    || element_precision (type) == 1))
   (bit_xor @0 @1)))))

/* PR53979: Transform ((a ^ b) | a) -> (a | b) */
(simplify
  (bit_ior:c (bit_xor:c @0 @1) @0)
  (bit_ior @0 @1))

/* (a & ~b) | (a ^ b)  -->  a ^ b  */
(simplify
 (bit_ior:c (bit_and:c @0 (bit_not @1)) (bit_xor:c@2 @0 @1))
 @2)

/* (a & ~b) ^ ~a  -->  ~(a & b)  */
(simplify
 (bit_xor:c (bit_and:cs @0 (bit_not @1)) (bit_not @0))
 (bit_not (bit_and @0 @1)))

/* (~a & b) ^ a  -->   (a | b)   */
(simplify
 (bit_xor:c (bit_and:cs (bit_not @0) @1) @0)
 (bit_ior @0 @1))

/* (a | b) & ~(a ^ b)  -->  a & b  */
(simplify
 (bit_and:c (bit_ior @0 @1) (bit_not (bit_xor:c @0 @1)))
 (bit_and @0 @1))

/* (a | b) & (a == b)  -->  a & b (boolean version of the above).  */
(simplify
 (bit_and:c (bit_ior @0 @1) (nop_convert? (eq:c @0 @1)))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  (bit_and @0 @1)))

/* a | ~(a ^ b)  -->  a | ~b  */
(simplify
 (bit_ior:c @0 (bit_not:s (bit_xor:c @0 @1)))
 (bit_ior @0 (bit_not @1)))

/* a | (a == b)  -->  a | (b^1) (boolean version of the above). */
(simplify
 (bit_ior:c @0 (nop_convert? (eq:c @0 @1)))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  (bit_ior @0 (bit_xor @1 { build_one_cst (type); }))))

/* a | ((~a) ^ b)  -->  a | (~b) (alt version of the above 2) */
(simplify
 (bit_ior:c @0 (bit_xor:cs @1 @2))
 (with { bool wascmp; }
 (if (bitwise_inverted_equal_p (@0, @1, wascmp)
      && (!wascmp || element_precision (type) == 1))
  (bit_ior @0 (bit_not @2)))))

/* a & ~(a ^ b)  -->  a & b  */
(simplify
 (bit_and:c @0 (bit_not (bit_xor:c @0 @1)))
 (bit_and @0 @1))

/* Transform:
   (a - 1) & -a ->  0.
   (a - 1) | -a -> -1.
   (a - 1) ^ -a -> -1.  */
(for bit_op (bit_ior bit_xor bit_and)
 (simplify
  (bit_op:c (plus @0 integer_minus_onep@1) (negate @0))
  (if (bit_op == BIT_AND_EXPR)
    { build_zero_cst (type); }
    @1))
 (simplify
  (bit_op:c (minus @0 integer_onep) (negate @0))
  (if (bit_op == BIT_AND_EXPR)
    { build_zero_cst (type); }
    { build_minus_one_cst (type); })))

/* a & (a == b)  -->  a & b (boolean version of the above). */
(simplify
 (bit_and:c @0 (nop_convert? (eq:c @0 @1)))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  (bit_and @0 @1)))

/* a & ((~a) ^ b)  -->  a & b (alt version of the above 2) */
(simplify
 (bit_and:c @0 (bit_xor:c @1 @2))
 (with { bool wascmp; }
 (if (bitwise_inverted_equal_p (@0, @1, wascmp)
      && (!wascmp || element_precision (type) == 1))
  (bit_and @0 @2))))

/* (a | b) | (a &^ b)  -->  a | b  */
(for op (bit_and bit_xor)
 (simplify
  (bit_ior:c (bit_ior@2 @0 @1) (op:c @0 @1))
  @2))

/* (a & b) | ~(a ^ b)  -->  ~(a ^ b)  */
(simplify
 (bit_ior:c (bit_and:c @0 @1) (bit_not@2 (bit_xor @0 @1)))
 @2)

/* (a & b) | (a == b)  -->  a == b  */
(simplify
 (bit_ior:c (bit_and:c @0 @1) (nop_convert?@2 (eq @0 @1)))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  @2))

/* ((~a) & b) ^ (a | b) --> a  */
/* ((~a) | b) ^ (a & b) --> ~a  */
(simplify
 (bit_xor:c (bit_ior:c @0 @1) (bit_and:c @2 @1))
 (with { bool wascmp; }
  (if (bitwise_inverted_equal_p (@0, @2, wascmp)
       && (!wascmp || element_precision (type) == 1))
   @0)))

/* ~(~a & b) --> a | ~b
   ~(~a | b) --> a & ~b  */
(for iop (bit_and bit_ior)
     res (bit_ior bit_and)
 (simplify
  (bit_not (iop:cs (bit_not @0) @1))
  (res @0 (bit_not @1))))

/* (a ^ b) & ((b ^ c) ^ a) --> (a ^ b) & ~c */
(simplify
 (bit_and:c (bit_xor:c@3 @0 @1) (bit_xor:cs (bit_xor:cs @1 @2) @0))
 (bit_and @3 (bit_not @2)))

/* (a ^ b) | ((b ^ c) ^ a) --> (a ^ b) | c */
(simplify
 (bit_ior:c (bit_xor:c@3 @0 @1) (bit_xor:c (bit_xor:c @1 @2) @0))
 (bit_ior @3 @2))

/* (~X | C) ^ D -> (X | C) ^ (~D ^ C) if (~D ^ C) can be simplified.  */
(simplify
 (bit_xor:c (bit_ior:cs (bit_not:s @0) @1) @2)
  (bit_xor (bit_ior @0 @1) (bit_xor! (bit_not! @2) @1)))

/* (~X & C) ^ D -> (X & C) ^ (D ^ C) if (D ^ C) can be simplified.  */
(simplify
 (bit_xor:c (bit_and:cs (bit_not:s @0) @1) @2)
  (bit_xor (bit_and @0 @1) (bit_xor! @2 @1)))

/* Simplify (~X & Y) to X ^ Y if we know that (X & ~Y) is 0.  */
(simplify
 (bit_and (bit_not SSA_NAME@0) INTEGER_CST@1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && wi::bit_and_not (get_nonzero_bits (@0), wi::to_wide (@1)) == 0)
  (bit_xor @0 @1)))

/* For constants M and N, if M == (1LL << cst) - 1 && (N & M) == M,
   ((A & N) + B) & M -> (A + B) & M
   Similarly if (N & M) == 0,
   ((A | N) + B) & M -> (A + B) & M
   and for - instead of + (or unary - instead of +)
   and/or ^ instead of |.
   If B is constant and (B & M) == 0, fold into A & M.  */
(for op (plus minus)
 (for bitop (bit_and bit_ior bit_xor)
  (simplify
   (bit_and (op:s (bitop:s@0 @3 INTEGER_CST@4) @1) INTEGER_CST@2)
    (with
     { tree pmop[2];
       tree utype = fold_bit_and_mask (TREE_TYPE (@0), @2, op, @0, bitop,
				       @3, @4, @1, ERROR_MARK, NULL_TREE,
				       NULL_TREE, pmop); }
     (if (utype)
      (convert (bit_and (op (convert:utype { pmop[0]; })
			    (convert:utype { pmop[1]; }))
			(convert:utype @2))))))
  (simplify
   (bit_and (op:s @0 (bitop:s@1 @3 INTEGER_CST@4)) INTEGER_CST@2)
    (with
     { tree pmop[2];
       tree utype = fold_bit_and_mask (TREE_TYPE (@0), @2, op, @0, ERROR_MARK,
				       NULL_TREE, NULL_TREE, @1, bitop, @3,
				       @4, pmop); }
     (if (utype)
      (convert (bit_and (op (convert:utype { pmop[0]; })
			    (convert:utype { pmop[1]; }))
			(convert:utype @2)))))))
 (simplify
  (bit_and (op:s @0 @1) INTEGER_CST@2)
   (with
    { tree pmop[2];
      tree utype = fold_bit_and_mask (TREE_TYPE (@0), @2, op, @0, ERROR_MARK,
				      NULL_TREE, NULL_TREE, @1, ERROR_MARK,
				      NULL_TREE, NULL_TREE, pmop); }
    (if (utype)
     (convert (bit_and (op (convert:utype { pmop[0]; })
			   (convert:utype { pmop[1]; }))
		       (convert:utype @2)))))))
(for bitop (bit_and bit_ior bit_xor)
 (simplify
  (bit_and (negate:s (bitop:s@0 @2 INTEGER_CST@3)) INTEGER_CST@1)
   (with
    { tree pmop[2];
      tree utype = fold_bit_and_mask (TREE_TYPE (@0), @1, NEGATE_EXPR, @0,
				      bitop, @2, @3, NULL_TREE, ERROR_MARK,
				      NULL_TREE, NULL_TREE, pmop); }
    (if (utype)
     (convert (bit_and (negate (convert:utype { pmop[0]; }))
		       (convert:utype @1)))))))

/* (signed)A >= 0 && (A & INT_MAX) != 0  -->  (signed)A > 0
   (signed)A <  0 || (A & INT_MAX) == 0  -->  (signed)A <= 0  */
(for op      (bit_and  truth_and  bit_ior  truth_or)
     signop  (ge       ge         lt       lt)
     maskop  (ne       ne         eq       eq)
     resop   (gt       gt         le       le)
 (simplify
  (op:c
   (signop (nop_convert?@2 @0) integer_zerop)
   (maskop (bit_and @0 INTEGER_CST@3) integer_zerop))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@2))
       && wi::eq_p (wi::to_wide (@3),
		    wi::max_value (TYPE_PRECISION (TREE_TYPE (@0)), SIGNED)))
   (resop @2 { build_zero_cst (TREE_TYPE (@2)); }))))

/* X % Y < Y -> Y >= 0
   X % Y >= Y -> Y < 0  */
(for ltge (lt ge)
     gelt (ge lt)
 (simplify
  (ltge:c (trunc_mod @0 @1) @1)
  (gelt @1 { build_zero_cst (TREE_TYPE (@1)); })))

/* x | ~0 -> ~0  */
(simplify
 (bit_ior @0 integer_all_onesp@1)
 @1)

/* x | 0 -> x  */
(simplify
 (bit_ior @0 integer_zerop)
 @0)

/* x & 0 -> 0  */
(simplify
 (bit_and @0 integer_zerop@1)
 @1)

/* ~x | x -> -1 */
/* ~x ^ x -> -1 */
(for op (bit_ior bit_xor)
 (simplify
  (op:c (convert? @0) (convert? (maybe_bit_not @1)))
  (with { bool wascmp; }
   (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1))
        && bitwise_inverted_equal_p (@0, @1, wascmp))
    (convert
     { wascmp
        ? constant_boolean_node (true, type)
	: build_all_ones_cst (TREE_TYPE (@0)); })))))

/* x ^ x -> 0 */
(simplify
  (bit_xor @0 @0)
  { build_zero_cst (type); })

/* Canonicalize X ^ ~0 to ~X.  */
(simplify
  (bit_xor @0 integer_all_onesp@1)
  (bit_not @0))

/* x & ~0 -> x  */
(simplify
 (bit_and @0 integer_all_onesp)
  (non_lvalue @0))

/* x & x -> x,  x | x -> x  */
(for bitop (bit_and bit_ior)
 (simplify
  (bitop @0 @0)
  (non_lvalue @0)))

/* x & C -> x if we know that x & ~C == 0.  */
#if GIMPLE
(simplify
 (bit_and SSA_NAME@0 INTEGER_CST@1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && wi::bit_and_not (get_nonzero_bits (@0), wi::to_wide (@1)) == 0)
  @0))

/* `a & (x | CST)` -> a if we know that (a & ~CST) == 0   */
(simplify
 (bit_and:c SSA_NAME@0 (bit_ior @1 INTEGER_CST@2))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && wi::bit_and_not (get_nonzero_bits (@0), wi::to_wide (@2)) == 0)
  @0))

/* x | C -> C if we know that x & ~C == 0.  */
(simplify
 (bit_ior SSA_NAME@0 INTEGER_CST@1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && wi::bit_and_not (get_nonzero_bits (@0), wi::to_wide (@1)) == 0)
  @1))
#endif

/* ~(~X - Y) -> X + Y and ~(~X + Y) -> X - Y.  */
(simplify
 (bit_not (minus (bit_not @0) @1))
 (plus @0 @1))
(simplify
 (bit_not (plus:c (bit_not @0) @1))
 (minus @0 @1))
/* (~X - ~Y) -> Y - X.  */
(simplify
 (minus (bit_not @0) (bit_not @1))
  (if (!TYPE_OVERFLOW_SANITIZED (type))
   (with { tree utype = unsigned_type_for (type); }
    (convert (minus (convert:utype @1) (convert:utype @0))))))

/* ~(X - Y) -> ~X + Y.  */
(simplify
 (bit_not (minus:s @0 @1))
 (plus (bit_not @0) @1))
(simplify
 (bit_not (plus:s @0 INTEGER_CST@1))
 (if ((INTEGRAL_TYPE_P (type)
       && TYPE_UNSIGNED (type))
      || (!TYPE_OVERFLOW_SANITIZED (type)
	  && may_negate_without_overflow_p (@1)))
  (plus (bit_not @0) { const_unop (NEGATE_EXPR, type, @1); })))

#if GIMPLE
/* ~X + Y -> (Y - X) - 1.  */
(simplify
 (plus:c (bit_not @0) @1)
  (if (ANY_INTEGRAL_TYPE_P (type)
       && TYPE_OVERFLOW_WRAPS (type)
       /* -1 - X is folded to ~X, so we'd recurse endlessly.  */
       && !integer_all_onesp (@1))
   (plus (minus @1 @0) { build_minus_one_cst (type); })
   (if (INTEGRAL_TYPE_P (type)
	&& TREE_CODE (@1) == INTEGER_CST
	&& wi::to_wide (@1) != wi::min_value (TYPE_PRECISION (type),
					      SIGNED))
    (minus (plus @1 { build_minus_one_cst (type); }) @0))))

/* (~X) >> (type1)X -> -1 for signed X.
   The cast needs not to be truncating cast.  */
(simplify
 (rshift (bit_not @0) (convert?@1 @0))
 (if (INTEGRAL_TYPE_P (type)
      && !TYPE_UNSIGNED (type)
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@1)))
  { build_minus_one_cst (type); }))
#endif

/* ~(X >> Y) -> ~X >> Y if ~X can be simplified.  */
(simplify
 (bit_not (rshift:s @0 @1))
  (if (!TYPE_UNSIGNED (TREE_TYPE (@0)))
   (rshift (bit_not! @0) @1)
   /* For logical right shifts, this is possible only if @0 doesn't
      have MSB set and the logical right shift is changed into
      arithmetic shift.  */
   (if (INTEGRAL_TYPE_P (type)
        && !wi::neg_p (tree_nonzero_bits (@0)))
    (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
     (convert (rshift (bit_not! (convert:stype @0)) @1))))))

/* x + (x & 1) -> (x + 1) & ~1 */
(simplify
 (plus:c @0 (bit_and:s @0 integer_onep@1))
 (bit_and (plus @0 @1) (bit_not @1)))

/* x & ~(x & y) -> x & ~y */
/* x | ~(x | y) -> x | ~y  */
(for bitop (bit_and bit_ior)
 (simplify
  (bitop:c @0 (bit_not (bitop:cs @0 @1)))
  (bitop @0 (bit_not @1))))

/* (~x & y) | ~(x | y) -> ~x */
(simplify
 (bit_ior:c (bit_and:c (bit_not@2 @0) @1) (bit_not (bit_ior:c @0 @1)))
 @2)

/* (x | y) ^ (x | ~y) -> ~x */
(simplify
 (bit_xor:c (bit_ior:c @0 @1) (bit_ior:c @0 (bit_not @1)))
 (bit_not @0))

(for op (bit_and bit_xor)
     rop (bit_xor bit_and)
 /* (x & y) | ~(x | y) -> ~(x ^ y)
    (x ^ y) | ~(x | y) -> ~(x & y)  */
 (simplify
  (bit_ior:c (op:s @0 @1) (bit_not:s (bit_ior:s @0 @1)))
  (bit_not (rop @0 @1)))
 /* (x & y) ^ (x | y) -> x ^ y
    (x ^ y) ^ (x | y) -> x & y  */
 (simplify
  (bit_xor:c (op @0 @1) (bit_ior @0 @1))
  (rop @0 @1))
 /* (x | y) - (x & y) -> x ^ y
    (x | y) - (x ^ y) -> x & y  */
 (simplify
  (minus (bit_ior @0 @1) (op @0 @1))
  (rop @0 @1)))

/* (~x | y) ^ (x ^ y) -> x | ~y */
(simplify
 (bit_xor:c (bit_ior:cs (bit_not @0) @1) (bit_xor:s @0 @1))
 (bit_ior @0 (bit_not @1)))

/* (x & y) + (x ^ y) -> x | y */
/* (x & y) | (x ^ y) -> x | y */
/* (x & y) ^ (x ^ y) -> x | y */
(for op (plus bit_ior bit_xor)
 (simplify
  (op:c (bit_and @0 @1) (bit_xor @0 @1))
  (bit_ior @0 @1)))

/* ((x >> 1) + (y >> 1)) + (x & y & 1) -> (x & y) + ((x ^ y) >> 1).
   Both are the average of x and y computed without overflowing, since
   x + y is 2 * (x & y) + (x ^ y), but the second form needs four
   operations instead of six.  */
(simplify
 (plus:c (plus (rshift @0 integer_onep@2) (rshift @1 @2))
	 (bit_and (bit_and:c@3 @0 @1) integer_onep))
 (plus @3 (rshift (bit_xor @0 @1) @2)))

/* ((x >> 1) + (y >> 1)) + ((x | y) & 1) -> (x | y) - ((x ^ y) >> 1).
   The rounded-up dual of the rule above: x + y is 2 * (x | y) - (x ^ y), so
   the average rounded towards +inf is (x | y) - ((x ^ y) >> 1), again four
   operations instead of six.  This is the form used by the rounding
   motion-compensation averages in media codecs.  */
(simplify
 (plus:c (plus (rshift @0 integer_onep@2) (rshift @1 @2))
	 (bit_and (bit_ior:c@3 @0 @1) integer_onep))
 (minus @3 (rshift (bit_xor @0 @1) @2)))

/* (x & y) + (x | y) -> x + y */
(simplify
 (plus:c (bit_and @0 @1) (bit_ior @0 @1))
 (plus @0 @1))

/* (x + y) - (x | y) -> x & y
   (x + y) - (x & y) -> x | y  */
(for rop (bit_ior bit_and)
     res (bit_and bit_ior)
 (simplify
  (minus (plus @0 @1) (rop @0 @1))
  (if (!TYPE_OVERFLOW_SANITIZED (type) && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_SATURATING (type))
   (res @0 @1))))

/* X + (X | -X) -> X & (X - 1).  X | -X has every bit from the lowest set
   bit of X upwards, so adding it clears that bit.  */
(simplify
 (plus:c @0 (bit_ior:cs @0 (negate @0)))
 (if (ANY_INTEGRAL_TYPE_P (type)
      && !TYPE_OVERFLOW_SANITIZED (type))
  (bit_and @0 (plus @0 { build_minus_one_cst (type); }))))

/* X + ((-X) & (C - 1)) -> (X + C - 1) & -C for a power of two C, the
   round up to a multiple of C written with the padding computed first.  */
(simplify
 (plus:c @0 (bit_and:cs (negate @0) uniform_integer_cst_p@1))
 (with { tree cst = uniform_integer_cst_p (@1);
	 tree etype = TREE_TYPE (cst);
	 wide_int c = wi::to_wide (cst); }
  (if (ANY_INTEGRAL_TYPE_P (type)
       && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_OVERFLOW_SANITIZED (type)
       && wi::popcount (c + 1) == 1)
   (bit_and (plus @0 @1)
	    { build_uniform_cst
		(type, wide_int_to_tree (etype, wi::bit_not (c))); }))))

/* ((X - 1) | C) + 1 -> (X + C) & ~C, for C a low mask 2^k - 1.
   Both forms round X up to a multiple of 2^k.  */
(simplify
 (plus (bit_ior:s (plus @0 integer_minus_onep)
		   uniform_integer_cst_p@1)
	integer_onep)
 (with { tree cst = uniform_integer_cst_p (@1);
	 tree etype = TREE_TYPE (cst);
	 wide_int c = wi::to_wide (cst); }
  (if (TYPE_OVERFLOW_WRAPS (type)
	&& c != 0
	&& wi::popcount (c + 1) == 1)
   (bit_and (plus @0 @1)
	    { build_uniform_cst
		(type, wide_int_to_tree (etype, wi::bit_not (c))); }))))

/* (x + y) - min (x, y) -> max (x, y)
   (x + y) - max (x, y) -> min (x, y)
   The sum of the minimum and the maximum is the sum of the operands.  */
(for minmax (min max)
     maxmin (max min)
 (simplify
  (minus (plus @0 @1) (minmax @0 @1))
  (if ((ANY_INTEGRAL_TYPE_P (type)
	&& !TYPE_OVERFLOW_SANITIZED (type) && !TYPE_OVERFLOW_TRAPS (type))
	|| (FLOAT_TYPE_P (type)
	    && flag_associative_math
	    && !HONOR_SIGNED_ZEROS (type)
	    && !flag_trapping_math))
   (maxmin @0 @1))))

/* (x | y) - y -> (x & ~y) */
(simplify
 (minus (bit_ior:cs @0 @1) @1)
 (bit_and @0 (bit_not @1)))

/* (x | y) & ~(x & y) -> x ^ y */
(simplify
 (bit_and:c (bit_ior @0 @1) (bit_not (bit_and @0 @1)))
 (bit_xor @0 @1))

/* (x | y) & (~x ^ y) -> x & y */
(simplify
 (bit_and:c (bit_ior:c @0 @1) (bit_xor:c @1 @2))
 (with { bool wascmp; }
  (if (bitwise_inverted_equal_p (@0, @2, wascmp)
       && (!wascmp || element_precision (type) == 1))
   (bit_and @0 @1))))

/* (~x | y) & (x | ~y) -> ~(x ^ y) */
(simplify
 (bit_and (bit_ior:cs (bit_not @0) @1) (bit_ior:cs @0 (bit_not @1)))
 (bit_not (bit_xor @0 @1)))

/* (~x | y) ^ (x | ~y) -> x ^ y */
(simplify
 (bit_xor (bit_ior:c (bit_not @0) @1) (bit_ior:c @0 (bit_not @1)))
 (bit_xor @0 @1))

/* ((x & y) - (x | y)) - 1 -> ~(x ^ y) */
(simplify
 (plus (nop_convert1? (minus@2 (nop_convert2? (bit_and:c @0 @1))
                              (nop_convert2? (bit_ior @0 @1))))
       integer_all_onesp)
 (if (!TYPE_OVERFLOW_SANITIZED (type) && !TYPE_OVERFLOW_TRAPS (type)
      && !TYPE_SATURATING (type) && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2))
      && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@2))
      && !TYPE_SATURATING (TREE_TYPE (@2)))
 (bit_not (convert (bit_xor @0 @1)))))
(simplify
 (minus (nop_convert1? (plus@2 (nop_convert2? (bit_and:c @0 @1))
                               integer_all_onesp))
       (nop_convert3? (bit_ior @0 @1)))
 (if (!TYPE_OVERFLOW_SANITIZED (type) && !TYPE_OVERFLOW_TRAPS (type)
      && !TYPE_SATURATING (type) && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2))
      && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@2))
      && !TYPE_SATURATING (TREE_TYPE (@2)))
 (bit_not (convert (bit_xor @0 @1)))))
(simplify
 (minus (nop_convert1? (bit_and @0 @1))
       (nop_convert2? (plus@2 (nop_convert3? (bit_ior:c @0 @1))
                               integer_onep)))
 (if (!TYPE_OVERFLOW_SANITIZED (type) && !TYPE_OVERFLOW_TRAPS (type)
      && !TYPE_SATURATING (type) && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2))
      && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@2))
      && !TYPE_SATURATING (TREE_TYPE (@2)))
 (bit_not (convert (bit_xor @0 @1)))))

/* ~x & ~y -> ~(x | y)
   ~x | ~y -> ~(x & y) */
(for op (bit_and bit_ior)
     rop (bit_ior bit_and)
 (simplify
  (op (convert1? (bit_not @0)) (convert2? (bit_not @1)))
  (if (element_precision (type) <= element_precision (TREE_TYPE (@0))
       && element_precision (type) <= element_precision (TREE_TYPE (@1)))
   (bit_not (rop (convert @0) (convert @1))))))

/* If we are XORing or adding two BIT_AND_EXPR's, both of which are and'ing
   with a constant, and the two constants have no bits in common,
   we should treat this as a BIT_IOR_EXPR since this may produce more
   simplifications.  */
(for op (bit_xor plus)
 (simplify
  (op (convert1? (bit_and@4 @0 INTEGER_CST@1))
      (convert2? (bit_and@5 @2 INTEGER_CST@3)))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
       && tree_nop_conversion_p (type, TREE_TYPE (@2))
       && (wi::to_wide (@1) & wi::to_wide (@3)) == 0)
   (bit_ior (convert @4) (convert @5)))))

/* (X | Y) ^ X -> Y & ~ X*/
(simplify
 (bit_xor:c (convert1? (bit_ior:c @@0 @1)) (convert2? @0))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (convert (bit_and @1 (bit_not @0)))))

/* (~X | Y) ^ X -> ~(X & Y).  */
(simplify
 (bit_xor:c (nop_convert1? (bit_ior:c (nop_convert2? (bit_not @0)) @1)) @2)
 (if (bitwise_equal_p (@0, @2))
  (convert (bit_not (bit_and @0 (convert @1))))))

/* Convert ~X ^ ~Y to X ^ Y.  */
(simplify
 (bit_xor (convert1? (bit_not @0)) (convert2? (bit_not @1)))
 (if (element_precision (type) <= element_precision (TREE_TYPE (@0))
      && element_precision (type) <= element_precision (TREE_TYPE (@1)))
  (bit_xor (convert @0) (convert @1))))

/* Convert ~X ^ C to X ^ ~C.  */
(simplify
 (bit_xor (convert? (bit_not @0)) INTEGER_CST@1)
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (bit_xor (convert @0) (bit_not @1))))

/* Fold (X & Y) ^ Y and (X ^ Y) & Y as ~X & Y.  */
(for opo (bit_and bit_xor)
     opi (bit_xor bit_and)
 (simplify
  (opo:c (opi:cs @0 @1) @1)
  (bit_and (bit_not @0) @1)))

/* Given a bit-wise operation CODE applied to ARG0 and ARG1, see if both
   operands are another bit-wise operation with a common input.  If so,
   distribute the bit operations to save an operation and possibly two if
   constants are involved.  For example, convert
     (A | B) & (A | C) into A | (B & C)
   Further simplification will occur if B and C are constants.  */
(for op (bit_and bit_ior bit_xor)
     rop (bit_ior bit_and bit_and)
 (simplify
  (op (convert? (rop:c @@0 @1)) (convert? (rop:c @0 @2)))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
       && tree_nop_conversion_p (type, TREE_TYPE (@2)))
   (rop (convert @0) (op (convert @1) (convert @2))))))

/* Some simple reassociation for bit operations, also handled in reassoc.  */
/* (X & Y) & Y -> X & Y
   (X | Y) | Y -> X | Y  */
(for op (bit_and bit_ior)
 (simplify
  (op:c (convert1?@2 (op:c @0 @@1)) (convert2? @1))
  @2))
/* (X ^ Y) ^ Y -> X  */
(simplify
 (bit_xor:c (convert1? (bit_xor:c @0 @@1)) (convert2? @1))
 (convert @0))

/* (X & ~Y) & Y -> 0 */
(simplify
 (bit_and:c (bit_and @0 @1) @2)
 (with { bool wascmp; }
  (if (bitwise_inverted_equal_p (@0, @2, wascmp)
       || bitwise_inverted_equal_p (@1, @2, wascmp))
   { wascmp ? constant_boolean_node (false, type) : build_zero_cst (type); })))
/* (X | ~Y) | Y -> -1 */
(simplify
 (bit_ior:c (bit_ior @0 @1) @2)
 (with { bool wascmp; }
  (if ((bitwise_inverted_equal_p (@0, @2, wascmp)
        || bitwise_inverted_equal_p (@1, @2, wascmp))
       && (!wascmp || element_precision (type) == 1))
   { build_all_ones_cst (TREE_TYPE (@0)); })))

/* (X & Y) & (X & Z) -> (X & Y) & Z
   (X | Y) | (X | Z) -> (X | Y) | Z  */
(for op (bit_and bit_ior)
 (simplify
  (op (convert1?@3 (op:c@4 @0 @1)) (convert2?@5 (op:c@6 @0 @2)))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
       && tree_nop_conversion_p (type, TREE_TYPE (@2)))
   (if (single_use (@5) && single_use (@6))
    (op @3 (convert @2))
    (if (single_use (@3) && single_use (@4))
     (op (convert @1) @5))))))
/* (X ^ Y) ^ (X ^ Z) -> Y ^ Z  */
(simplify
 (bit_xor (convert1? (bit_xor:c @0 @1)) (convert2? (bit_xor:c @0 @2)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
      && tree_nop_conversion_p (type, TREE_TYPE (@2)))
  (bit_xor (convert @1) (convert @2))))

/* Convert abs (abs (X)) into abs (X).
   also absu (absu (X)) into absu (X).  */
(simplify
 (abs (abs@1 @0))
 @1)

(simplify
 (absu (convert@2 (absu@1 @0)))
 (if (tree_nop_conversion_p (TREE_TYPE (@2), TREE_TYPE (@1)))
  @1))

/* Convert abs[u] (-X) -> abs[u] (X).  */
(simplify
 (abs (negate @0))
 (abs @0))

/* fabs(x + 0.0) -> fabs(x), safe even with signed zeros when -fno-trapping-math.
   With non-default exception handling denormal + 0.0 might trap.
   Otherwise !HONOR_SNANS would be sufficient here.  */
(for op (plus minus)
 (simplify
  (abs (op @0 real_zerop@1))
  (if (!flag_trapping_math)
   (abs @0))))

(simplify
 (absu (negate @0))
 (absu @0))

/* Convert abs[u] (X)  where X is nonnegative -> (X).  */
(simplify
 (abs tree_expr_nonnegative_p@0)
 @0)

(simplify
 (absu tree_expr_nonnegative_p@0)
 (convert @0))

/* Simplify (-(X < 0) | 1) * X into abs (X) or absu(X).  */
(simplify
 (mult:c (nop_convert1?
	  (bit_ior (nop_convert2? (negate (convert? (lt @0 integer_zerop))))
		    integer_onep))
	 (nop_convert3? @0))
 (if (INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (if (TYPE_UNSIGNED (type))
   (absu @0)
   (abs @0)
  )
 )
)

/* A few cases of fold-const.cc negate_expr_p predicate.  */
(match negate_expr_p
 INTEGER_CST
 (if ((INTEGRAL_TYPE_P (type)
       && TYPE_UNSIGNED (type))
      || (!TYPE_OVERFLOW_SANITIZED (type)
	  && may_negate_without_overflow_p (t)))))
(match negate_expr_p
 FIXED_CST)
(match negate_expr_p
 (negate @0)
 (if (!TYPE_OVERFLOW_SANITIZED (type))))
(match negate_expr_p
 REAL_CST
 (if (REAL_VALUE_NEGATIVE (TREE_REAL_CST (t)))))
/* VECTOR_CST handling of non-wrapping types would recurse in unsupported
   ways.  */
(match negate_expr_p
 VECTOR_CST
 (if (FLOAT_TYPE_P (TREE_TYPE (type)) || TYPE_OVERFLOW_WRAPS (type))))
(match negate_expr_p
 (minus @0 @1)
 (if ((ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_WRAPS (type))
      || (FLOAT_TYPE_P (type)
	  && !HONOR_SIGN_DEPENDENT_ROUNDING (type)
	  && !HONOR_SIGNED_ZEROS (type)))))

/* (-A) * (-B) -> A * B  */
(simplify
 (mult:c (convert1? (negate @0)) (convert2? negate_expr_p@1))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
       && tree_nop_conversion_p (type, TREE_TYPE (@1)))
   (mult (convert @0) (convert (negate @1)))))

/* -(A + B) -> (-B) - A.  */
(simplify
 (negate (plus:c @0 negate_expr_p@1))
 (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
      && !HONOR_SIGNED_ZEROS (type))
  (minus (negate @1) @0)))

/* -(A - B) -> B - A.  */
(simplify
 (negate (minus @0 @1))
 (if ((ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_SANITIZED (type))
      || (FLOAT_TYPE_P (type)
	  && !HONOR_SIGN_DEPENDENT_ROUNDING (type)
	  && !HONOR_SIGNED_ZEROS (type)))
  (minus @1 @0)))
(simplify
 (negate (nop_convert? (pointer_diff@2 @0 @1)))
 (if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_SANITIZED (type))
  (with { tree ptrdifftype = TREE_TYPE (@2); }
   (convert (pointer_diff:ptrdifftype @1 @0)))))

/* A - B -> A + (-B) if B is easily negatable.  */
(simplify
 (minus @0 negate_expr_p@1)
 (if (!FIXED_POINT_TYPE_P (type))
 (plus @0 (negate @1))))

/* A - MIN (B, C) -> A + MAX (-B, -C)
   A - MAX (B, C) -> A + MIN (-B, -C)  if B and C are easily negatable.  */
(for minmax (min max)
     maxmin (max min)
 (simplify
  (minus @0 (minmax:s negate_expr_p@1 negate_expr_p@2))
  (if (ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_UNDEFINED (type))
   (plus @0 (maxmin (negate @1) (negate @2))))))

/* 1 - a is a ^ 1 if a had a bool range. */
/* This is only enabled for gimple as sometimes
   cfun is not set for the function which contains
   the SSA_NAME (e.g. while IPA passes are happening,
   fold might be called).  */
(simplify
 (minus integer_onep@0 SSA_NAME@1)
  (if (INTEGRAL_TYPE_P (type)
       && ssa_name_has_boolean_range (@1))
   (bit_xor @1 @0)))

/* Other simplifications of negation (c.f. fold_negate_expr_1).  */
(simplify
 (negate (mult:c@0 @1 negate_expr_p@2))
 (if (! TYPE_UNSIGNED (type)
      && ! HONOR_SIGN_DEPENDENT_ROUNDING (type)
      && single_use (@0))
  (mult @1 (negate @2))))

(simplify
 (negate (rdiv@0 @1 negate_expr_p@2))
 (if (! HONOR_SIGN_DEPENDENT_ROUNDING (type)
      && single_use (@0))
  (rdiv @1 (negate @2))))

(simplify
 (negate (rdiv@0 negate_expr_p@1 @2))
 (if (! HONOR_SIGN_DEPENDENT_ROUNDING (type)
      && single_use (@0))
  (rdiv (negate @1) @2)))

/* Fold -((int)x >> (prec - 1)) into (unsigned)x >> (prec - 1).  */
(simplify
 (negate (convert? (rshift @0 INTEGER_CST@1)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && wi::to_wide (@1) == element_precision (type) - 1)
  (with { tree stype = TREE_TYPE (@0);
	  tree ntype = TYPE_UNSIGNED (stype) ? signed_type_for (stype)
					     : unsigned_type_for (stype); }
   (if (VECTOR_TYPE_P (type))
    (view_convert (rshift (view_convert:ntype @0) @1))
    (convert (rshift (convert:ntype @0) @1))))))

/* IFN_AVG_FLOOR (x, x) and IFN_AVG_CEIL (x, x) are both x.  */
#if GIMPLE
(for avg (IFN_AVG_FLOOR IFN_AVG_CEIL)
 (simplify
  (avg @0 @0)
  @0))

 /* Fold (x + y) >> 1 into IFN_AVG_FLOOR (x, y) and ((x + y) + 1) >> 1
    into IFN_AVG_CEIL (x, y) if x and y are vectors in which the sum
    cannot wrap: overflow is undefined, or each element of x and y is
    known to have at least one leading zero bit, or two if the type is
    signed, since the sum of two nonnegative values can then wrap to a
    negative one.  The IFN_AVG_CEIL fold comes first because
    ((x + y) + 1) >> 1 is also (x' + 1) >> 1.  */
(simplify
 (rshift (plus:s (plus:cs @0 @1) integer_onep) integer_onep)
 (if (VECTOR_TYPE_P (type)
      && direct_internal_fn_supported_p (IFN_AVG_CEIL, type,
					 OPTIMIZE_FOR_BOTH))
  (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; }
   (if (TYPE_OVERFLOW_UNDEFINED (type)
	|| (wi::clz (get_nonzero_bits (@0)) >= min_clz
	    && wi::clz (get_nonzero_bits (@1)) >= min_clz))
    (IFN_AVG_CEIL @0 @1)))))
(simplify
 (rshift (plus:cs @0 @1) integer_onep)
 (if (VECTOR_TYPE_P (type)
      && direct_internal_fn_supported_p (IFN_AVG_FLOOR, type,
					 OPTIMIZE_FOR_BOTH))
  (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; }
   (if (TYPE_OVERFLOW_UNDEFINED (type)
	|| (wi::clz (get_nonzero_bits (@0)) >= min_clz
	    && wi::clz (get_nonzero_bits (@1)) >= min_clz))
    (IFN_AVG_FLOOR @0 @1)))))
#endif

/* Try to fold (type) X op CST -> (type) (X op ((type-x) CST))
   when profitable.
   For bitwise binary operations apply operand conversions to the
   binary operation result instead of to the operands.  This allows
   to combine successive conversions and bitwise binary operations.
   We combine the above two cases by using a conditional convert.  */
(for bitop (bit_and bit_ior bit_xor)
 (simplify
  (bitop (convert@2 @0) (convert?@3 @1))
  (if (((TREE_CODE (@1) == INTEGER_CST
	 && INTEGRAL_TYPE_P (TREE_TYPE (@0))
	 && (int_fits_type_p (@1, TREE_TYPE (@0))
	     || tree_nop_conversion_p (TREE_TYPE (@0), type)))
	|| types_match (@0, @1))
       && !POINTER_TYPE_P (TREE_TYPE (@0))
       && !VECTOR_TYPE_P (TREE_TYPE (@0))
       && TREE_CODE (TREE_TYPE (@0)) != OFFSET_TYPE
       /* ???  This transform conflicts with fold-const.cc doing
	  Convert (T)(x & c) into (T)x & (T)c, if c is an integer
	  constants (if x has signed type, the sign bit cannot be set
	  in c).  This folds extension into the BIT_AND_EXPR.
	  Restrict it to GIMPLE to avoid endless recursions.  */
       && (bitop != BIT_AND_EXPR || GIMPLE)
       && (/* That's a good idea if the conversion widens the operand, thus
	      after hoisting the conversion the operation will be narrower.
	      It is also a good if the conversion is a nop as moves the
	      conversion to one side; allowing for combining of the conversions.  */
	   TYPE_PRECISION (TREE_TYPE (@0)) < TYPE_PRECISION (type)
	   /* The conversion check for being a nop can only be done at the gimple
	      level as fold_binary has some re-association code which can conflict
	      with this if there is a "constant" which is not a full INTEGER_CST.  */
	   || (GIMPLE && TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (type))
	   /* It's also a good idea if the conversion is to a non-integer
	      mode.  */
	   || GET_MODE_CLASS (TYPE_MODE (type)) != MODE_INT
	   /* Or if the precision of TO is not the same as the precision
	      of its mode.  */
	   || !type_has_mode_precision_p (type)
	   /* In GIMPLE, getting rid of 2 conversions for one new results
	      in smaller IL.  */
	   || (GIMPLE
	       && TREE_CODE (@1) != INTEGER_CST
	       && tree_nop_conversion_p (type, TREE_TYPE (@0))
	       && single_use (@2)
	       && single_use (@3))))
   (convert (bitop @0 (convert @1)))))
 /* In GIMPLE, getting rid of 2 conversions for one new results
    in smaller IL.  */
 (simplify
  (convert (bitop:cs@2 (nop_convert:s @0) @1))
  (if (GIMPLE
       && TREE_CODE (@1) != INTEGER_CST
       && tree_nop_conversion_p (type, TREE_TYPE (@2))
       && types_match (type, @0)
       && !POINTER_TYPE_P (TREE_TYPE (@0))
       && TREE_CODE (TREE_TYPE (@0)) != OFFSET_TYPE)
   (bitop @0 (convert @1))))
 /* Similar as above, but @0 has a widen type.  */
 (simplify
  (convert (bitop:cs@2 (convert:s @0) @1))
   (if (GIMPLE
	&& INTEGRAL_TYPE_P (type)
	&& INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TREE_CODE (@1) != INTEGER_CST
	&& tree_nop_conversion_p (type, TREE_TYPE (@2))
	&& TYPE_PRECISION (TREE_TYPE (@0)) > TYPE_PRECISION (type))
   (bitop:type (convert @0) (convert @1))
   /* Similar as above, but the outer and inner most types match
       and it was widening cast; replacing 2 casts with only one.  */
   (if (GIMPLE
        && INTEGRAL_TYPE_P (type)
        && types_match (type, TREE_TYPE (@0))
        && TYPE_PRECISION (TREE_TYPE (@2)) > TYPE_PRECISION (type))
    (bitop:type @0 (convert @1))))))

(for bitop (bit_and bit_ior)
     rbitop (bit_ior bit_and)
  /* (x | y) & x -> x */
  /* (x & y) | x -> x */
 (simplify
  (bitop:c (rbitop:c @0 @1) @0)
  @0)
  /* (x | y) & (x & z) -> (x & z) */
  /* (x & y) | (x | z) -> (x | z) */
 (simplify
  (bitop:c (rbitop:c @0 @1) (bitop:c@3 @0 @2))
  @3)
 /* (x | c) & ~(y | c) -> x & ~(y | c) */
 /* (x & c) | ~(y & c) -> x | ~(y & c) */
 (simplify
  (bitop:c (rbitop:c @0 @1) (bit_not@3 (rbitop:c @1 @2)))
  (bitop @0 @3))
 /* x & ~(y | x) -> 0 */
 /* x | ~(y & x) -> -1 */
 (simplify
  (bitop:c @0 (bit_not (rbitop:c @0 @1)))
  (if (bitop == BIT_AND_EXPR)
   { build_zero_cst (type); }
   { build_minus_one_cst (type); })))

/* (~x | y) & x -> x & y
   (~x & y) | x -> x | y
   (~x & y) ^ x -> x | y */
(for bitop (bit_and bit_ior bit_xor)
     rbitop (bit_ior bit_and bit_and)
     resbitop (bit_and bit_ior bit_ior)
 (simplify
  (bitop:c (rbitop:c @2 @1) @0)
  (with { bool wascmp; }
   (if (bitwise_inverted_equal_p (@0, @2, wascmp)
	&& (!wascmp || element_precision (type) == 1))
    (resbitop @0 @1)))))

/* ((x | y) & z) | x -> (z & y) | x
   ((x ^ y) & z) | x -> (z & y) | x  */
(for op (bit_ior bit_xor)
 (simplify
  (bit_ior:c (nop_convert1?:s
	       (bit_and:cs (nop_convert2?:s (op:cs @0 @1)) @2)) @3)
  (if (bitwise_equal_p (@0, @3))
   (convert (bit_ior (bit_and @1 (convert @2)) (convert @0))))))

/* (x | CST1) & CST2 -> (x & CST2) | (CST1 & CST2) */
(simplify
  (bit_and (bit_ior @0 CONSTANT_CLASS_P@1) CONSTANT_CLASS_P@2)
  (bit_ior (bit_and @0 @2) (bit_and! @1 @2)))

/* Combine successive equal operations with constants.  */
(for bitop (bit_and bit_ior bit_xor)
 (simplify
  (bitop (bitop @0 CONSTANT_CLASS_P@1) CONSTANT_CLASS_P@2)
  (if (!CONSTANT_CLASS_P (@0))
   /* This is the canonical form regardless of whether (bitop @1 @2) can be
      folded to a constant.  */
   (bitop @0 (bitop! @1 @2))
   /* In this case we have three constants and (bitop @0 @1) doesn't fold
      to a constant.  This can happen if @0 or @1 is a POLY_INT_CST and if
      the values involved are such that the operation can't be decided at
      compile time.  Try folding one of @0 or @1 with @2 to see whether
      that combination can be decided at compile time.

      Keep the existing form if both folds fail, to avoid endless
      oscillation.  */
   (with { tree cst1 = const_binop (bitop, type, @0, @2); }
    (if (cst1)
     (bitop @1 { cst1; })
     (with { tree cst2 = const_binop (bitop, type, @1, @2); }
      (if (cst2)
       (bitop @0 { cst2; }))))))))

/* Try simple folding for X op !X, and X op X with the help
   of the truth_valued_p and logical_inverted_value predicates.  */
(match truth_valued_p
 @0
 (if (INTEGRAL_TYPE_P (type) && TYPE_PRECISION (type) == 1)))
(for op (tcc_comparison truth_and truth_andif truth_or truth_orif truth_xor)
 (match truth_valued_p
  (op @0 @1)))
(match truth_valued_p
  (truth_not @0))

(match (logical_inverted_value @0)
 (truth_not @0))
(match (logical_inverted_value @0)
 (bit_not truth_valued_p@0))
(match (logical_inverted_value @0)
 (eq @0 integer_zerop))
(match (logical_inverted_value @0)
 (ne truth_valued_p@0 integer_truep))
(match (logical_inverted_value @0)
 (bit_xor truth_valued_p@0 integer_truep))

/* X & !X -> 0.  */
(simplify
 (bit_and:c @0 (logical_inverted_value @0))
 { build_zero_cst (type); })
/* X | !X and X ^ !X -> 1, if X is truth-valued.
   X ==/!= !X is false/true.  */
(for op (bit_ior bit_xor eq ne)
 (simplify
  (op:c truth_valued_p@0 (logical_inverted_value @0))
  { constant_boolean_node (op != EQ_EXPR, type); }))

/* ~~x -> x */
(simplify
  (bit_not (bit_not @0))
  (non_lvalue @0))

/* zero_one_valued_p will match when a value is known to be either
   0 or 1 including constants 0 or 1.
   Signed 1-bits includes -1 so they cannot match here. */
(match zero_one_valued_p
 @0
 (if (INTEGRAL_TYPE_P (type)
      && (TYPE_UNSIGNED (type)
	  || TYPE_PRECISION (type) > 1)
      && wi::leu_p (tree_nonzero_bits (@0), 1))))
(match zero_one_valued_p
 truth_valued_p@0
 (if (INTEGRAL_TYPE_P (type)
      && (TYPE_UNSIGNED (type)
	  || TYPE_PRECISION (type) > 1))))

/* (a&1) is always [0,1] too. This is useful again when
   the range is not known. */
/* Note this can't be recursive due to VN handling of equivalents,
   VN and would cause an infinite recursion. */
(match zero_one_valued_p
 (bit_and:c@0 @1 integer_onep)
 (if (INTEGRAL_TYPE_P (type))))

/* A conversion from an zero_one_valued_p is still a [0,1].
   This is useful when the range of a variable is not known */
/* Note this matches can't be recursive because of the way VN handles
   nop conversions being equivalent and then recursive between them. */
(match zero_one_valued_p
 (convert@0 @1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
      && (TYPE_UNSIGNED (TREE_TYPE (@1))
	  || TYPE_PRECISION (TREE_TYPE (@1)) > 1)
      && INTEGRAL_TYPE_P (type)
      && (TYPE_UNSIGNED (type)
	  || TYPE_PRECISION (type) > 1)
      && wi::leu_p (tree_nonzero_bits (@1), 1))))

/* VCE<bool>(zero_one) ==/!= false -> zero_one ==/!= 0.
   This is done as SRA likes to create VCE for boolean accesses
   and it helps code gneration and uninitialized variable warnings.  */
(for neeq (ne eq)
 (simplify
  (neeq (view_convert@0 zero_one_valued_p@1) integer_zerop)
  (if (TREE_CODE (TREE_TYPE (@0)) == BOOLEAN_TYPE
       && TYPE_PRECISION (TREE_TYPE (@0)) == 1
       && TYPE_UNSIGNED (TREE_TYPE (@0)))
   (neeq @1 { build_zero_cst (TREE_TYPE (@1)); } ))))

/* Transform { 0 or 1 } * { 0 or 1 } into { 0 or 1 } & { 0 or 1 }.  */
(simplify
 (mult zero_one_valued_p@0 zero_one_valued_p@1)
 (if (INTEGRAL_TYPE_P (type))
  (bit_and @0 @1)))

/* Transform { 0 or 1 } * -{ 0 or 1 } into -({ 0 or 1 } & { 0 or 1 }).  */
(simplify
 (mult:c (negate zero_one_valued_p@0) zero_one_valued_p@1)
 (negate (bit_and @0 @1)))

/* Fold `(a ? b : 0) | (a ? 0 : c)` into (a ? b : c).
    Handle also ^ and + in replacement of `|`. */
(for cnd (cond vec_cond)
 (for op (bit_ior bit_xor plus)
  (simplify
   (op:c
    (cnd:s @0 @00 integer_zerop)
    (cnd:s @0 integer_zerop @01))
   (cnd @0 @00 @01))))

(for cmp (tcc_comparison)
     icmp (inverted_tcc_comparison)
 /* Fold (((a < b) & c) | ((a >= b) & d)) into (a < b ? c : d) & 1.  */
 (simplify
  (bit_ior
   (bit_and:c (convert? (cmp@0  @01 @02)) @3)
   (bit_and:c (convert? (icmp@4 @01 @02)) @5))
    (if (INTEGRAL_TYPE_P (type)
	 && invert_tree_comparison (cmp, HONOR_NANS (@01)) == icmp
	 /* The scalar version has to be canonicalized after vectorization
	    because it makes unconditional loads conditional ones, which
	    means we lose vectorization because the loads may trap.  */
	 && canonicalize_math_after_vectorization_p ())
     (bit_and (cond @0 @3 @5) { build_one_cst (type); })))

 /* Fold ((-(a < b) & c) | (-(a >= b) & d)) into a < b ? c : d.  This is
    canonicalized further and we recognize the conditional form:
    (a < b ? c : 0) | (a >= b ? d : 0) into a < b ? c : d.
    Handle also ^ and + in replacement of `|`. */
 (for op (bit_ior bit_xor plus)
  (simplify
   (op
    (cond (cmp@0  @01 @02) @3 zerop)
    (cond (icmp@4 @01 @02) @5 zerop))
     (if (INTEGRAL_TYPE_P (type)
	  && invert_tree_comparison (cmp, HONOR_NANS (@01)) == icmp
	  /* The scalar version has to be canonicalized after vectorization
	     because it makes unconditional loads conditional ones, which
	     means we lose vectorization because the loads may trap.  */
	  && canonicalize_math_after_vectorization_p ())
     (cond @0 @3 @5))))

 /* Vector Fold (((a < b) & c) | ((a >= b) & d)) into a < b ? c : d.
    and ((~(a < b) & c) | (~(a >= b) & d)) into a < b ? c : d.  */
 (simplify
  (bit_ior
   (bit_and:c (vec_cond:s (cmp@0 @6 @7) @4 @5) @2)
   (bit_and:c (vec_cond:s (icmp@1 @6 @7) @4 @5) @3))
    (if (integer_zerop (@5)
	 && invert_tree_comparison (cmp, HONOR_NANS (@6)) == icmp)
     (switch
      (if (integer_onep (@4))
       (bit_and (vec_cond @0 @2 @3) @4))
	(if (integer_minus_onep (@4))
	 (vec_cond @0 @2 @3)))
    (if (integer_zerop (@4)
	 && invert_tree_comparison (cmp, HONOR_NANS (@6)) == icmp)
     (switch
      (if (integer_onep (@5))
       (bit_and (vec_cond @0 @3 @2) @5))
      (if (integer_minus_onep (@5))
       (vec_cond @0 @3 @2))))))

 /* Scalar Vectorized Fold ((-(a < b) & c) | (-(a >= b) & d))
    into a < b ? d : c.
    Handle also ^ and + in replacement of `|`. */
 (for op (bit_ior bit_xor plus)
  (simplify
   (op
    (vec_cond:s (cmp@0 @4 @5) @2 integer_zerop)
    (vec_cond:s (icmp@1 @4 @5) @3 integer_zerop))
   (if (invert_tree_comparison (cmp, HONOR_NANS (@4)) == icmp)
    (vec_cond @0 @2 @3)))))

/* Transform X & -Y into X * Y when Y is { 0 or 1 }.  */
(simplify
 (bit_and:c (convert? (negate zero_one_valued_p@0)) @1)
 (if (INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TREE_CODE (TREE_TYPE (@0)) != BOOLEAN_TYPE
      /* Sign extending of the neg or a truncation of the neg
         is needed. */
      && (!TYPE_UNSIGNED (TREE_TYPE (@0))
	  || TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))))
  (mult (convert @0) @1)))

/* Narrow integer multiplication by a zero_one_valued_p operand.
   Multiplication by [0,1] is guaranteed not to overflow except for
   1bit signed types.  */
(simplify
 (convert (mult@0 zero_one_valued_p@1 INTEGER_CST@2))
 (if (INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@0))
      && (TYPE_UNSIGNED (type) || TYPE_PRECISION (type) > 1))
  (mult (convert @1) (convert @2))))

/* (X << C) != 0 can be simplified to X, when X is zero_one_valued_p.
   (X << C) == 0 can be simplified to X == 0, when X is zero_one_valued_p.
   Check that the shift is well-defined (C is less than TYPE_PRECISION)
   as some targets (such as x86's SSE) may return zero for larger C.  */
(for neeq (ne eq)
 (simplify
  (neeq (lshift zero_one_valued_p@0 INTEGER_CST@1) integer_zerop@2)
   (if (tree_fits_uhwi_p (@1)
        && tree_to_uhwi (@1) < TYPE_PRECISION (TREE_TYPE (@0)))
    (if (neeq == NE_EXPR)
     (convert @0)
     (eq @0 @2)))))

/* PR110010: (A >> C) != (B >> C) -> (A ^ B) >= (1 << C)
   and likewise (A >> C) == (B >> C) -> (A ^ B) < (1 << C).

   Note that maybe_canonicalize_comparison_1 related patterns
   might change ">= (1 << C)" to "> (1 << C - 1)" and
   "< (1 << C)" to "<= (1 << C - 1)".  */
(for cmp (ne eq)
     icmp (ge lt)
 (simplify
  (cmp (rshift @0 INTEGER_CST@2) (rshift @1 INTEGER_CST@2))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_UNSIGNED (TREE_TYPE (@0))
	&& type_has_mode_precision_p (TREE_TYPE (@0))
	&& types_match (@0, @1)
	&& tree_fits_shwi_p (@2)
	&& tree_to_shwi (@2) > 0
	&& tree_to_shwi (@2) < TYPE_PRECISION (TREE_TYPE (@0)))
  (icmp (bit_xor @0 @1) (lshift { build_one_cst (TREE_TYPE (@0)); } @2)))))

/* Convert ~ (-A) to A - 1.  */
(simplify
 (bit_not (convert? (negate @0)))
 (if (element_precision (type) <= element_precision (TREE_TYPE (@0))
      || !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (convert (minus @0 { build_each_one_cst (TREE_TYPE (@0)); }))))

/* Convert - (~A) to A + 1.  */
(simplify
 (negate (nop_convert? (bit_not @0)))
 (plus (view_convert @0) { build_each_one_cst (type); }))

/* (a & b) ^ (a == b) -> !(a | b) */
/* (a & b) == (a ^ b) -> !(a | b) */
(for first_op (bit_xor eq)
     second_op (eq bit_xor)
 (simplify
  (first_op:c (bit_and:c truth_valued_p@0 truth_valued_p@1) (second_op @0 @1))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_PRECISION (TREE_TYPE (@0)) == 1)
    (convert:type (bit_not (bit_ior @0 @1)))
    (convert:type (bit_xor (bit_ior @0 @1) { build_one_cst (TREE_TYPE (@0)); })))))

/* Convert ~ (A - 1) or ~ (A + -1) to -A.  */
(simplify
 (bit_not (convert? (minus @0 integer_each_onep)))
 (if (element_precision (type) <= element_precision (TREE_TYPE (@0))
      || !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (convert (negate @0))))
(simplify
 (bit_not (convert? (plus @0 integer_all_onesp)))
 (if (element_precision (type) <= element_precision (TREE_TYPE (@0))
      || !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (convert (negate @0))))

/* Part of convert ~(X ^ Y) to ~X ^ Y or X ^ ~Y if ~X or ~Y simplify.  */
(simplify
 (bit_not (convert? (bit_xor @0 INTEGER_CST@1)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (convert (bit_xor @0 (bit_not @1)))))
(simplify
 (bit_not (convert? (bit_xor:c (bit_not @0) @1)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (convert (bit_xor @0 @1))))

/* Otherwise prefer ~(X ^ Y) to ~X ^ Y as more canonical.  */
(simplify
 (bit_xor:c (nop_convert?:s (bit_not:s @0)) @1)
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (bit_not (bit_xor (view_convert @0) @1))))

/* ~(a ^ b) is a == b for truth valued a and b.  */
(simplify
 (bit_not (bit_xor:s truth_valued_p@0 truth_valued_p@1))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  (convert (eq @0 @1))))

/* (~a) == b is a ^ b for truth valued a and b.  */
(simplify
 (eq:c (bit_not:s truth_valued_p@0) truth_valued_p@1)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (TREE_TYPE (@0)) == 1)
  (convert (bit_xor @0 @1))))

/* (x & ~m) | (y & m) -> ((x ^ y) & m) ^ x */
(simplify
 (bit_ior:c (bit_and:cs @0 (bit_not @2)) (bit_and:cs @1 @2))
 (bit_xor (bit_and (bit_xor @0 @1) @2) @0))

/* Fold A - (A & B) into ~B & A.  */
(simplify
 (minus (convert1? @0) (convert2?:s (bit_and:cs @@0 @1)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && tree_nop_conversion_p (type, TREE_TYPE (@1)))
  (convert (bit_and (bit_not @1) @0))))

/* (m1 CMP m2) * d -> (m1 CMP m2) ? d : 0  */
(if (fold_before_rtl_expansion_p ())
 (simplify
  (mult:c (convert (tcc_comparison@0 @1 @2)) @3)
  (if (INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (cond @0 @3 { build_zero_cst (type); })))
/* (-(m1 CMP m2)) & d -> (m1 CMP m2) ? d : 0  */
 (simplify
  (bit_and:c (negate (convert (tcc_comparison@0 @1 @2))) @3)
  (if (INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (cond @0 @3 { build_zero_cst (type); })))
)

/* Canonicalize `(type) (a CMP b)` to `(a CMP b) ? (type) true : (type) false`
   for floating point types.   */
(simplify
 (float (convert:s (tcc_comparison@0 @1 @2)))
 (cond @0 { build_one_cst (type); } { build_zero_cst (type); } ))

/* For integral types with undefined overflow and C != 0 fold
   x * C EQ/NE y * C into x EQ/NE y.  */
(for cmp (eq ne)
 (simplify
  (cmp (mult:c @0 @1) (mult:c @2 @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
       && tree_expr_nonzero_p (@1))
   (cmp @0 @2))))

/* For integral types with wrapping overflow and C odd fold
   x * C EQ/NE y * C into x EQ/NE y.  */
(for cmp (eq ne)
 (simplify
  (cmp (mult @0 INTEGER_CST@1) (mult @2 @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))
       && (TREE_INT_CST_LOW (@1) & 1) != 0)
   (cmp @0 @2))))

/* For integral types with undefined overflow and C != 0 fold
   x * C RELOP y * C into:

   x RELOP y for nonnegative C
   y RELOP x for negative C  */
(for cmp (lt gt le ge)
 (simplify
  (cmp (mult:c @0 @1) (mult:c @2 @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (if (tree_expr_nonnegative_p (@1) && tree_expr_nonzero_p (@1))
    (cmp @0 @2)
   (if (TREE_CODE (@1) == INTEGER_CST
	&& wi::neg_p (wi::to_wide (@1), TYPE_SIGN (TREE_TYPE (@1))))
    (cmp @2 @0))))))

/* (X - 1U) <= INT_MAX-1U into (int) X > 0.  */
(for cmp (le gt)
     icmp (gt le)
 (simplify
  (cmp (plus @0 integer_minus_onep@1) INTEGER_CST@2)
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_UNSIGNED (TREE_TYPE (@0))
	&& TYPE_PRECISION (TREE_TYPE (@0)) > 1
	&& (wi::to_wide (@2)
	    == wi::max_value (TYPE_PRECISION (TREE_TYPE (@0)), SIGNED) - 1))
    (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
     (icmp (convert:stype @0) { build_int_cst (stype, 0); })))))

/* X / 4 < Y / 4 iff X < Y when the division is known to be exact.  */
(for cmp (simple_comparison)
 (simplify
  (cmp (convert?@3 (exact_div @0 INTEGER_CST@2)) (convert? (exact_div @1 @2)))
  (if (element_precision (@3) >= element_precision (@0)
       && types_match (@0, @1))
   (if (wi::lt_p (wi::to_wide (@2), 0, TYPE_SIGN (TREE_TYPE (@2))))
    (if (!TYPE_UNSIGNED (TREE_TYPE (@3)))
     (cmp @1 @0)
     (if (tree_expr_nonzero_p (@0) && tree_expr_nonzero_p (@1))
      (with
       {
	tree utype = unsigned_type_for (TREE_TYPE (@0));
       }
       (cmp (convert:utype @1) (convert:utype @0)))))
    (if (wi::gt_p (wi::to_wide (@2), 1, TYPE_SIGN (TREE_TYPE (@2))))
     (if (TYPE_UNSIGNED (TREE_TYPE (@0)) || !TYPE_UNSIGNED (TREE_TYPE (@3)))
      (cmp @0 @1)
      (with
       {
	tree utype = unsigned_type_for (TREE_TYPE (@0));
       }
       (cmp (convert:utype @0) (convert:utype @1)))))))))

/* X / C1 op C2 into a simple range test.  */
(for cmp (simple_comparison)
 (simplify
  (cmp (trunc_div:s @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && integer_nonzerop (@1)
       && !TREE_OVERFLOW (@1)
       && !TREE_OVERFLOW (@2))
   (with { tree lo, hi; bool neg_overflow;
	   enum tree_code code = fold_div_compare (cmp, @1, @2, &lo, &hi,
						   &neg_overflow); }
    (switch
     (if (code == LT_EXPR || code == GE_EXPR)
       (if (TREE_OVERFLOW (lo))
	{ build_int_cst (type, (code == LT_EXPR) ^ neg_overflow); }
	(if (code == LT_EXPR)
	 (lt @0 { lo; })
	 (ge @0 { lo; }))))
     (if (code == LE_EXPR || code == GT_EXPR)
       (if (TREE_OVERFLOW (hi))
	{ build_int_cst (type, (code == LE_EXPR) ^ neg_overflow); }
	(if (code == LE_EXPR)
	 (le @0 { hi; })
	 (gt @0 { hi; }))))
     (if (!lo && !hi)
      { build_int_cst (type, code == NE_EXPR); })
     (if (code == EQ_EXPR && !hi)
      (ge @0 { lo; }))
     (if (code == EQ_EXPR && !lo)
      (le @0 { hi; }))
     (if (code == NE_EXPR && !hi)
      (lt @0 { lo; }))
     (if (code == NE_EXPR && !lo)
      (gt @0 { hi; }))
     (if (GENERIC)
      { build_range_check (UNKNOWN_LOCATION, type, @0, code == EQ_EXPR,
			   lo, hi); })
     (with
      {
	tree etype = range_check_type (TREE_TYPE (@0));
	if (etype)
	  {
	    hi = fold_convert (etype, hi);
	    lo = fold_convert (etype, lo);
	    hi = const_binop (MINUS_EXPR, etype, hi, lo);
	  }
      }
      (if (etype && hi && !TREE_OVERFLOW (hi))
       (if (code == EQ_EXPR)
	(le (minus (convert:etype @0) { lo; }) { hi; })
	(gt (minus (convert:etype @0) { lo; }) { hi; })))))))))

/* Fold (X >> C) != 0 into X < 0 if C is one less than the width
   of X.  Similarly fold (X >> C) == 0 into X >= 0.  */
(for neeq (ne eq)
     ltge (lt ge)
 (simplify
  (neeq
   (rshift@2 @0 INTEGER_CST@1)
   integer_zerop)
  (with { tree itype = signed_type_for (TREE_TYPE (@0)); }
   /* Make sure to transform vector compares only to supported
      ones or from unsupported ones and check that only after
      IPA so offloaded code is handled correctly in this regard.  */
   (if (wi::to_wide (@1) == element_precision (itype) - 1
	&& (!VECTOR_TYPE_P (itype)
	    || (cfun
		&& cfun->after_inlining
		&& VECTOR_BOOLEAN_TYPE_P (type)
		&& (expand_vec_cmp_expr_p (itype, type, ltge)
		    || !expand_vec_cmp_expr_p (TREE_TYPE (@2),
					       type, neeq)))))
    (ltge (convert:itype @0) { build_zero_cst (itype); })))))

/* X + Z < Y + Z is the same as X < Y when there is no overflow.  */
(for op (lt le ge gt)
 (simplify
  (op (plus:c @0 @2) (plus:c @1 @2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (op @0 @1))))

/* As a special case, X + C < Y + C is the same as (signed) X < (signed) Y
   when C is an unsigned integer constant with only the MSB set, and X and
   Y have types of equal or lower integer conversion rank than C's.  */
(for op (lt le ge gt)
 (simplify
  (op (plus @1 INTEGER_CST@0) (plus @2 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_UNSIGNED (TREE_TYPE (@0))
       && wi::only_sign_bit_p (wi::to_wide (@0)))
   (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
    (op (convert:stype @1) (convert:stype @2))))))

/* For equality and subtraction, this is also true with wrapping overflow.  */
(for op (eq ne minus)
 (simplify
  (op (plus:c @0 @2) (plus:c @1 @2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	   || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))))
   (op @0 @1))))
/* And similar for pointers.  */
(for op (eq ne)
 (simplify
  (op (pointer_plus @0 @1) (pointer_plus @0 @2))
  (op @1 @2)))
(simplify
 (pointer_diff (pointer_plus @0 @1) (pointer_plus @0 @2))
 (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
  (convert (minus @1 @2))))
(simplify
 (pointer_diff (pointer_plus @0 @2) (pointer_plus @1 @2))
 (pointer_diff @0 @1))

/* X - Z < Y - Z is the same as X < Y when there is no overflow.  */
(for op (lt le ge gt)
 (simplify
  (op (minus @0 @2) (minus @1 @2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (op @0 @1))))
/* For equality and subtraction, this is also true with wrapping overflow.  */
(for op (eq ne minus)
 (simplify
  (op (minus @0 @2) (minus @1 @2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	   || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))))
   (op @0 @1))))
/* And for pointers...  */
(for op (simple_comparison)
 (simplify
  (op (pointer_diff@3 @0 @2) (pointer_diff @1 @2))
  (if (!TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2)))
   (op @0 @1))))
(simplify
 (minus (pointer_diff@3 @0 @2) (pointer_diff @1 @2))
 (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@3))
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2)))
  (pointer_diff @0 @1)))

/* Z - X < Z - Y is the same as Y < X when there is no overflow.  */
(for op (lt le ge gt)
 (simplify
  (op (minus @2 @0) (minus @2 @1))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (op @1 @0))))
/* For equality and subtraction, this is also true with wrapping overflow.  */
(for op (eq ne minus)
 (simplify
  (op (minus @2 @0) (minus @2 @1))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	   || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))))
   (op @1 @0))))
/* And for pointers...  */
(for op (simple_comparison)
 (simplify
  (op (pointer_diff@3 @2 @0) (pointer_diff @2 @1))
  (if (!TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2)))
   (op @1 @0))))
(simplify
 (minus (pointer_diff@3 @2 @0) (pointer_diff @2 @1))
 (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@3))
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@2)))
  (pointer_diff @1 @0)))

/* X + Y < Y is the same as X < 0 when there is no overflow.  */
(for op (lt le gt ge)
 (simplify
  (op:c (plus:c@2 @0 @1) @1)
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
       && (CONSTANT_CLASS_P (@0) || single_use (@2)))
   (op @0 { build_zero_cst (TREE_TYPE (@0)); }))))
/* For equality, this is also true with wrapping overflow.  */
(for op (eq ne)
 (simplify
  (op:c (nop_convert?@3 (plus:c@2 @0 (convert1? @1))) (convert2? @1))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	   || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
       && (CONSTANT_CLASS_P (@0) || (single_use (@2) && single_use (@3)))
       && tree_nop_conversion_p (TREE_TYPE (@3), TREE_TYPE (@2))
       && tree_nop_conversion_p (TREE_TYPE (@3), TREE_TYPE (@1)))
   (op @0 { build_zero_cst (TREE_TYPE (@0)); })))
 (simplify
  (op:c (nop_convert?@3 (pointer_plus@2 (convert1? @0) @1)) (convert2? @0))
  (if (tree_nop_conversion_p (TREE_TYPE (@2), TREE_TYPE (@0))
       && tree_nop_conversion_p (TREE_TYPE (@3), TREE_TYPE (@0))
       && (CONSTANT_CLASS_P (@1) || (single_use (@2) && single_use (@3))))
   (op @1 { build_zero_cst (TREE_TYPE (@1)); }))))

/* (&a + b) !=/== (&a[1] + c) -> (&a[0] - &a[1]) + b !=/== c */
(for neeq (ne eq)
 (simplify
  (neeq:c (pointer_plus @2 @3) @0)
   (with { poly_int64 diff; tree inner_type = TREE_TYPE (@3);}
    (if (ptr_difference_const (@0, @2, &diff))
     (neeq { build_int_cst_type (inner_type, diff); } @3))))
 (simplify
  (neeq (pointer_plus @0 @1) (pointer_plus @2 @3))
   (with { poly_int64 diff; tree inner_type = TREE_TYPE (@1);}
    (if (ptr_difference_const (@0, @2, &diff))
     (neeq (plus { build_int_cst_type (inner_type, diff); } @1) @3)))))

/* X - Y < X is the same as Y > 0 when there is no overflow.
   For equality, this is also true with wrapping overflow.  */
(for op (simple_comparison)
 (simplify
  (op:c @0 (minus@2 @0 @1))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	   || ((op == EQ_EXPR || op == NE_EXPR)
	       && TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))))
       && (CONSTANT_CLASS_P (@1) || single_use (@2)))
   (op @1 { build_zero_cst (TREE_TYPE (@1)); }))))

/* Transform:
   (X / Y) == 0 -> X < Y if X, Y are non-negative.
   (X / Y) != 0 -> X >= Y, if X, Y are non-negative.
   (X % Y) == X -> X < Y if X, Y are non-negative.
   (X % Y) != X -> X >= Y, if X, Y are non-negative.  */
(for cmp (eq ne)
     ocmp (lt ge)
 (simplify
  (cmp (trunc_div tree_expr_nonnegative_p@0 tree_expr_nonnegative_p@1) integer_zerop)
       /* Complex ==/!= is allowed, but not </>=.  */
  (if (!integer_zerop (@1)
       && (!flag_non_call_exceptions || tree_expr_nonzero_p (@1))
       && TREE_CODE (TREE_TYPE (@0)) != COMPLEX_TYPE
       && (VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (TREE_TYPE (@0))))
   (ocmp @0 @1)))
 (simplify
  (cmp:c (trunc_mod tree_expr_nonnegative_p@0 tree_expr_nonnegative_p@1) @0)
  /* A complex modulo cannot exist, so unlike the division above this
     needs no complex exclusion.  */
  (if (!integer_zerop (@1)
       && (!flag_non_call_exceptions || tree_expr_nonzero_p (@1))
       && (VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (TREE_TYPE (@0))))
   (ocmp @0 @1))))

/* X == C - X can never be true if C is odd.  */
(for cmp (eq ne)
 (simplify
  (cmp:c (convert? @0) (convert1? (minus INTEGER_CST@1 (convert2? @0))))
  (if (TREE_INT_CST_LOW (@1) & 1)
   { constant_boolean_node (cmp == NE_EXPR, type); })))

/*
   U & N <= U  -> true
   U & N >  U  -> false
   U needs to be non-negative.

   U | N <  U  -> false
   U | N >= U  -> true
   U and N needs to be non-negative

   U | N <  U  -> true
   U | N >= U  -> false
   U needs to be non-negative and N needs to be a negative constant.
   */
(for cmp   (lt      ge      le      gt     )
     bitop (bit_ior bit_ior bit_and bit_and)
 (simplify
  (cmp:c (bitop:c tree_expr_nonnegative_p@0 @1) @0)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (if (bitop == BIT_AND_EXPR || tree_expr_nonnegative_p (@1))
    { constant_boolean_node (cmp == GE_EXPR || cmp == LE_EXPR, type); }
    /* The sign is opposite now so the comparison is swapped around. */
    (if (TREE_CODE (@1) == INTEGER_CST && wi::neg_p (wi::to_wide (@1)))
     { constant_boolean_node (cmp == LT_EXPR, type); })))))

/* Arguments on which one can call get_nonzero_bits to get the bits
   possibly set.  with_possible_nonzero_bits_1 is an internal version,
   use with_possible_nonzero_bits.  */
(match with_possible_nonzero_bits_1
 INTEGER_CST@0)
(match with_possible_nonzero_bits_1
 POLY_INT_CST@0)
(match with_possible_nonzero_bits_1
 SSA_NAME@0
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))))
/* Slightly extended version, do not make it recursive to keep it cheap.  */
(match with_possible_nonzero_bits
 with_possible_nonzero_bits_1@0)
#if GENERIC
(match with_possible_nonzero_bits
 (bit_and:c with_possible_nonzero_bits_1@0 @1))
#endif

/* Arguments on which one can call get_known_nonzero_bits to get the
   bits known to be set.  with_known_nonzero_bits_1 is an internal version,
   use with_known_nonzero_bits.  */
(match with_known_nonzero_bits_1
 INTEGER_CST@0)
(match with_known_nonzero_bits_1
 SSA_NAME@0
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))))
/* Slightly extended version, do not make it recursive to keep it cheap.  */
(match with_known_nonzero_bits
 with_known_nonzero_bits_1@0)
#if GENERIC
(match with_known_nonzero_bits
 (bit_ior:c with_known_nonzero_bits_1@0 @1))
#endif

/* X == C (or X & Z == Y | C) is impossible if ~nonzero(X) & C != 0.  */
(for cmp (eq ne)
 (simplify
  (cmp:c with_possible_nonzero_bits@0 with_known_nonzero_bits@1)
  (if (wi::bit_and_not (get_known_nonzero_bits (@1),
			get_nonzero_bits (@0)) != 0)
   { constant_boolean_node (cmp == NE_EXPR, type); })))

/* ((X inner_op C0) outer_op C1)
   With X being a tree where range has reasoned certain bits to always be
   zero throughout its computed value range,
   inner_op = {|,^}, outer_op = {|,^} and inner_op != outer_op
   where zero_mask has 1's for all bits that are sure to be 0 in
   and 0's otherwise.
   if (inner_op == '^') C0 &= ~C1;
   if ((C0 & ~zero_mask) == 0) then emit (X outer_op (C0 outer_op C1)
   if ((C1 & ~zero_mask) == 0) then emit (X inner_op (C0 outer_op C1)
*/
(for inner_op (bit_ior bit_xor)
     outer_op (bit_xor bit_ior)
(simplify
 (outer_op
  (inner_op:s @2 INTEGER_CST@0) INTEGER_CST@1)
 (with
  {
    bool fail = false;
    wide_int zero_mask_not;
    wide_int C0;
    wide_int cst_emit;

    if (TREE_CODE (@2) == SSA_NAME)
      zero_mask_not = get_nonzero_bits (@2);
    else
      fail = true;

    if (inner_op == BIT_XOR_EXPR)
      {
	C0 = wi::bit_and_not (wi::to_wide (@0), wi::to_wide (@1));
	cst_emit = C0 | wi::to_wide (@1);
      }
    else
      {
	C0 = wi::to_wide (@0);
	cst_emit = C0 ^ wi::to_wide (@1);
      }
  }
  (if (!fail && (C0 & zero_mask_not) == 0)
   (outer_op @2 { wide_int_to_tree (type, cst_emit); })
   (if (!fail && (wi::to_wide (@1) & zero_mask_not) == 0)
    (inner_op @2 { wide_int_to_tree (type, cst_emit); }))))))

/* Associate (p +p off1) +p off2 as (p +p (off1 + off2)).  */
(simplify
  (pointer_plus (pointer_plus:s @0 @1) @3)
  (pointer_plus @0 (plus @1 @3)))
#if GENERIC
(simplify
  (pointer_plus (convert:s (pointer_plus:s @0 @1)) @3)
  (pointer_plus (convert:type @0) (plus @1 @3)))
#endif

/* Pattern match
     tem1 = (long) ptr1;
     tem2 = (long) ptr2;
     tem3 = tem2 - tem1;
     tem4 = (unsigned long) tem3;
     tem5 = ptr1 + tem4;
   and produce
     tem5 = ptr2;  */
(simplify
  (pointer_plus @0 (convert?@2 (minus@3 (convert @1) (convert @0))))
  /* Conditionally look through a sign-changing conversion.  */
  (if (TYPE_PRECISION (TREE_TYPE (@2)) == TYPE_PRECISION (TREE_TYPE (@3))
       && ((GIMPLE && useless_type_conversion_p (type, TREE_TYPE (@1)))
	    || (GENERIC && type == TREE_TYPE (@1))))
   @1))
(simplify
  (pointer_plus @0 (convert?@2 (pointer_diff@3 @1 @@0)))
  (if (TYPE_PRECISION (TREE_TYPE (@2)) >= TYPE_PRECISION (TREE_TYPE (@3)))
   (convert @1)))

/* Pattern match
     tem = (sizetype) ptr;
     tem = tem & algn;
     tem = -tem;
     ... = ptr p+ tem;
   and produce the simpler and easier to analyze with respect to alignment
     ... = ptr & ~algn;  */
(simplify
  (pointer_plus @0 (negate (bit_and (convert @0) INTEGER_CST@1)))
  (with { tree algn = wide_int_to_tree (TREE_TYPE (@0), ~wi::to_wide (@1)); }
   (bit_and @0 { algn; })))

/* Also match cases in which a constant is applied:

   (1) tem = (sizetype) ptr;  ---> tem = (sizetype) (ptr + CST);
   (2) tem = -tem             ---> tem = CST - tem;

   and where "& align" masks only trailing zeros of CST.  (1) then has no
   effect, whereas (2) adds CST to the result.  */
(simplify
 (pointer_plus @0 (negate (bit_and (convert (pointer_plus @0 INTEGER_CST@1))
				   INTEGER_CST@2)))
 (if (tree_int_cst_min_precision (@2, UNSIGNED) <= tree_ctz (@1))
  (with { tree algn = wide_int_to_tree (TREE_TYPE (@0), ~wi::to_wide (@2)); }
   (bit_and @0 { algn; }))))
(simplify
 (pointer_plus @0 (minus:s INTEGER_CST@1 (bit_and (convert @0) INTEGER_CST@2)))
 (if (tree_int_cst_min_precision (@2, UNSIGNED) <= tree_ctz (@1))
  (with { tree algn = wide_int_to_tree (TREE_TYPE (@0), ~wi::to_wide (@2)); }
   (pointer_plus (bit_and @0 { algn; }) @1))))
(simplify
 (pointer_plus @0 (minus:s INTEGER_CST@1
			   (bit_and (convert (pointer_plus @0 INTEGER_CST@2))
				    INTEGER_CST@3)))
 (with { auto mask_width = tree_int_cst_min_precision (@3, UNSIGNED); }
  (if (mask_width <= tree_ctz (@1) && mask_width <= tree_ctz (@2))
   (with { tree algn = wide_int_to_tree (TREE_TYPE (@0), ~wi::to_wide (@3)); }
    (pointer_plus (bit_and @0 { algn; }) @1)))))

/* Try folding difference of addresses.  */
(simplify
 (minus (convert @0) (convert (pointer_plus @1 @2)))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && tree_nop_conversion_p (type, TREE_TYPE (@1)))
  (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @1, &diff))
    (minus { build_int_cst_type (type, diff); } (convert @2))))))
(simplify
 (minus (convert (pointer_plus @0 @2)) (convert @1))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && tree_nop_conversion_p (type, TREE_TYPE (@1)))
  (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @1, &diff))
    (plus (convert @2) { build_int_cst_type (type, diff); })))))
(simplify
 (minus (convert @0) (convert @1))
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && tree_nop_conversion_p (type, TREE_TYPE (@1))
      && tree_nop_conversion_p (type, ptr_type_node))
  (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @1, &diff))
    { build_int_cst_type (type, diff); }))))
(simplify
 (pointer_diff (convert?@2 @0) (convert1?@3 @1))
 (if (tree_nop_conversion_p (TREE_TYPE(@2), TREE_TYPE (@0))
      && tree_nop_conversion_p (TREE_TYPE(@3), TREE_TYPE (@1)))
  (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @1, &diff))
    { build_int_cst_type (type, diff); }))))

/* (&a+b) - (&a[1] + c) -> sizeof(a[0]) + (b - c) */
(simplify
 (pointer_diff (pointer_plus @0 @1) (pointer_plus @2 @3))
 (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @2, &diff))
    (plus { build_int_cst_type (type, diff); } (convert (minus @1 @3))))))
/* (p + b) - &p->d -> offsetof (*p, d) + b */
(simplify
 (pointer_diff (pointer_plus @0 @1) @2)
 (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @2, &diff))
    (plus { build_int_cst_type (type, diff); } (convert @1)))))
(simplify
 (pointer_diff @0 (pointer_plus @1 @2))
 (with { poly_int64 diff; }
   (if (ptr_difference_const (@0, @1, &diff))
    (minus { build_int_cst_type (type, diff); } (convert @2)))))

/* Canonicalize (T *)(ptr - ptr-cst) to (T *)ptr + -ptr-cst.  */
(simplify
 (convert (pointer_diff @0 INTEGER_CST@1))
 (if (POINTER_TYPE_P (type))
  (pointer_plus
   (convert @0) { wide_int_to_tree (sizetype,
				    wi::neg (wi::to_wide (@1))); })))

/* If arg0 is derived from the address of an object or function, we may
   be able to fold this expression using the object or function's
   alignment.  */
(simplify
 (bit_and (convert? @0) INTEGER_CST@1)
 (if (POINTER_TYPE_P (TREE_TYPE (@0))
      && tree_nop_conversion_p (type, TREE_TYPE (@0)))
  (with
   {
     unsigned int align;
     unsigned HOST_WIDE_INT bitpos;
     get_pointer_alignment_1 (@0, &align, &bitpos);
   }
   (if (wi::ltu_p (wi::to_wide (@1), align / BITS_PER_UNIT))
    { wide_int_to_tree (type, (wi::to_wide (@1)
			       & (bitpos / BITS_PER_UNIT))); }))))

(match min_value
 uniform_integer_cst_p
 (with {
   tree int_cst = uniform_integer_cst_p (t);
   tree inner_type = TREE_TYPE (int_cst);
  }
  (if ((INTEGRAL_TYPE_P (inner_type)
        || POINTER_TYPE_P (inner_type))
       && wi::eq_p (wi::to_wide (int_cst), wi::min_value (inner_type))))))

(match max_value
 uniform_integer_cst_p
 (with {
   tree int_cst = uniform_integer_cst_p (t);
   tree itype = TREE_TYPE (int_cst);
  }
 (if ((INTEGRAL_TYPE_P (itype)
       || POINTER_TYPE_P (itype))
      && wi::eq_p (wi::to_wide (int_cst), wi::max_value (itype))))))

/* x >  y  &&  x != XXX_MIN  -->  x > y
   x >  y  &&  x == XXX_MIN  -->  false . */
(for eqne (eq ne)
 (simplify
  (bit_and:c (gt:c@2 @0 @1) (eqne @0 min_value))
   (switch
    (if (eqne == EQ_EXPR)
     { constant_boolean_node (false, type); })
    (if (eqne == NE_EXPR)
     @2)
    )))

/* x <  y  &&  x != XXX_MAX  -->  x < y
   x <  y  &&  x == XXX_MAX  -->  false.  */
(for eqne (eq ne)
 (simplify
  (bit_and:c (lt:c@2 @0 @1) (eqne @0 max_value))
   (switch
    (if (eqne == EQ_EXPR)
     { constant_boolean_node (false, type); })
    (if (eqne == NE_EXPR)
     @2)
    )))

/* x <=  y  &&  x == XXX_MIN  -->  x == XXX_MIN.  */
(simplify
 (bit_and:c (le:c @0 @1) (eq@2 @0 min_value))
  @2)

/* x >=  y  &&  x == XXX_MAX  -->  x == XXX_MAX.  */
(simplify
 (bit_and:c (ge:c @0 @1) (eq@2 @0 max_value))
  @2)

/* x >  y  ||  x != XXX_MIN   -->  x != XXX_MIN.  */
(simplify
 (bit_ior:c (gt:c @0 @1) (ne@2 @0 min_value))
  @2)

/* x <=  y  ||  x != XXX_MIN   -->  true.  */
(simplify
 (bit_ior:c (le:c @0 @1) (ne @0 min_value))
  { constant_boolean_node (true, type); })

/* x <=  y  ||  x == XXX_MIN   -->  x <= y.  */
(simplify
 (bit_ior:c (le:c@2 @0 @1) (eq @0 min_value))
  @2)

/* x <  y  ||  x != XXX_MAX   -->  x != XXX_MAX.  */
(simplify
 (bit_ior:c (lt:c @0 @1) (ne@2 @0 max_value))
  @2)

/* x >=  y  ||  x != XXX_MAX   -->  true
   x >=  y  ||  x == XXX_MAX   -->  x >= y.  */
(for eqne (eq ne)
 (simplify
  (bit_ior:c (ge:c@2 @0 @1) (eqne @0 max_value))
   (switch
    (if (eqne == EQ_EXPR)
     @2)
    (if (eqne == NE_EXPR)
     { constant_boolean_node (true, type); }))))

/* y == XXX_MIN || x < y --> x <= y - 1 */
(simplify
 (bit_ior:c (eq:s @1 min_value) (lt:cs @0 @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
  (le @0 (minus @1 { build_int_cst (TREE_TYPE (@1), 1); }))))

/* y != XXX_MIN && x >= y --> x > y - 1 */
(simplify
 (bit_and:c (ne:s @1 min_value) (ge:cs @0 @1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
  (gt @0 (minus @1 { build_int_cst (TREE_TYPE (@1), 1); }))))

/* Optimize (a CMP b) &| (a CMP b)
   using the helper combine_comparisons function.  */
(for bitop (bit_and bit_ior bit_xor simple_comparison)
 (for cmp1 (tcc_comparison)
  (for cmp2 (tcc_comparison)
   (simplify
    (bitop (cmp1@2 @0 @1) (cmp2 @0 @1))
    (with rescmpcode (tcc_comparison)
      {
        tree_code rescmpcode;
        tree res;
        bool honor_nans = HONOR_NANS (@0);
        rescmpcode = combine_comparisons (bitop, cmp1, cmp2,
					  type, TREE_TYPE (@2),
					  honor_nans, &res);
      }
     (if (rescmpcode == INTEGER_CST)
      { res; }
      (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
       (rescmpcode @0 @1))))))))

/* Convert (X == CST1) && ((other)X OP2 CST2) to a known value
   based on CST1 OP2 CST2.  Similarly for (X != CST1).  */
/* Convert (X == Y) && (X OP2 Y) to a known value if X is an integral type.
   Similarly for (X != Y).  */

(for code1 (eq ne)
 (for code2 (eq ne lt gt le ge)
  (simplify
   (bit_and:c (code1:c@3 @0 @1) (code2:c@4 (convert?@c0 @0) @2))
   (if ((TREE_CODE (@1) == INTEGER_CST
	 && TREE_CODE (@2) == INTEGER_CST)
	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
   || ((VECTOR_INTEGER_TYPE_P (TREE_TYPE (@1))
	|| VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (@1)))
   && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
	     || POINTER_TYPE_P (TREE_TYPE (@1)))
	    && bitwise_equal_p (@1, @2)))
    (with
     {
      bool one_before = false;
      bool one_after = false;
      int cmp = 0;
      bool allbits = true;
      if (TREE_CODE (@1) == INTEGER_CST
	  && TREE_CODE (@2) == INTEGER_CST)
	{
	  allbits = TYPE_PRECISION (TREE_TYPE (@1)) <= TYPE_PRECISION (TREE_TYPE (@2));
	  auto t1 = wi::to_wide (fold_convert (TREE_TYPE (@2), @1));
	  auto t2 = wi::to_wide (@2);
	  cmp = wi::cmp (t1, t2, TYPE_SIGN (TREE_TYPE (@2)));
	  if (cmp < 0
	      && t1 == t2 - 1)
	    one_before = true;
	  if (cmp > 0
	      && t1 == t2 + 1)
	    one_after = true;
	}
      bool val;
      switch (code2)
	 {
	case EQ_EXPR: val = (cmp == 0); break;
	case NE_EXPR: val = (cmp != 0); break;
	case LT_EXPR: val = (cmp < 0); break;
	case GT_EXPR: val = (cmp > 0); break;
	case LE_EXPR: val = (cmp <= 0); break;
	case GE_EXPR: val = (cmp >= 0); break;
	default: gcc_unreachable ();
	}
     }
     (switch
      (if (code1 == EQ_EXPR && val) @3)
      (if (code1 == EQ_EXPR && !val) { constant_boolean_node (false, type); })
      (if (code1 == NE_EXPR && !val && allbits) @4)
      (if ((code1 == NE_EXPR
           && code2 == GE_EXPR
	   && cmp == 0
	   && allbits)
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (gt @c0 (convert @1)))
      (if ((code1 == NE_EXPR
           && code2 == LE_EXPR
	   && cmp == 0
	   && allbits)
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (lt @c0 (convert @1)))
      /* (a != (b+1)) & (a > b) -> a > (b+1) */
      (if ((code1 == NE_EXPR
           && code2 == GT_EXPR
	   && one_after
	   && allbits)
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (gt @c0 (convert @1)))
      /* (a != (b-1)) & (a < b) -> a < (b-1) */
      (if ((code1 == NE_EXPR
           && code2 == LT_EXPR
	   && one_before
	   && allbits)
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (lt @c0 (convert @1)))
     )
    )
   )
  )
 )
)

/* Convert (X OP1 CST1) && (X OP2 CST2).
   Convert (X OP1 Y) && (X OP2 Y).  */

(for code1 (lt le gt ge)
 (for code2 (lt le gt ge)
  (simplify
  (bit_and (code1:c@3 @0 @1) (code2:c@4 @0 @2))
  (if ((TREE_CODE (@1) == INTEGER_CST
	&& TREE_CODE (@2) == INTEGER_CST)
       || ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
       || ((VECTOR_INTEGER_TYPE_P (TREE_TYPE (@1))
	    || VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (@1)))
       && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
	    || POINTER_TYPE_P (TREE_TYPE (@1)))
	   && operand_equal_p (@1, @2)))
   (with
    {
     int cmp = 0;
     if (TREE_CODE (@1) == INTEGER_CST
	 && TREE_CODE (@2) == INTEGER_CST)
       cmp = tree_int_cst_compare (@1, @2);
    }
    (switch
     /* Choose the more restrictive of two < or <= comparisons.  */
     (if ((code1 == LT_EXPR || code1 == LE_EXPR)
	  && (code2 == LT_EXPR || code2 == LE_EXPR))
      (if ((cmp < 0) || (cmp == 0 && code1 == LT_EXPR))
       @3
       @4))
     /* Likewise chose the more restrictive of two > or >= comparisons.  */
     (if ((code1 == GT_EXPR || code1 == GE_EXPR)
	  && (code2 == GT_EXPR || code2 == GE_EXPR))
      (if ((cmp > 0) || (cmp == 0 && code1 == GT_EXPR))
       @3
       @4))
     /* Check for singleton ranges.  */
     (if (cmp == 0
	  && ((code1 == LE_EXPR && code2 == GE_EXPR)
	    || (code1 == GE_EXPR && code2 == LE_EXPR)))
      (eq @0 @1))
     /* Check for disjoint ranges.  */
     (if (cmp <= 0
	  && (code1 == LT_EXPR || code1 == LE_EXPR)
	  && (code2 == GT_EXPR || code2 == GE_EXPR))
      { constant_boolean_node (false, type); })
     (if (cmp >= 0
	  && (code1 == GT_EXPR || code1 == GE_EXPR)
	  && (code2 == LT_EXPR || code2 == LE_EXPR))
      { constant_boolean_node (false, type); })
     ))))))

/* Convert (X == CST1) || (X OP2 CST2) to a known value
   based on CST1 OP2 CST2.  Similarly for (X != CST1).  */
/* Convert (X == Y) || (X OP2 Y) to a known value if X is an integral type.
   Similarly for (X != Y).  */

(for code1 (eq ne)
 (for code2 (eq ne lt gt le ge)
  (simplify
   (bit_ior:c (code1:c@3 @0 @1) (code2:c@4 (convert?@c0 @0) @2))
   (if ((TREE_CODE (@1) == INTEGER_CST
	 && TREE_CODE (@2) == INTEGER_CST)
	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
   || (VECTOR_TYPE_P (TREE_TYPE (@1)))
	    || POINTER_TYPE_P (TREE_TYPE (@1)))
	    && bitwise_equal_p (@1, @2)))
    (with
     {
      bool one_before = false;
      bool one_after = false;
      int cmp = 0;
      bool allbits = true;
      if (TREE_CODE (@1) == INTEGER_CST
	  && TREE_CODE (@2) == INTEGER_CST)
	{
	  allbits = TYPE_PRECISION (TREE_TYPE (@1)) <= TYPE_PRECISION (TREE_TYPE (@2));
	  auto t1 = wi::to_wide (fold_convert (TREE_TYPE (@2), @1));
	  auto t2 = wi::to_wide (@2);
	  cmp = wi::cmp (t1, t2, TYPE_SIGN (TREE_TYPE (@2)));
	  if (cmp < 0
	      && t1 == t2 - 1)
	    one_before = true;
	  if (cmp > 0
	      && t1 == t2 + 1)
	    one_after = true;
	}
      bool val;
      switch (code2)
	{
	case EQ_EXPR: val = (cmp == 0); break;
	case NE_EXPR: val = (cmp != 0); break;
	case LT_EXPR: val = (cmp < 0); break;
	case GT_EXPR: val = (cmp > 0); break;
	case LE_EXPR: val = (cmp <= 0); break;
	case GE_EXPR: val = (cmp >= 0); break;
	default: gcc_unreachable ();
	}
     }
     (switch
      (if (code1 == EQ_EXPR && val) @4)
      (if (code1 == NE_EXPR && val && allbits) { constant_boolean_node (true, type); })
      (if (code1 == NE_EXPR && !val && allbits) @3)
      (if (code1 == EQ_EXPR
           && code2 == GT_EXPR
	   && cmp == 0
	   && allbits
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (ge @c0 @2))
      (if (code1 == EQ_EXPR
           && code2 == LT_EXPR
	   && cmp == 0
	   && allbits
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (le @c0 @2))
      /* (a == (b-1)) | (a >= b) -> a >= (b-1) */
      (if (code1 == EQ_EXPR
           && code2 == GE_EXPR
	   && one_before
	   && allbits
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (ge @c0 (convert @1)))
      /* (a == (b+1)) | (a <= b) -> a <= (b-1) */
      (if (code1 == EQ_EXPR
           && code2 == LE_EXPR
	   && one_after
	   && allbits
      && ((VECTOR_BOOLEAN_TYPE_P (type)
      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
       (le @c0 (convert @1)))
     )
    )
   )
  )
 )
)

/* Convert (X OP1 CST1) || (X OP2 CST2).
   Convert (X OP1 Y)    || (X OP2 Y).  */

(for code1 (lt le gt ge)
 (for code2 (lt le gt ge)
  (simplify
  (bit_ior (code1@3 @0 @1) (code2@4 @0 @2))
  (if ((TREE_CODE (@1) == INTEGER_CST
        && TREE_CODE (@2) == INTEGER_CST)
       || ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
	    || POINTER_TYPE_P (TREE_TYPE (@1)))
	   && operand_equal_p (@1, @2)))
   (with
    {
     int cmp = 0;
     if (TREE_CODE (@1) == INTEGER_CST
	 && TREE_CODE (@2) == INTEGER_CST)
       cmp = tree_int_cst_compare (@1, @2);
    }
    (switch
     /* Choose the more restrictive of two < or <= comparisons.  */
     (if ((code1 == LT_EXPR || code1 == LE_EXPR)
	  && (code2 == LT_EXPR || code2 == LE_EXPR))
      (if ((cmp < 0) || (cmp == 0 && code1 == LT_EXPR))
       @4
       @3))
     /* Likewise chose the more restrictive of two > or >= comparisons.  */
     (if ((code1 == GT_EXPR || code1 == GE_EXPR)
	  && (code2 == GT_EXPR || code2 == GE_EXPR))
      (if ((cmp > 0) || (cmp == 0 && code1 == GT_EXPR))
       @4
       @3))
     /* Check for singleton ranges.  */
     (if (cmp == 0
	  && ((code1 == LT_EXPR && code2 == GT_EXPR)
	      || (code1 == GT_EXPR && code2 == LT_EXPR)))
      (ne @0 @2))
     /* Check for disjoint ranges.  */
     (if (cmp >= 0
	  && (code1 == LT_EXPR || code1 == LE_EXPR)
	  && (code2 == GT_EXPR || code2 == GE_EXPR))
      { constant_boolean_node (true, type); })
     (if (cmp <= 0
	  && (code1 == GT_EXPR || code1 == GE_EXPR)
	  && (code2 == LT_EXPR || code2 == LE_EXPR))
      { constant_boolean_node (true, type); })
     ))))))

/* (type)([0,1]@a != 0) -> (type)a
   (type)([0,1]@a == 1) -> (type)a
   (type)([0,1]@a == 0) -> a ^ 1
   (type)([0,1]@a != 1) -> a ^ 1.  */
(for eqne (eq ne)
 (simplify
  (convert (eqne zero_one_valued_p@0 INTEGER_CST@1))
  (if ((integer_zerop (@1) || integer_onep (@1)))
   (if ((eqne == EQ_EXPR) ^ integer_zerop (@1))
    (convert @0)
    /* Only do this if the types match as (type)(a == 0) is
       canonical form normally, while `a ^ 1` is canonical when
       there is no type change. */
    (if (types_match (type, TREE_TYPE (@0)))
     (bit_xor @0 { build_one_cst (type); } ))))))

/* For zero_one_valued operands:
   (a == 0) != b -> a == b.  */
(simplify
 (ne:c (convert (eq zero_one_valued_p@0 integer_zerop))
       (convert zero_one_valued_p@1))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
  (eq @0 @1)))

/* For zero_one_valued operands:
   (a op1 0) cmp (b op2 0) -> a out_cmp b
   where out_cmp is cmp if op1 == op2, and the inverse of cmp otherwise.  */
(if (INTEGRAL_TYPE_P (type))
  (for cmp     (ne ne eq eq ne ne eq eq)
       op1     (eq ne eq ne eq ne eq ne)
       op2     (eq ne eq ne ne eq ne eq)
       out_cmp (ne ne eq eq eq eq ne ne)
    (simplify
      (cmp:c (op1 zero_one_valued_p@0 integer_zerop)
             (op2 zero_one_valued_p@1 integer_zerop))
      (out_cmp (convert:type @0) (convert:type @1)))))

/* ((a ^ b) & c) cmp d || a != b --> (0 cmp d || a != b). */
/* ((a ^ b) & c) cmp d && a == b --> (0 cmp d && a == b). */
(for bitop (bit_ior bit_and)
     neeq  (ne      eq)
 (for cmp  (simple_comparison)
  (simplify
    (bitop:c
      (cmp:c
	(bit_and:c
	  (bit_xor @0 @1)
	  tree_expr_nonzero_p@2)
	@3)
      (neeq@4 @0 @1))
    (bitop
      (cmp
	{ build_zero_cst (TREE_TYPE (@0)); }
	@3)
      @4))
/* (a ^ b) cmp c || a != b --> (0 cmp c || a != b). */
/* (a ^ b) cmp c && a == b --> (0 cmp c || a == b). */
  (simplify
    (bitop:c
      (cmp:c
	(bit_xor @0 @1)
	@2)
      (neeq@3 @0 @1))
    (bitop
      (cmp
	{ build_zero_cst (TREE_TYPE (@0)); } @2)
      @3))))

/* We can't reassociate at all for saturating types.  */
(if (!TYPE_SATURATING (type))

 /* Contract negates.  */
 /* A + (-B) -> A - B */
 (simplify
  (plus:c @0 (convert? (negate @1)))
  /* Apply STRIP_NOPS on the negate.  */
  (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with
    {
     tree t1 = type;
     if (INTEGRAL_TYPE_P (type)
	 && TYPE_OVERFLOW_WRAPS (type) != TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
       t1 = TYPE_OVERFLOW_WRAPS (type) ? type : TREE_TYPE (@1);
    }
    (convert (minus (convert:t1 @0) (convert:t1 @1))))))
 /* A - (-B) -> A + B */
 (simplify
  (minus @0 (convert? (negate @1)))
  (if (tree_nop_conversion_p (type, TREE_TYPE (@1))
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with
    {
     tree t1 = type;
     if (INTEGRAL_TYPE_P (type)
	 && TYPE_OVERFLOW_WRAPS (type) != TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
       t1 = TYPE_OVERFLOW_WRAPS (type) ? type : TREE_TYPE (@1);
    }
    (convert (plus (convert:t1 @0) (convert:t1 @1))))))
 /* -(T)(-A) -> (T)A
    Sign-extension is ok except for INT_MIN, which thankfully cannot
    happen without overflow.  */
 (simplify
  (negate (convert (negate @1)))
  (if (INTEGRAL_TYPE_P (type)
       && (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@1))
	   || (!TYPE_UNSIGNED (TREE_TYPE (@1))
	       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1))))
       && !TYPE_OVERFLOW_SANITIZED (type)
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@1)))
   (convert @1)))
 (simplify
  (negate (convert negate_expr_p@1))
  (if (SCALAR_FLOAT_TYPE_P (type)
       && ((DECIMAL_FLOAT_TYPE_P (type)
	    == DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@1))
	    && TYPE_PRECISION (type) >= TYPE_PRECISION (TREE_TYPE (@1)))
	   || !HONOR_SIGN_DEPENDENT_ROUNDING (type)))
   (convert (negate @1))))
 (simplify
  (negate (nop_convert? (negate @1)))
  (if (!TYPE_OVERFLOW_SANITIZED (type)
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@1)))
   (non_lvalue (view_convert @1))))

 /* We can't reassociate floating-point unless -fassociative-math
    or fixed-point plus or minus because of saturation to +-Inf.  */
 (if ((!FLOAT_TYPE_P (type) || flag_associative_math)
      && !FIXED_POINT_TYPE_P (type))

  /* Match patterns that allow contracting a plus-minus pair
     irrespective of overflow issues.  */
  /* (A +- B) - A       ->  +- B */
  /* (A +- B) -+ B      ->  A */
  /* A - (A +- B)       -> -+ B */
  /* A +- (B -+ A)      ->  +- B */
  (simplify
   (minus (nop_convert1? (plus:c (nop_convert2? @0) @1)) @0)
   (view_convert @1))
  (simplify
   (minus (nop_convert1? (minus (nop_convert2? @0) @1)) @0)
   (if (!ANY_INTEGRAL_TYPE_P (type)
	|| TYPE_OVERFLOW_WRAPS (type))
   (negate (view_convert @1))
   (view_convert (negate @1))))
  (simplify
   (plus:c (nop_convert1? (minus @0 (nop_convert2? @1))) @1)
   (view_convert @0))
  (simplify
   (minus @0 (nop_convert1? (plus:c (nop_convert2? @0) @1)))
    (if (!ANY_INTEGRAL_TYPE_P (type)
	 || TYPE_OVERFLOW_WRAPS (type))
     (negate (view_convert @1))
     (view_convert (negate @1))))
  (simplify
   (minus @0 (nop_convert1? (minus (nop_convert2? @0) @1)))
   (view_convert @1))
  /* (A +- B) + (C - A)   -> C +- B */
  /* (A +  B) - (A - C)   -> B + C */
  /* More cases are handled with comparisons.  */
  (simplify
   (plus:c (plus:c @0 @1) (minus @2 @0))
   (plus @2 @1))
  (simplify
   (plus:c (minus @0 @1) (minus @2 @0))
   (minus @2 @1))
  (simplify
   (plus:c (pointer_diff @0 @1) (pointer_diff @2 @0))
   (if (TYPE_OVERFLOW_UNDEFINED (type)
	&& !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))
    (pointer_diff @2 @1)))
  (simplify
   (minus (plus:c @0 @1) (minus @0 @2))
   (plus @1 @2))

  /* (A +- CST1) +- CST2 -> A + CST3
     Use view_convert because it is safe for vectors and equivalent for
     scalars.  */
  (for outer_op (plus minus)
   (for inner_op (plus minus)
	neg_inner_op (minus plus)
    (simplify
     (outer_op (nop_convert? (inner_op @0 CONSTANT_CLASS_P@1))
	       CONSTANT_CLASS_P@2)
     /* If one of the types wraps, use that one.  */
     (if (!ANY_INTEGRAL_TYPE_P (type) || TYPE_OVERFLOW_WRAPS (type))
      /* If all 3 captures are CONSTANT_CLASS_P, punt, as we might recurse
	 forever if something doesn't simplify into a constant.  */
      (if (!CONSTANT_CLASS_P (@0))
       (if (outer_op == PLUS_EXPR)
	(plus (view_convert @0) (inner_op! @2 (view_convert @1)))
	(minus (view_convert @0) (neg_inner_op! @2 (view_convert @1)))))
      (if (!ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
       (if (outer_op == PLUS_EXPR)
	(view_convert (plus @0 (inner_op! (view_convert @2) @1)))
	(view_convert (minus @0 (neg_inner_op! (view_convert @2) @1))))
       /* If the constant operation overflows we cannot do the transform
	  directly as we would introduce undefined overflow, for example
	  with (a - 1) + INT_MIN.  */
       (if (types_match (type, @0) && !TYPE_OVERFLOW_SANITIZED (type))
	(with { tree cst = const_binop (outer_op == inner_op
					? PLUS_EXPR : MINUS_EXPR,
					type, @1, @2); }
	 (if (cst)
	  (if (INTEGRAL_TYPE_P (type) && !TREE_OVERFLOW (cst))
	   (inner_op @0 { cst; } )
	   /* X+INT_MAX+1 is X-INT_MIN.  */
	   (if (INTEGRAL_TYPE_P (type)
	        && wi::to_wide (cst) == wi::min_value (type))
	    (neg_inner_op @0 { wide_int_to_tree (type, wi::to_wide (cst)); })
	    /* Last resort, use some unsigned type.  */
	    (with { tree utype = unsigned_type_for (type); }
	     (if (utype)
	      (view_convert (inner_op
			     (view_convert:utype @0)
			     (view_convert:utype
			      { TREE_OVERFLOW (cst)
			        ? drop_tree_overflow (cst) : cst; })))))))))))))))

  /* (CST1 - A) +- CST2 -> CST3 - A  */
  (for outer_op (plus minus)
   (simplify
    (outer_op (nop_convert? (minus CONSTANT_CLASS_P@1 @0)) CONSTANT_CLASS_P@2)
    /* If one of the types wraps, use that one.  */
    (if (!ANY_INTEGRAL_TYPE_P (type) || TYPE_OVERFLOW_WRAPS (type))
     /* If all 3 captures are CONSTANT_CLASS_P, punt, as we might recurse
	forever if something doesn't simplify into a constant.  */
     (if (!CONSTANT_CLASS_P (@0))
      (minus (outer_op! (view_convert @1) @2) (view_convert @0)))
     (if (!ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	  || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
      (view_convert (minus (outer_op! @1 (view_convert @2)) @0))
      (if (types_match (type, @0) && !TYPE_OVERFLOW_SANITIZED (type))
       (with { tree cst = const_binop (outer_op, type, @1, @2); }
	(if (cst && !TREE_OVERFLOW (cst))
	 (minus { cst; } @0))))))))

  /* CST1 - (CST2 - A) -> CST3 + A
     Use view_convert because it is safe for vectors and equivalent for
     scalars.  */
  (simplify
   (minus CONSTANT_CLASS_P@1 (nop_convert? (minus CONSTANT_CLASS_P@2 @0)))
   /* If one of the types wraps, use that one.  */
   (if (!ANY_INTEGRAL_TYPE_P (type) || TYPE_OVERFLOW_WRAPS (type))
    /* If all 3 captures are CONSTANT_CLASS_P, punt, as we might recurse
      forever if something doesn't simplify into a constant.  */
    (if (!CONSTANT_CLASS_P (@0))
     (plus (view_convert @0) (minus! @1 (view_convert @2))))
    (if (!ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	 || TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
     (view_convert (plus @0 (minus! (view_convert @1) @2)))
     (if (types_match (type, @0) && !TYPE_OVERFLOW_SANITIZED (type))
      (with { tree cst = const_binop (MINUS_EXPR, type, @1, @2); }
       (if (cst && !TREE_OVERFLOW (cst))
	(plus { cst; } @0)))))))

/* ((T)(A)) + CST -> (T)(A + CST)  */
#if GIMPLE
  (simplify
   (plus (convert:s@2 SSA_NAME@0) INTEGER_CST@1)
    (if (TREE_CODE (TREE_TYPE (@0)) == INTEGER_TYPE
         && TREE_CODE (type) == INTEGER_TYPE
         && TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (@0))
         && int_fits_type_p (@1, TREE_TYPE (@0)))
     /* Perform binary operation inside the cast if the constant fits
        and (A + CST)'s range does not overflow.  */
     (with
      {
	wi::overflow_type min_ovf = wi::OVF_OVERFLOW,
			  max_ovf = wi::OVF_OVERFLOW;
        tree inner_type = TREE_TYPE (@0);

	wide_int w1
	  = wide_int::from (wi::to_wide (@1), TYPE_PRECISION (inner_type),
			    TYPE_SIGN (inner_type));

	int_range_max vr;
	if (gimple_match_range_of_expr (vr, @0, @2) && !vr.varying_p ())
          {
	    wide_int wmin0 = vr.lower_bound ();
	    wide_int wmax0 = vr.upper_bound ();
            wi::add (wmin0, w1, TYPE_SIGN (inner_type), &min_ovf);
            wi::add (wmax0, w1, TYPE_SIGN (inner_type), &max_ovf);
          }
      }
     (if (min_ovf == wi::OVF_NONE && max_ovf == wi::OVF_NONE)
      (convert (plus @0 { wide_int_to_tree (TREE_TYPE (@0), w1); } )))
     )))
#endif

/* ((T)(A + CST1)) + CST2 -> (T)(A) + (T)CST1 + CST2  */
#if GIMPLE
  (for op (plus minus)
   (simplify
    (plus (convert:s (op:s @0 INTEGER_CST@1)) INTEGER_CST@2)
     (if (TREE_CODE (TREE_TYPE (@0)) == INTEGER_TYPE
	  && TREE_CODE (type) == INTEGER_TYPE
	  && TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (@0))
	  && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	  && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
	  && TYPE_OVERFLOW_WRAPS (type))
       (plus (convert @0) (op @2 (convert @1))))))
#endif

/* (T)(A) +- (T)(B) -> (T)(A +- B) only when (A +- B) could be simplified
   to a simple value.  */
  (for op (plus minus)
   (simplify
    (op (convert @0) (convert @1))
     (if (INTEGRAL_TYPE_P (type)
	  && INTEGRAL_TYPE_P (TREE_TYPE (@0))
	  && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))
	  && types_match (TREE_TYPE (@0), TREE_TYPE (@1))
	  && !TYPE_OVERFLOW_TRAPS (type)
	  && !TYPE_OVERFLOW_SANITIZED (type))
      (convert (op! @0 @1)))))

  /* ~A + A -> -1 */
  (simplify
   (plus:c (convert? (bit_not @0)) (convert? @0))
   (if (!TYPE_OVERFLOW_TRAPS (type))
    (convert { build_all_ones_cst (TREE_TYPE (@0)); })))

  /* ~A + 1 -> -A */
  (simplify
   (plus (convert? (bit_not @0)) integer_each_onep)
   (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
    (negate (convert @0))))

  /* -A - 1 -> ~A */
  (simplify
   (minus (convert? (negate @0)) integer_each_onep)
   (if (!TYPE_OVERFLOW_TRAPS (type)
	&& TREE_CODE (type) != COMPLEX_TYPE
	&& tree_nop_conversion_p (type, TREE_TYPE (@0)))
    (bit_not (convert @0))))

  /* -1 - A -> ~A */
  (simplify
   (minus integer_all_onesp @0)
   (if (TREE_CODE (type) != COMPLEX_TYPE)
    (bit_not @0)))

  /* (T)(P + A) - (T)P -> (T) A */
  (simplify
   (minus (convert (plus:c @@0 @1))
    (convert? @0))
   (if (element_precision (type) <= element_precision (TREE_TYPE (@1))
	/* For integer types, if A has a smaller type
	   than T the result depends on the possible
	   overflow in P + A.
	   E.g. T=size_t, A=(unsigned)429497295, P>0.
	   However, if an overflow in P + A would cause
	   undefined behavior, we can assume that there
	   is no overflow.  */
	|| (INTEGRAL_TYPE_P (TREE_TYPE (@1))
	    && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1))))
    (convert @1)))
  (simplify
   (minus (convert (pointer_plus @@0 @1))
    (convert @0))
   (if (element_precision (type) <= element_precision (TREE_TYPE (@1))
	/* For pointer types, if the conversion of A to the
	   final type requires a sign- or zero-extension,
	   then we have to punt - it is not defined which
	   one is correct.  */
	|| (POINTER_TYPE_P (TREE_TYPE (@0))
	    && TREE_CODE (@1) == INTEGER_CST
	    && tree_int_cst_sign_bit (@1) == 0))
    (convert @1)))
   (simplify
    (pointer_diff (pointer_plus @@0 @1) @0)
    /* The second argument of pointer_plus must be interpreted as signed, and
       thus sign-extended if necessary.  */
    (with { tree stype = signed_type_for (TREE_TYPE (@1)); }
     /* Use view_convert instead of convert here, as POINTER_PLUS_EXPR
	second arg is unsigned even when we need to consider it as signed,
	we don't want to diagnose overflow here.  */
     (convert (view_convert:stype @1))))

  /* (T)P - (T)(P + A) -> -(T) A */
  (simplify
   (minus (convert? @0)
    (convert (plus:c @@0 @1)))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	/* For integer literals, using an intermediate unsigned type to avoid
	   an overflow at run time is counter-productive because it introduces
	   spurious overflows at compile time, in the form of TREE_OVERFLOW on
	   the result, which may be problematic in GENERIC for some front-ends:
	     (T)P - (T)(P + 4) -> (T)(-(U)4) -> (T)(4294967292) -> -4(OVF)
	   so we use the direct path for them.  */
	&& TREE_CODE (@1) != INTEGER_CST
	&& element_precision (type) <= element_precision (TREE_TYPE (@1)))
    (with { tree utype = unsigned_type_for (type); }
     (convert (negate (convert:utype @1))))
    (if (element_precision (type) <= element_precision (TREE_TYPE (@1))
	 /* For integer types, if A has a smaller type
	    than T the result depends on the possible
	    overflow in P + A.
	    E.g. T=size_t, A=(unsigned)429497295, P>0.
	    However, if an overflow in P + A would cause
	    undefined behavior, we can assume that there
	    is no overflow.  */
	 || (INTEGRAL_TYPE_P (TREE_TYPE (@1))
	     && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1))))
     (negate (convert @1)))))
  (simplify
   (minus (convert @0)
    (convert (pointer_plus @@0 @1)))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	/* See above the rationale for this condition.  */
	&& TREE_CODE (@1) != INTEGER_CST
	&& element_precision (type) <= element_precision (TREE_TYPE (@1)))
    (with { tree utype = unsigned_type_for (type); }
     (convert (negate (convert:utype @1))))
    (if (element_precision (type) <= element_precision (TREE_TYPE (@1))
	 /* For pointer types, if the conversion of A to the
	    final type requires a sign- or zero-extension,
	    then we have to punt - it is not defined which
	    one is correct.  */
	 || (POINTER_TYPE_P (TREE_TYPE (@0))
	     && TREE_CODE (@1) == INTEGER_CST
	     && tree_int_cst_sign_bit (@1) == 0))
     (negate (convert @1)))))
   (simplify
    (pointer_diff @0 (pointer_plus @@0 @1))
    /* The second argument of pointer_plus must be interpreted as signed, and
       thus sign-extended if necessary.  */
    (with { tree stype = signed_type_for (TREE_TYPE (@1)); }
     /* Use view_convert instead of convert here, as POINTER_PLUS_EXPR
	second arg is unsigned even when we need to consider it as signed,
	we don't want to diagnose overflow here.  */
     (negate (convert (view_convert:stype @1)))))

  /* (T)(P + A) - (T)(P + B) -> (T)A - (T)B */
  (simplify
   (minus (convert (plus:c @@0 @1))
    (convert (plus:c @0 @2)))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& element_precision (type) <= element_precision (TREE_TYPE (@1))
	&& element_precision (type) <= element_precision (TREE_TYPE (@2)))
    (with { tree utype = unsigned_type_for (type); }
     (convert (minus (convert:utype @1) (convert:utype @2))))
    (if (((element_precision (type) <= element_precision (TREE_TYPE (@1)))
	  == (element_precision (type) <= element_precision (TREE_TYPE (@2))))
	 && (element_precision (type) <= element_precision (TREE_TYPE (@1))
	     /* For integer types, if A has a smaller type
		than T the result depends on the possible
		overflow in P + A.
		E.g. T=size_t, A=(unsigned)429497295, P>0.
		However, if an overflow in P + A would cause
		undefined behavior, we can assume that there
		is no overflow.  */
	     || (INTEGRAL_TYPE_P (TREE_TYPE (@1))
		 && INTEGRAL_TYPE_P (TREE_TYPE (@2))
		 && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1))
		 && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@2)))))
     (minus (convert @1) (convert @2)))))
  (simplify
   (minus (convert (pointer_plus @@0 @1))
    (convert (pointer_plus @0 @2)))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& element_precision (type) <= element_precision (TREE_TYPE (@1)))
    (with { tree utype = unsigned_type_for (type); }
     (convert (minus (convert:utype @1) (convert:utype @2))))
    (if (element_precision (type) <= element_precision (TREE_TYPE (@1))
	 /* For pointer types, if the conversion of A to the
	    final type requires a sign- or zero-extension,
	    then we have to punt - it is not defined which
	    one is correct.  */
	 || (POINTER_TYPE_P (TREE_TYPE (@0))
	     && TREE_CODE (@1) == INTEGER_CST
	     && tree_int_cst_sign_bit (@1) == 0
	     && TREE_CODE (@2) == INTEGER_CST
	     && tree_int_cst_sign_bit (@2) == 0))
     (minus (convert @1) (convert @2)))))
   (simplify
    (pointer_diff (pointer_plus @@0 @1) (pointer_plus @0 @2))
    /* The second argument of pointer_plus must be interpreted as signed, and
       thus sign-extended if necessary.  */
    (with { tree stype = signed_type_for (TREE_TYPE (@1)); }
     /* Use view_convert instead of convert here, as POINTER_PLUS_EXPR
	second arg is unsigned even when we need to consider it as signed,
	we don't want to diagnose overflow here.  */
     (minus (convert (view_convert:stype @1))
	    (convert (view_convert:stype @2)))))))

/* (A * C) +- (B * C) -> (A+-B) * C and (A * C) +- A -> A * (C+-1).
    Modeled after fold_plusminus_mult_expr.  */
(if (!TYPE_SATURATING (type)
     && (!FLOAT_TYPE_P (type) || flag_associative_math))
 (for plusminus (plus minus)
  (simplify
   (plusminus (mult:cs@3 @0 @1) (mult:cs@4 @0 @2))
   (if (!ANY_INTEGRAL_TYPE_P (type)
	|| TYPE_OVERFLOW_WRAPS (type)
	|| (INTEGRAL_TYPE_P (type)
	    && tree_expr_nonzero_p (@0)
	    && expr_not_equal_to (@0, wi::minus_one (TYPE_PRECISION (type)),
				  gimple_match_ctx (@3))))
    (if (single_use (@3) || single_use (@4))
     /* If @1 +- @2 is constant require a hard single-use on either
	original operand (but not on both).  */
     (mult (plusminus @1 @2) @0)
     (mult! (plusminus @1 @2) @0)
  )))
  /* We cannot generate constant 1 for fract.  */
  (if (!ALL_FRACT_MODE_P (TYPE_MODE (type)))
   (simplify
    (plusminus @0 (mult:c@3 @0 @2))
    (if ((!ANY_INTEGRAL_TYPE_P (type)
	  || TYPE_OVERFLOW_WRAPS (type)
	  /* For @0 + @0*@2 this transformation would introduce UB
	     (where there was none before) for @0 in [-1,0] and @2 max.
	     For @0 - @0*@2 this transformation would introduce UB
	     for @0 0 and @2 in [min,min+1] or @0 -1 and @2 min+1.  */
	  || (INTEGRAL_TYPE_P (type)
	      && ((tree_expr_nonzero_p (@0)
		   && expr_not_equal_to (@0,
				wi::minus_one (TYPE_PRECISION (type)),
				gimple_match_ctx (@3)))
		  || (plusminus == PLUS_EXPR
		      ? expr_not_equal_to (@2,
			    wi::max_value (TYPE_PRECISION (type), SIGNED),
			    gimple_match_ctx (@3))
		      /* Let's ignore the @0 -1 and @2 min case.  */
		      : (expr_not_equal_to (@2,
			    wi::min_value (TYPE_PRECISION (type), SIGNED),
			    gimple_match_ctx (@3))
			 && expr_not_equal_to (@2,
				wi::min_value (TYPE_PRECISION (type), SIGNED)
				+ 1, gimple_match_ctx (@3)))))))
	 && single_use (@3))
     (mult (plusminus { build_one_cst (type); } @2) @0)))
   (simplify
    (plusminus (mult:c@3 @0 @2) @0)
    (if ((!ANY_INTEGRAL_TYPE_P (type)
	  || TYPE_OVERFLOW_WRAPS (type)
	  /* For @0*@2 + @0 this transformation would introduce UB
	     (where there was none before) for @0 in [-1,0] and @2 max.
	     For @0*@2 - @0 this transformation would introduce UB
	     for @0 0 and @2 min.  */
	  || (INTEGRAL_TYPE_P (type)
	      && ((tree_expr_nonzero_p (@0)
		   && (plusminus == MINUS_EXPR
		       || expr_not_equal_to (@0,
				wi::minus_one (TYPE_PRECISION (type)),
				gimple_match_ctx (@3))))
		  || expr_not_equal_to (@2,
			(plusminus == PLUS_EXPR
			 ? wi::max_value (TYPE_PRECISION (type), SIGNED)
			 : wi::min_value (TYPE_PRECISION (type), SIGNED)),
			gimple_match_ctx (@3)))))
	 && single_use (@3))
      (mult (plusminus @2 { build_one_cst (type); }) @0)))))
 /* (A * B) + (-C) -> (B - C/A) * A, if C is a multiple of A.  */
 (if (!ALL_FRACT_MODE_P (TYPE_MODE (type)))
  (simplify
    (plus (mult:cs integer_nonzerop@0 @1) INTEGER_CST@2)
    /* Exclude the case that @2 == min to prevent UB when calculating abs
       and (B - C/A).  */
    (if (TREE_CODE (type) == INTEGER_TYPE
	&& wi::neg_p (wi::to_wide (@2))
	&& wi::to_wide (@2) != wi::min_value (TYPE_PRECISION (type), SIGNED))
      (with {
	wide_int c0 = wi::to_wide (@0);
	wide_int c2 = wi::to_wide (@2);
	wide_int c2_abs = wi::abs (c2); }
      (if (wi::multiple_of_p (c2_abs, c0, TYPE_SIGN (type)))
	(with {
	  /* Calculate @2 / @0 in order to factorize the expression.  */
	  wide_int div_res = wi::sdiv_trunc (c2, c0);
	  tree div_cst = wide_int_to_tree (type, div_res); }
	(mult (plus @1 { div_cst; }) @0))))))))

#if GIMPLE
/* Canonicalize X + (X << C) into X * (1 + (1 << C)) and
   (X << C1) + (X << C2) into X * ((1 << C1) + (1 << C2)).  */
(simplify
 (plus:c @0 (lshift:s @0 INTEGER_CST@1))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && tree_fits_uhwi_p (@1)
       && tree_to_uhwi (@1) < element_precision (type)
       && (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   || optab_handler (smul_optab,
			     TYPE_MODE (type)) != CODE_FOR_nothing))
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t)) t = unsigned_type_for (t);
	   wide_int w = wi::set_bit_in_zero (tree_to_uhwi (@1),
					     element_precision (type));
	   w += 1;
	   tree cst = wide_int_to_tree (VECTOR_TYPE_P (t) ? TREE_TYPE (t)
					: t, w);
	   cst = build_uniform_cst (t, cst); }
    (convert (mult (convert:t @0) { cst; })))))
(simplify
 (plus (lshift:s @0 INTEGER_CST@1) (lshift:s @0 INTEGER_CST@2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && tree_fits_uhwi_p (@1)
       && tree_to_uhwi (@1) < element_precision (type)
       && tree_fits_uhwi_p (@2)
       && tree_to_uhwi (@2) < element_precision (type)
       && (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   || optab_handler (smul_optab,
			     TYPE_MODE (type)) != CODE_FOR_nothing))
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t)) t = unsigned_type_for (t);
	   unsigned int prec = element_precision (type);
	   wide_int w = wi::set_bit_in_zero (tree_to_uhwi (@1), prec);
	   w += wi::set_bit_in_zero (tree_to_uhwi (@2), prec);
	   tree cst = wide_int_to_tree (VECTOR_TYPE_P (t) ? TREE_TYPE (t)
					: t, w);
	   cst = build_uniform_cst (t, cst); }
    (convert (mult (convert:t @0) { cst; })))))

/* (X * C1) + (X << C2) -> X * (C1 + (1 << C2))  */
(simplify
 (plus:c (nop_convert? (mult:s (nop_convert? @0) uniform_integer_cst_p@1))
	 (lshift:s @0 INTEGER_CST@2))
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && tree_fits_uhwi_p (@2)
       && tree_to_uhwi (@2) < element_precision (type))
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t)) t = unsigned_type_for (t);
	   unsigned int prec = element_precision (type);
	   wide_int w = wi::to_wide (uniform_integer_cst_p (@1));
	   w += wi::set_bit_in_zero (tree_to_uhwi (@2), prec);
	   tree cst = wide_int_to_tree (VECTOR_TYPE_P (t) ? TREE_TYPE (t)
					: t, w);
	   cst = build_uniform_cst (t, cst); }
    (convert (mult (convert:t @0) { cst; })))))
#endif

/* Canonicalize (X*C1)|(X*C2) and (X*C1)^(X*C2) to (C1+C2)*X when
   tree_nonzero_bits allows IOR and XOR to be treated like PLUS.
   Likewise, handle (X<<C3) and X as legitimate variants of X*C.  */
(for op (bit_ior bit_xor)
 (simplify
  (op (nop_convert?:s@6 (mult:s@0 (nop_convert? @1) INTEGER_CST@2))
      (nop_convert?:s@7 (mult:s@3 (nop_convert? @5) INTEGER_CST@4)))
  (if (INTEGRAL_TYPE_P (type)
       && operand_equal_p (@1, @5, 0)
       && (tree_nonzero_bits (@6) & tree_nonzero_bits (@7)) == 0)
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t))
	     t = unsigned_type_for (t);
	   wide_int c = wi::add (wi::to_wide (@2), wi::to_wide (@4)); }
    (convert (mult:t (convert:t @1) { wide_int_to_tree (t, c); })))))
 (simplify
  (op:c (nop_convert?:s@5 (mult:s@0 (nop_convert? @1) INTEGER_CST@2))
	(lshift:s@3 @1 INTEGER_CST@4))
  (if (INTEGRAL_TYPE_P (type)
       && tree_int_cst_sgn (@4) > 0
       && (tree_nonzero_bits (@5) & tree_nonzero_bits (@3)) == 0)
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t))
	     t = unsigned_type_for (t);
	   wide_int wone = wi::one (TYPE_PRECISION (type));
	   wide_int c = wi::add (wi::to_wide (@2),
				 wi::lshift (wone, wi::to_wide (@4))); }
    (convert (mult:t (convert:t @1) { wide_int_to_tree (t, c); })))))
 (simplify
  (op:c (nop_convert?:s@3 (mult:s@0 (nop_convert? @1) INTEGER_CST@2))
	@1)
  (if (INTEGRAL_TYPE_P (type)
       && (tree_nonzero_bits (@3) & tree_nonzero_bits (@1)) == 0)
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t))
	     t = unsigned_type_for (t);
	   wide_int c = wi::add (wi::to_wide (@2), 1); }
    (convert (mult:t (convert:t @1) { wide_int_to_tree (t, c); })))))
 (simplify
  (op (lshift:s@0 @1 INTEGER_CST@2)
      (lshift:s@3 @1 INTEGER_CST@4))
  (if (INTEGRAL_TYPE_P (type)
       && tree_int_cst_sgn (@2) > 0
       && tree_int_cst_sgn (@4) > 0
       && (tree_nonzero_bits (@0) & tree_nonzero_bits (@3)) == 0)
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t))
	     t = unsigned_type_for (t);
	   wide_int wone = wi::one (TYPE_PRECISION (t));
	   wide_int c = wi::add (wi::lshift (wone, wi::to_wide (@2)),
				 wi::lshift (wone, wi::to_wide (@4))); }
    (convert (mult:t (convert:t @1) { wide_int_to_tree (t,c); })))))
 (simplify
  (op:c (lshift:s@0 @1 INTEGER_CST@2)
	@1)
  (if (INTEGRAL_TYPE_P (type)
       && tree_int_cst_sgn (@2) > 0
       && (tree_nonzero_bits (@0) & tree_nonzero_bits (@1)) == 0)
   (with { tree t = type;
	   if (!TYPE_OVERFLOW_WRAPS (t))
	     t = unsigned_type_for (t);
	   wide_int wone = wi::one (TYPE_PRECISION (t));
	   wide_int c = wi::add (wi::lshift (wone, wi::to_wide (@2)), wone); }
    (convert (mult:t (convert:t @1) { wide_int_to_tree (t, c); }))))))

/* Simplifications of MIN_EXPR, MAX_EXPR, fmin() and fmax().  */

(for minmax (min max)
     maxmin (max min)
 (simplify
  (minmax @0 @0)
  @0)
/* max(max(x,y),x) -> max(x,y)  */
 (simplify
  (minmax:c (minmax:c@2 @0 @1) @0)
  @2)

/* min(max(x,y),y) -> y.
   max(min(x,y),y) -> y.  */
 (simplify
  (minmax:c (maxmin:c @0 @1) @1)
  @1))

/* For fmin() and fmax(), skip folding when both are sNaN.  */
(for minmax (FMIN_ALL FMAX_ALL)
 (simplify
  (minmax @0 @0)
  (if (!tree_expr_maybe_signaling_nan_p (@0))
    @0)))
/* max(a,-a) -> abs(a).  */
(simplify
 (max:c @0 (negate @0))
 (if (TREE_CODE (type) != COMPLEX_TYPE
      && (! ANY_INTEGRAL_TYPE_P (type)
	  || TYPE_OVERFLOW_UNDEFINED (type)))
  (abs @0)))
/* min(a,-a) -> -abs(a).  */
(simplify
 (min:c @0 (negate @0))
 (if (TREE_CODE (type) != COMPLEX_TYPE
      && (! ANY_INTEGRAL_TYPE_P (type)
	  || TYPE_OVERFLOW_UNDEFINED (type)))
  (negate (abs @0))))
(simplify
 (min @0 @1)
 (switch
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_MIN_VALUE (type)
       && operand_equal_p (@1, TYPE_MIN_VALUE (type), OEP_ONLY_CONST))
   @1)
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_MAX_VALUE (type)
       && operand_equal_p (@1, TYPE_MAX_VALUE (type), OEP_ONLY_CONST))
   @0)))
(simplify
 (max @0 @1)
 (switch
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_MAX_VALUE (type)
       && operand_equal_p (@1, TYPE_MAX_VALUE (type), OEP_ONLY_CONST))
   @1)
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_MIN_VALUE (type)
       && operand_equal_p (@1, TYPE_MIN_VALUE (type), OEP_ONLY_CONST))
   @0)))

/* minmax (a + c, b + c) -> minmax (a, b) + c */
(for minmax (min max)
 (simplify
  (minmax:c (plus:cs @0 @2) (plus:s @1 @2))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& !TYPE_OVERFLOW_SANITIZED (type))
    (plus (minmax @0 @1) @2))))

/* minmax (a - c, b - c) -> minmax (a, b) - c */
(for minmax (min max)
 (simplify
  (minmax:c (minus:s @0 @2) (minus:s @1 @2))
   (if (INTEGRAL_TYPE_P (type)
	&& TYPE_OVERFLOW_UNDEFINED (type)
	&& !TYPE_OVERFLOW_SANITIZED (type))
    (minus (minmax @0 @1) @2))))

/* minmax ((wide_type) a, (wide_type) b) -> (wide_type) minmax (a, b)
   when type matches the type of a and b, and wide_type is a wider
   type with the same signedness as type.  Extension is monotone, so it
   commutes with the comparison, and the truncation is then exact.  The
   same holds lanewise for a vector, where the narrow operation has to be
   available so that lowering does not scalarise it.  The wide operation
   must become dead before a narrow vector operation is introduced.  */
(for minmax (min max)
     MINMAX (MIN_EXPR MAX_EXPR)
 (simplify
  (minmax (convert @0) (convert @1))
  (if (ANY_INTEGRAL_TYPE_P (type)
       && ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && types_match (TREE_TYPE (@0), TREE_TYPE (@1))
       && element_precision (TREE_TYPE (@0)) < element_precision (type)
       && TYPE_UNSIGNED (TREE_TYPE (@0)) == TYPE_UNSIGNED (type)
       && (!VECTOR_TYPE_P (type)
	   || target_supports_op_p (TREE_TYPE (@0), MINMAX, optab_vector)))
   (convert (minmax @0 @1)))))

/* max (a, a + CST) -> a + CST where CST is positive.  */
/* max (a, a + CST) -> a where CST is negative.  */
(simplify
 (max:c @0 (plus@2 @0 INTEGER_CST@1))
  (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (if (tree_int_cst_sgn (@1) > 0)
    @2
    @0)))

/* min (a, a + CST) -> a where CST is positive.  */
/* min (a, a + CST) -> a + CST where CST is negative. */
(simplify
 (min:c @0 (plus@2 @0 INTEGER_CST@1))
  (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (if (tree_int_cst_sgn (@1) > 0)
    @0
    @2)))

/* Narrow a saturating add or subtract that promotions or explicit vector
   conversions widened.
   For an unsigned type,
     (type) MAX ((wide_type) a - (wide_type) b, 0) -> a - MIN (a, b)
     (type) MIN ((wide_type) a + (wide_type) b, MAX) -> a + MIN (~a, b)
   Both results are exact in the narrow type: the difference is clamped at
   zero and the sum at MAX, so the truncation never loses a bit.  Require the
   outer clamp to have one use.  For a vector, also require target support for
   each narrow operation that the result introduces.  */
(simplify
 (convert (max:s (minus@2 (convert @0) (convert @1)) integer_zerop))
 (if (ANY_INTEGRAL_TYPE_P (type)
      && TYPE_UNSIGNED (type)
      && types_match (type, @0, @1)
      && !TYPE_UNSIGNED (TREE_TYPE (@2))
      && element_precision (TREE_TYPE (@2)) > element_precision (type)
      && (!VECTOR_TYPE_P (type)
	  || (target_supports_op_p (type, MIN_EXPR, optab_vector)
	      && target_supports_op_p (type, MINUS_EXPR, optab_vector))))
  (minus @0 (min @0 @1))))

(simplify
 (convert (min:s (plus@2 (convert @0) (convert @1))
		 uniform_integer_cst_p@3))
 (if (ANY_INTEGRAL_TYPE_P (type)
      && TYPE_UNSIGNED (type)
      && types_match (type, @0, @1)
      && (!VECTOR_TYPE_P (type)
	  || (target_supports_op_p (type, BIT_NOT_EXPR, optab_vector)
	      && target_supports_op_p (type, MIN_EXPR, optab_vector)
	      && target_supports_op_p (type, PLUS_EXPR, optab_vector))))
  (with
   {
    unsigned precision = element_precision (type);
    tree wide_type = TREE_TYPE (@2);
    unsigned wide_precision = element_precision (wide_type);
    tree max_cst = uniform_integer_cst_p (@3);
    /* The widened sum reaches 2 * (2^precision - 1), which needs one bit
       more than the narrow type, and one further bit for a sign.  */
    unsigned needed = precision + (TYPE_UNSIGNED (wide_type) ? 1 : 2);
   }
   (if (wide_precision >= needed
	&& wi::eq_p (wi::to_wide (max_cst),
		     wi::mask (precision, false, wide_precision)))
    (plus @0 (min @1 (bit_not @0)))))))

/* MIN (MAX (X, Y + CST), Y) -> Y for a positive CST, and the dual
   MAX (MIN (X, Y + CST), Y) -> Y for a negative one.  The inner select is
   at least Y + CST, which the absence of wrapping puts strictly beyond Y,
   so the outer select always takes Y and the whole nest is dead.  */
(for outer (min max)
     inner (max min)
 (simplify
  (outer:c (inner:c @0 (plus @1 INTEGER_CST@2)) @1)
  (if (TYPE_OVERFLOW_UNDEFINED (type)
       && !TYPE_OVERFLOW_SANITIZED (type)
       && ((outer == MIN_EXPR && tree_int_cst_sgn (@2) > 0)
	   || (outer == MAX_EXPR && tree_int_cst_sgn (@2) < 0)))
   @1)))

/* min (a, b) op max (a, b) -> a op b */
(for op (plus mult bit_and bit_xor bit_ior eq ne min max)
 (simplify
  (op:c (min:c @0 @1) (max @0 @1))
   (if (!HONOR_NANS (@0))
    (op @0 @1))))

/* min (a, b) <= max (a, b) -> true
   max (a, b) >= min (a, b) -> true
   min (a, b) >  max (a, b) -> false
   max (a, b) <  min (a, b) -> false */
(for legt (le gt)
 (simplify
  (legt:c (min @0 @1) (max @0 @1))
  (if (!HONOR_NANS (@0))
   { constant_boolean_node (legt == LE_EXPR, type); })))

/* min (a, b) <  max (a, b) -> a != b
   max (a, b) >  min (a, b) -> a != b
   min (a, b) >= max (a, b) -> a == b
   max (a, b) <= min (a, b) -> a == b */
(for ltge (lt ge)
     neeq (ne eq)
 (simplify
  (ltge:c (min @0 @1) (max @0 @1))
  (if (!HONOR_NANS (@0))
   (neeq @0 @1))))

/* min (a & CST0, a & CST1) -> a & CST0 IFF CST0 & CST1 == CST0 */
/* max (a & CST0, a & CST1) -> a & CST0 IFF CST0 & CST1 == CST1 */
/* If signed a, then both the constants should have same sign. */
(for minmax (min max)
 (simplify
  (minmax (bit_and@3 @0 INTEGER_CST@1) (bit_and@4 @0 INTEGER_CST@2))
   (if (TYPE_UNSIGNED (type)
        || (tree_int_cst_sgn (@1) == tree_int_cst_sgn (@2)))
    (with { auto andvalue = wi::to_wide (@1) & wi::to_wide (@2); }
     (if (andvalue == ((minmax == MIN_EXPR)
		        ? wi::to_wide (@1) : wi::to_wide (@2)))
      @3
      (if (andvalue == ((minmax != MIN_EXPR)
			 ? wi::to_wide (@1) : wi::to_wide (@2)))
       @4))))))

/* min (a, a & CST) --> a & CST */
/* max (a, a & CST) --> a */
(for minmax (min max)
 (simplify
  (minmax @0 (bit_and@1 @0 INTEGER_CST@2))
   (if (TYPE_UNSIGNED(type))
    (if (minmax == MIN_EXPR)
     @1
     @0))))

/* Simplify min (&var[off0], &var[off1]) etc. depending on whether
   the addresses are known to be less, equal or greater.  */
(for minmax (min max)
     cmp (lt gt)
 (simplify
  (minmax (convert1?@2 addr@0) (convert2?@3 addr@1))
  (with
   {
     poly_int64 off0, off1;
     tree base0, base1;
     int equal = address_compare (cmp, TREE_TYPE (@2), @0, @1, base0, base1,
				  off0, off1, GENERIC);
   }
   (if (equal == 1)
    (if (minmax == MIN_EXPR)
     (if (known_le (off0, off1))
      @2
      (if (known_gt (off0, off1))
       @3))
     (if (known_ge (off0, off1))
      @2
      (if (known_lt (off0, off1))
       @3)))))))

/* (convert (minmax ((convert (x) c)))) -> minmax (x c) if x is promoted
   and the outer convert demotes the expression back to x's type.  */
(for minmax (min max)
 (simplify
  (convert (minmax@0 (convert @1) INTEGER_CST@2))
  (if (INTEGRAL_TYPE_P (type)
       && types_match (@1, type) && int_fits_type_p (@2, type)
       && TYPE_SIGN (TREE_TYPE (@0)) == TYPE_SIGN (type)
       && TYPE_PRECISION (TREE_TYPE (@0)) > TYPE_PRECISION (type))
   (minmax @1 (convert @2)))))

(for minmax (FMIN_ALL FMAX_ALL)
 /* If either argument is NaN and other one is not sNaN, return the other
    one.  Avoid the transformation if we get (and honor) a signalling NaN.  */
 (simplify
  (minmax:C @0 REAL_CST@1)
   (if (real_isnan (TREE_REAL_CST_PTR (@1))
       && (!HONOR_SNANS (@1) || !TREE_REAL_CST (@1).signalling)
       && !tree_expr_maybe_signaling_nan_p (@0))
   @0)))
/* Convert fmin/fmax to MIN_EXPR/MAX_EXPR.  C99 requires these
   functions to return the numeric arg if the other one is NaN.
   MIN and MAX don't honor that, so only transform if -ffinite-math-only
   is set.  C99 doesn't require -0.0 to be handled, so we don't have to
   worry about it either.  */
(if (flag_finite_math_only)
 (simplify
  (FMIN_ALL @0 @1)
  (min @0 @1))
 (simplify
  (FMAX_ALL @0 @1)
  (max @0 @1)))
/* min (-A, -B) -> -max (A, B)  */
(for minmax (min max FMIN_ALL FMAX_ALL)
     maxmin (max min FMAX_ALL FMIN_ALL)
 (simplify
  (minmax (negate:s@2 @0) (negate:s@3 @1))
  (if (FLOAT_TYPE_P (TREE_TYPE (@0))
       || (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
           && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))))
   (negate (maxmin @0 @1)))))
/* MIN (~X, ~Y) -> ~MAX (X, Y)
   MAX (~X, ~Y) -> ~MIN (X, Y)  */
(for minmax (min max)
 maxmin (max min)
 (simplify
  (minmax (bit_not:s@2 @0) (bit_not:s@3 @1))
  (bit_not (maxmin @0 @1)))
/* ~MAX(~X, Y) --> MIN(X, ~Y) */
/* ~MIN(~X, Y) --> MAX(X, ~Y) */
 (simplify
  (bit_not (minmax:cs (bit_not @0) @1))
  (maxmin @0 (bit_not @1))))

/* umin (a, 1) | umin (b, 1) -> umin (a | b, 1).  */
 (simplify
  (bit_ior (min @0 integer_onep@2) (min @1 @2))
    (if (tree_expr_nonnegative_p (@0) && tree_expr_nonnegative_p (@1))
     (min (bit_ior @0 @1) @2)))
/* MIN (X, Y) == X -> X <= Y  */
/* MIN (X, Y) < X -> X > Y  */
/* MIN (X, Y) >= X -> X <= Y  */
(for minmax (min min min min max max max max)
     cmp    (eq  ne  lt  ge  eq  ne  gt  le )
     out    (le  gt  gt  le  ge  lt  lt  ge )
 (simplify
  (cmp:c (minmax:c @0 @1) @0)
  (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && (!VECTOR_TYPE_P (TREE_TYPE (@0))
	   || (VECTOR_TYPE_P (type)
	       && (!expand_vec_cmp_expr_p (TREE_TYPE (@0), type, cmp)
		   || expand_vec_cmp_expr_p (TREE_TYPE (@0), type, out)))))
   (out @0 @1))))
/* MIN (X, 5) == 0 -> X == 0
   MIN (X, 5) == 7 -> false  */
(for cmp (eq ne)
 (simplify
  (cmp (min @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (wi::lt_p (wi::to_wide (@1), wi::to_wide (@2),
		 TYPE_SIGN (TREE_TYPE (@0))))
   { constant_boolean_node (cmp == NE_EXPR, type); }
   (if (wi::gt_p (wi::to_wide (@1), wi::to_wide (@2),
		  TYPE_SIGN (TREE_TYPE (@0))))
    (cmp @0 @2)))))
(for cmp (eq ne)
 (simplify
  (cmp (max @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (wi::gt_p (wi::to_wide (@1), wi::to_wide (@2),
		 TYPE_SIGN (TREE_TYPE (@0))))
   { constant_boolean_node (cmp == NE_EXPR, type); }
   (if (wi::lt_p (wi::to_wide (@1), wi::to_wide (@2),
		  TYPE_SIGN (TREE_TYPE (@0))))
    (cmp @0 @2)))))

/* X <= MAX(X, Y) -> true
   X > MAX(X, Y) -> false
   X >= MIN(X, Y) -> true
   X < MIN(X, Y) -> false */
(for minmax (min     min     max     max     )
     cmp    (ge      lt      le      gt      )
 (simplify
  (cmp:c @0 (minmax:c @0 @1))
  { constant_boolean_node (cmp == GE_EXPR || cmp == LE_EXPR, type); } ))

/* MIN (X, C1) < C2 -> X < C2 || C1 < C2  */
(for minmax (min     min     max     max     min     min     max     max    )
     cmp    (lt      le      gt      ge      gt      ge      lt      le     )
     comb   (bit_ior bit_ior bit_ior bit_ior bit_and bit_and bit_and bit_and)
 (simplify
  (cmp (minmax @0 INTEGER_CST@1) INTEGER_CST@2)
  (comb (cmp @0 @2) (cmp @1 @2))))

/* MAX (A, B) == 0 -> (A|B) == 0 iff non-negative.
   MAX (A, B) != 0 -> (A|B) != 0 iff non-negative.  */
(for cmp (eq ne)
 (simplify
  (cmp (max tree_expr_nonnegative_p@0 tree_expr_nonnegative_p@1) integer_zerop)
  (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
   (cmp (bit_ior (convert:utype @0) (convert:utype @1))
        { build_zero_cst (utype); } ))))

/* MIN (A, B) == 0 -> A == 0 if B is positive.
   MIN (A, B) != 0 -> A != 0 if B is positive.  */
(for cmp (eq ne)
 (simplify
  (cmp (min:c @0 tree_expr_nonnegative_p@1) integer_zerop@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && tree_expr_nonzero_p (@1))
   (cmp @0 @2))))

/* Optimize MIN (a, b) >= 0 to a >= 0 if b is non-negative.
   Optimize MIN (a, b) < 0 to a < 0 if b is non-negative.
   Optimize (a | b) >= 0 to a >= 0 if b is non-negative.
   Optimize (a | b) < 0 to a < 0 if b is non-negative.  */
(for op (min bit_ior)
  (for cmp (ge lt)
    (simplify
      (cmp (op:c @0 tree_expr_nonnegative_p@1) integer_zerop@2)
        (cmp @0 @2))))

#if GIMPLE
/* ((X + C1) | (C2 - X)) < 0 -> (unsigned) (X + C1) > C1 + C2, also spelled
   (X | (C2 - X)) < 0, ((X + C1) | -X) < 0 and (X | -X) < 0, and with the
   sign bit extracted by a shift rather than compared.  */
(match (ior_range_test @0 @1 @2)
 (bit_ior:c@3 (plus@0 @4 INTEGER_CST@1) (minus INTEGER_CST@2 @4))
 (if (single_use (@3)
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
      && wi::ge_p (wi::to_wide (@1) + wi::to_wide (@2), 0, SIGNED))))
(match (ior_range_test_1 @0 @1)
 (bit_ior:c@2 @0 (minus INTEGER_CST@1 @0))
 (if (single_use (@2)
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
      && wi::ge_p (wi::to_wide (@1), 0, SIGNED))))
(match (ior_range_test_1 @0 @1)
 (bit_ior:c@2 (plus@0 @3 INTEGER_CST@1) (negate @3))
 (if (single_use (@2)
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
      && wi::ge_p (wi::to_wide (@1), 0, SIGNED))))
(match (ior_range_test_0 @0)
 (bit_ior:c@1 @0 (negate @0))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && single_use (@1)
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))))
(for cmp (lt ge)
     out (gt le)
 (simplify
  (cmp (nop_convert?@5 (ior_range_test @0 @1 @2)) integer_zerop)
  (if (!TYPE_UNSIGNED (TREE_TYPE (@5)))
   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
    (out (convert:utype @0)
	 { wide_int_to_tree (utype, wi::to_wide (@1) + wi::to_wide (@2)); }))))
 (simplify
  (cmp (nop_convert?@3 (ior_range_test_1 @0 @1)) integer_zerop)
  (if (!TYPE_UNSIGNED (TREE_TYPE (@3)))
   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
    (out (convert:utype @0) (convert:utype @1)))))
 (simplify
  (cmp (nop_convert?@2 (ior_range_test_0 @0)) integer_zerop@3)
  (if (!TYPE_UNSIGNED (TREE_TYPE (@2)))
   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
    (out (convert:utype @0) (convert:utype @3))))))
(simplify
 (rshift (nop_convert? (ior_range_test @0 @1 @2)) INTEGER_CST@5)
 (if (TYPE_UNSIGNED (type)
      && wi::to_wide (@5) == element_precision (type) - 1)
  (convert (gt (convert:type @0)
	       { wide_int_to_tree (type, wi::to_wide (@1) + wi::to_wide (@2)); }))))
(simplify
 (rshift (nop_convert? (ior_range_test_1 @0 @1)) INTEGER_CST@2)
 (if (TYPE_UNSIGNED (type)
      && wi::to_wide (@2) == element_precision (type) - 1)
  (convert (gt (convert:type @0) (convert:type @1)))))
(simplify
 (rshift (nop_convert? (ior_range_test_0 @0)) INTEGER_CST@1)
 (if (TYPE_UNSIGNED (type)
      && wi::to_wide (@1) == element_precision (type) - 1)
  (convert (ne @0 { build_zero_cst (TREE_TYPE (@0)); }))))
#endif

/* Undo fancy ways of writing max/min or other ?: expressions, like
   a - ((a - b) & -(a < b))  and  a - (a - b) * (a < b) into (a < b) ? b : a.
   People normally use ?: and that is what we actually try to optimize.  */
/* Transform A + (B-A)*cmp into cmp ? B : A.  */
(simplify
 (plus:c @0 (mult:c (minus @1 @0) zero_one_valued_p@2))
 (if (INTEGRAL_TYPE_P (type)
      && (GIMPLE || !TREE_SIDE_EFFECTS (@1)))
  (cond (convert:boolean_type_node @2) @1 @0)))
/* Transform A - (A-B)*cmp into cmp ? B : A.  */
(simplify
 (minus @0 (mult:c (minus @0 @1) zero_one_valued_p@2))
 (if (INTEGRAL_TYPE_P (type)
      && (GIMPLE || !TREE_SIDE_EFFECTS (@1)))
  (cond (convert:boolean_type_node @2) @1 @0)))
/* Transform A ^ (A^B)*cmp into cmp ? B : A.  */
(simplify
 (bit_xor:c @0 (mult:c (bit_xor:c @0 @1) zero_one_valued_p@2))
 (if (INTEGRAL_TYPE_P (type)
      && (GIMPLE || !TREE_SIDE_EFFECTS (@1)))
  (cond (convert:boolean_type_node @2) @1 @0)))

/* Transform (A ^ -cmp) + cmp into cmp ? -A : A.  */
(simplify
 (plus:c (bit_xor:c @0 (negate zero_one_valued_p@1)) @1)
 (if (INTEGRAL_TYPE_P (type)
      && !TYPE_SATURATING (type)
      && (GIMPLE || !TREE_SIDE_EFFECTS (@0)))
  /* Do the negate in unsigned type always; otherwise
     we would be introducing an overflow.  */
  (with { tree utype = unsigned_type_for (type); }
   (cond
    (convert:boolean_type_node @1)
    (convert:type (negate (convert:utype @0)))
    @0))))

/* Transform A & (B*cmp) into (A&B)*cmp.  */
(simplify
 (bit_and:c (mult:cs zero_one_valued_p@0 @1) @2)
 (mult @0 (bit_and! @1 @2)))

/* (x <= 0 ? -x : 0) -> max(-x, 0).  */
(simplify
 (cond (le @0 integer_zerop@1) (negate@2 @0) integer_zerop@1)
 (if (ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_UNDEFINED (type))
  (max @2 @1)))

/* (zero_one == 0) ? y : z <op> y -> ((typeof(y))zero_one * z) <op> y */
(for op (bit_xor bit_ior plus)
 (simplify
  (cond (eq zero_one_valued_p@0
            integer_zerop)
        @1
        (op:c @2 @1))
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_PRECISION (type) > 1
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && expr_no_side_effects_p (@2))
       (op (mult (convert:type @0) @2) @1))))

/* (zero_one != 0) ? z <op> y : y -> ((typeof(y))zero_one * z) <op> y */
(for op (bit_xor bit_ior plus)
 (simplify
  (cond (ne zero_one_valued_p@0
            integer_zerop)
       (op:c @2 @1)
        @1)
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_PRECISION (type) > 1
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && expr_no_side_effects_p (@2))
       (op (mult (convert:type @0) @2) @1))))

/* ?: Value replacement. */
/* a == 0 ? b : b + a  -> b + a */
(for op (plus bit_ior bit_xor)
 (simplify
  (cond (eq @0 integer_zerop) @1 (op:c@2 @1 @0))
   @2))
/* a == 0 ? b : b - a  -> b - a */
/* a == 0 ? b : b ptr+ a  -> b ptr+ a */
/* a == 0 ? b : b shift/rotate a -> b shift/rotate a */
(for op (lrotate rrotate lshift rshift minus pointer_plus)
 (simplify
  (cond (eq @0 integer_zerop) @1 (op@2 @1 @0))
   @2))

/* a == 1 ? b : b / a  -> b / a */
(for op (trunc_div ceil_div floor_div round_div exact_div)
 (simplify
  (cond (eq @0 integer_onep) @1 (op@2 @1 @0))
   @2))

/* a == 1 ? b : a * b -> a * b */
(for op (mult)
 (simplify
  (cond (eq @0 integer_onep) @1 (op:c@2 @1 @0))
   @2))

/* a == -1 ? b : a & b -> a & b */
(for op (bit_and)
 (simplify
  (cond (eq @0 integer_all_onesp) @1 (op:c@2 @1 @0))
   @2))

/* a != 0 ? a / b : 0  -> a / b iff b is nonzero. */
(for op (trunc_div ceil_div floor_div round_div exact_div)
 (simplify
  (cond (ne @0 integer_zerop) (op@2 @3 @1) integer_zerop )
   (if (bitwise_equal_p (@0, @3)
        && tree_expr_nonzero_p (@1)
	/* Cannot make a expression with side effects
	   unconditional. */
	&& expr_no_side_effects_p (@1))
    @2)))

/* Note we prefer the != case here
   as (a != 0) * (a * b) will generate that version. */
/* a != 0 ? a * b : 0 -> a * b */
/* a != 0 ? a & b : 0 -> a & b */
(for op (mult bit_and)
 (simplify
  (cond (ne @0 integer_zerop) (op:c@2 @1 @3) integer_zerop)
  (if (bitwise_equal_p (@0, @3)
	/* Cannot make a expression with side effects
	   unconditional. */
	&& expr_no_side_effects_p (@1))
   @2)))

/* x != CST1 ? x + CST2 : CST3 -> x + CST2 when CST1 + CST2 == CST3.
   This handles both scalars and uniform vectors.  */
(for cnd (cond vec_cond)
 (simplify
  (cnd (ne @0 uniform_integer_cst_p@1)
       (plus@2 @0 uniform_integer_cst_p@3)
       uniform_integer_cst_p@4)
  (with {
    tree cst1 = uniform_integer_cst_p (@1);
    tree cst2 = uniform_integer_cst_p (@3);
    tree cst3 = uniform_integer_cst_p (@4);
    wi::overflow_type ovf;
   }
   (if (wi::add (wi::to_wide (cst1), wi::to_wide (cst2),
		 TYPE_SIGN (TREE_TYPE (cst1)), &ovf) == wi::to_wide (cst3))
    (if (!ovf || TYPE_OVERFLOW_WRAPS (TREE_TYPE (cst1)))
     @2
     (with {
       tree utype = unsigned_type_for (type);
      }
      (convert (plus (convert:utype @0) (convert:utype @3)))))))))

/* Simplifications of shift and rotates.  */

(for rotate (lrotate rrotate)
 (simplify
  (rotate integer_all_onesp@0 @1)
  @0))

/* Optimize -1 >> x for arithmetic right shifts.  */
(simplify
 (rshift integer_all_onesp@0 @1)
 (if (!TYPE_UNSIGNED (type))
  @0))

/* Optimize (x >> c) << c into x & (-1<<c).  */
(simplify
 (lshift (nop_convert? (rshift @0 INTEGER_CST@1)) @1)
 (if (wi::ltu_p (wi::to_wide (@1), element_precision (type)))
  /* It doesn't matter if the right shift is arithmetic or logical.  */
  (bit_and (view_convert @0) (lshift { build_minus_one_cst (type); } @1))))

(simplify
 (lshift (convert (convert@2 (rshift @0 INTEGER_CST@1))) @1)
 (if (wi::ltu_p (wi::to_wide (@1), element_precision (type))
      /* Allow intermediate conversion to integral type with whatever sign, as
	 long as the low TYPE_PRECISION (type)
	 - TYPE_PRECISION (TREE_TYPE (@2)) bits are preserved.  */
      && INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@2))
      && INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (@0))
      && (TYPE_PRECISION (TREE_TYPE (@2)) >= TYPE_PRECISION (type)
	  || wi::geu_p (wi::to_wide (@1),
			TYPE_PRECISION (type)
			- TYPE_PRECISION (TREE_TYPE (@2)))))
  (bit_and (convert @0) (lshift { build_minus_one_cst (type); } @1))))

/* (X op Y) >> C -> X >> C when every set bit of Y lies below bit C.
   Neither an inclusive nor an exclusive or can carry into the bits the
   shift keeps, so Y contributes nothing to the result.  */
(for op (bit_ior bit_xor)
 (simplify
  (rshift (op @0 @1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (type)
       && tree_fits_uhwi_p (@2)
       && tree_to_uhwi (@2) < TYPE_PRECISION (type))
   (with { wide_int mask = wi::mask (tree_to_uhwi (@2), true,
				     TYPE_PRECISION (type)); }
    (if ((tree_nonzero_bits (@1) & mask) == 0)
     (rshift @0 @2)
     (if ((tree_nonzero_bits (@0) & mask) == 0)
      (rshift @1 @2)))))))
#if GIMPLE
/* (X >> C1) << (C1 + C2) -> X << C2 if the low C1 bits of X are zero.  */
(simplify
 (lshift (convert? (rshift with_possible_nonzero_bits@0 INTEGER_CST@1))
         INTEGER_CST@2)
 (if (INTEGRAL_TYPE_P (type)
      && wi::ltu_p (wi::to_wide (@1), element_precision (type))
      && wi::ltu_p (wi::to_wide (@2), element_precision (type))
      && wi::to_widest (@2) >= wi::to_widest (@1)
      && wi::to_widest (@1) <= wi::ctz (get_nonzero_bits (@0)))
  (lshift (convert @0) (minus @2 @1))))

/* (X >> C1) * (C2 << C1) -> X * C2 if the low C1 bits of X are zero.  */
(simplify
 (mult (convert? (rshift with_possible_nonzero_bits@0 INTEGER_CST@1))
       poly_int_tree_p@2)
 (with { poly_widest_int factor; }
  (if (INTEGRAL_TYPE_P (type)
       && wi::ltu_p (wi::to_wide (@1), element_precision (type))
       && wi::to_widest (@1) <= wi::ctz (get_nonzero_bits (@0))
       && multiple_p (wi::to_poly_widest (@2),
		      widest_int (1) << tree_to_uhwi (@1),
		      &factor))
   (mult (convert @0) { wide_int_to_tree (type, factor); }))))
#endif

/* For (x << c) >> c, optimize into x & ((unsigned)-1 >> c) for
   unsigned x OR truncate into the precision(type) - c lowest bits
   of signed x (if they have mode precision or a precision of 1).  */
(simplify
 (rshift (nop_convert? (lshift @0 INTEGER_CST@1)) @@1)
 (if (wi::ltu_p (wi::to_wide (@1), element_precision (type)))
  (if (TYPE_UNSIGNED (type))
   (bit_and (convert @0) (rshift { build_minus_one_cst (type); } @1))
   (if (INTEGRAL_TYPE_P (type))
    (with {
      int width = element_precision (type) - tree_to_uhwi (@1);
      tree stype = NULL_TREE;
      if (width <= MAX_FIXED_MODE_SIZE)
	stype = build_nonstandard_integer_type (width, 0);
     }
     (if (stype && (width == 1 || type_has_mode_precision_p (stype)))
      (convert (convert:stype @0))))))))

/* PR125405: (x & ((1 << (N - 1)) - 1)) - (x & (1 << (N - 1)))
   -> sign_extend<N> (x). */
(simplify
 (minus (bit_and @0 INTEGER_CST@1) (bit_and @0 integer_pow2p@2))
 (if (INTEGRAL_TYPE_P (type)
      && TYPE_PRECISION (type) <= MAX_FIXED_MODE_SIZE
      && wi::to_wide (@1) + 1 == wi::to_wide (@2))
  (with {
    int width = wi::exact_log2 (wi::to_wide (@2)) + 1;
    tree ntype = NULL_TREE;

    if (width > 0 && width < TYPE_PRECISION (type))
      ntype = build_nonstandard_integer_type (width, SIGNED);
   }
   (if (ntype && type_has_mode_precision_p (ntype))
    (convert (convert:ntype @0))))))

/* Optimize x >> x into 0 */
(simplify
 (rshift @0 @0)
  { build_zero_cst (type); })

(for shiftrotate (lrotate rrotate lshift rshift)
 (simplify
  (shiftrotate @0 integer_zerop)
  (non_lvalue @0))
 (simplify
  (shiftrotate integer_zerop@0 @1)
  @0)
 /* Prefer vector1 << scalar to vector1 << vector2
    if vector2 is uniform.  */
 (for vec (VECTOR_CST CONSTRUCTOR)
  (simplify
   (shiftrotate @0 vec@1)
   (with { tree tem = uniform_vector_p (@1); }
    (if (tem)
     (shiftrotate @0 { tem; }))))))

/* Simplify X << Y where Y's low width bits are 0 to X, as only valid
   Y is 0.  Similarly for X >> Y.  */
#if GIMPLE
(for shift (lshift rshift)
 (simplify
  (shift @0 SSA_NAME@1)
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@1)))
    (with {
      int width = ceil_log2 (element_precision (TREE_TYPE (@0)));
      int prec = TYPE_PRECISION (TREE_TYPE (@1));
     }
     (if ((get_nonzero_bits (@1) & wi::mask (width, false, prec)) == 0)
      @0)))))
#endif

#if GIMPLE
/* PR119420: (A >> bool) == 0 -> (unsigned)a <= (unsigned)bool
   and its variant:
   (A >> bool) != 0 -> (unsigned)a > (unsigned)bool;  */
(for cmp (eq ne)
     icmp (le gt)
 (simplify
  (cmp (rshift@2 @0 zero_one_valued_p@1) integer_zerop)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) && single_use (@2))
   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
    (icmp (convert:utype @0) (convert:utype @1))))))
#endif

/* PR119422:
  (bool >= A) >= A -> bool >= A
  (bool >= A) <  A -> bool <  A.  */
(for cmp (ge lt)
 (simplify
  (cmp (convert? (ge zero_one_valued_p@0 @1)) @1)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_UNSIGNED (TREE_TYPE (@1)))
   (cmp @0 @1))))

/* PR124808: (A >> CST1) CMP CST2 -> A CMP (CST2 << CST1)
   Canonicalize non-equality comparisons between a right
   shift and a constant, turning it into a comparison
   with a constant that is left shifted.  If we view A as:

   A: "|--- CST2 ----|--CST1--|"

   A >> CST1 will be equal to CST2 for all A values in the
   range CST2 << CST1 to (CST2 << CST1) | 1s_mask (CST1).

   Therefore:
   - (A >> CST1) < CST2 -> A < (CST2 << CST1), A must
     be smaller than all values from the range;
   - (A >> CST1) <= CST2 -> A <= (CST2 << CST1) | mask,
     A must be smaller or equal than the range end;
   - (A >> CST1) > CST2 -> A > (CST2 << CST1) | mask,
     A must be greater than all values from the range;
   - (A >> CST1) >= CST2 -> A >= (CST2 << CST1), A must
     be greater or equal than the range start.

   We're also using "single_use" and wrapping around "if GIMPLE"
   because (1) this class "VA1 LSHIFT/RSHIFT VAL2 CMP VAL3" of
   optimizations tend to match CTZ|CLZ builtin patterns and we
   don't want to trip on them and (2) we will get in the way of
   certain optimizations (see ARM's sat-1.c test) that are done
   using GENERIC due to how forwprop currently works.  */
#if GIMPLE
(for cmp (le lt ge gt)
 (simplify
  (cmp (rshift@3 @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && single_use (@3)
       && tree_fits_uhwi_p (@1)
       && tree_to_uhwi (@1) < TYPE_PRECISION (TREE_TYPE (@0))
       && tree_int_cst_sgn (@2) >= 0
       && tree_to_uhwi (@1) < TYPE_PRECISION (TREE_TYPE (@2))
       && (wi::to_wide (@2) == 0
	   /* If @2 is nonzero, verify if we'll overflow when
	      doing @2 << @1 by checking if the shift amount
	      can start throwing 1s away if type_unsigned, or
	      if the shift amount can reach the sign bit.  */
	   || (TYPE_UNSIGNED (TREE_TYPE (@2))
	       && wi::leu_p (wi::to_wide (@1), wi::clz (wi::to_wide (@2))))
	   || (!TYPE_UNSIGNED (TREE_TYPE (@2))
	       && wi::ltu_p (wi::to_wide (@1), wi::clz (wi::to_wide (@2))))))

   /* No need to set the lower @1 bits of the resulting
      lshift for "<" and ">=" comparisons.  */
   (if (cmp == LT_EXPR || cmp == GE_EXPR)
    (cmp @0 (lshift @2 @1))

     /* For "<=" and ">" set the lower @1 lshift bits.  */
     (with {
	tree type2 = TREE_TYPE (@2);
	unsigned prec = TYPE_PRECISION (type2);
	unsigned mask_len = tree_to_uhwi (@1);
	wide_int cst1_mask = wi::mask (mask_len, false, prec);
      }
       (cmp @0 (bit_ior (lshift @2 @1)
			{ wide_int_to_tree (type2, cst1_mask); })))))))
#endif

/* Rewrite an LROTATE_EXPR by a constant into an
   RROTATE_EXPR by a new constant.  */
(simplify
 (lrotate @0 INTEGER_CST@1)
 (rrotate @0 { const_binop (MINUS_EXPR, TREE_TYPE (@1),
			    build_int_cst (TREE_TYPE (@1),
					   element_precision (type)), @1); }))

/* a rrotate (bitsize-b) -> a lrotate b */
/* a lrotate (bitsize-b) -> a rrotate b */
/* Only do the above when it is known that b != bitsize.
   Otherwise canonicalize to a orotate (b & mask) when the subtraction
   has single use and prec is a power of two.  */
(for rotate (lrotate rrotate)
     orotate (rrotate lrotate)
 (simplify
  (rotate @0 (minus@3 INTEGER_CST@1 @2))
  (with { auto prec = element_precision (TREE_TYPE (@0)); }
   (if (prec == wi::to_wide (@1))
    (switch
     (if (expr_not_equal_to (@2, wi::uhwi (prec,
					   TYPE_PRECISION (TREE_TYPE (@2))),
			     gimple_match_ctx (@3)))
      (orotate @0 @2))
     (if (single_use (@3)
	  && pow2p_hwi (prec)
	  /* Defer this case until last forwprop, so that VRP could be run and
	     expr_not_equal_to had a chance to match.  Otherwise we'd do
	     pretty much always just the second case.  */
	  && cfun
	  && (fold_before_rtl_expansion_p ()
	      || !flag_tree_vrp
	      || optimize_debug))
      (orotate @0
	(bit_and @2 { build_int_cst (TREE_TYPE (@2), prec - 1); }))))))))

/* Turn (a OP c1) OP c2 into a OP (c1+c2).  */
(for op (lrotate rrotate rshift lshift)
 (simplify
  (op (op @0 INTEGER_CST@1) INTEGER_CST@2)
  (with { unsigned int prec = element_precision (type); }
   (if (wi::ge_p (wi::to_wide (@1), 0, TYPE_SIGN (TREE_TYPE (@1)))
        && wi::lt_p (wi::to_wide (@1), prec, TYPE_SIGN (TREE_TYPE (@1)))
        && wi::ge_p (wi::to_wide (@2), 0, TYPE_SIGN (TREE_TYPE (@2)))
	&& wi::lt_p (wi::to_wide (@2), prec, TYPE_SIGN (TREE_TYPE (@2))))
    (with { unsigned int low = (tree_to_uhwi (@1)
				+ tree_to_uhwi (@2)); }
     /* Deal with a OP (c1 + c2) being undefined but (a OP c1) OP c2
        being well defined.  */
     (if (low >= prec)
      (if (op == LROTATE_EXPR || op == RROTATE_EXPR)
       (op @0 { build_int_cst (TREE_TYPE (@1), low % prec); })
       (if (TYPE_UNSIGNED (type) || op == LSHIFT_EXPR)
        { build_zero_cst (type); }
        (op @0 { build_int_cst (TREE_TYPE (@1), prec - 1); })))
      (op @0 { build_int_cst (TREE_TYPE (@1), low); })))))))

/* Fold x | (x >> (TYPE_PRECISION (type) - 1)) to max (x, -1)
   Fold x | (~ (x >> (TYPE_PRECISION (type) - 1))) to min (x, -1)
   Fold x & (x >> (TYPE_PRECISION (type) - 1)) to min (x, 0)
   Fold x & (~ (x >> (TYPE_PRECISION (type) - 1))) to max (x, 0)
*/
#if GIMPLE
(for bitop (bit_ior bit_and)
     result (max min)
     rresult (min max)
 (simplify
  (bitop:c @0 (rshift @0 INTEGER_CST@1))
  (if (INTEGRAL_TYPE_P (type)
       && !TYPE_UNSIGNED (type)
       && wi::eq_p (wi::to_wide (@1), TYPE_PRECISION (type) - 1))
   (result @0 { build_int_cst (type, bitop == BIT_IOR_EXPR ? -1 : 0); })))
 (simplify
  (bitop:c @0 (bit_not (rshift @0 INTEGER_CST@1)))
  (if (INTEGRAL_TYPE_P (type)
       && !TYPE_UNSIGNED (type)
       && wi::eq_p (wi::to_wide (@1), TYPE_PRECISION (type) - 1))
   (rresult @0 { build_int_cst (type, bitop == BIT_IOR_EXPR ? -1 : 0); }))))
#endif

/* Complete the branchless min and max the rules above leave half folded,
   and the same identity with the addition inside the select:

     Y + MIN (X - Y, 0) -> MIN (X, Y)    X - MIN (X - Y, 0) -> MAX (X, Y)
     Y + MAX (X - Y, 0) -> MAX (X, Y)    X - MAX (X - Y, 0) -> MIN (X, Y)
     MIN (X + Y, Y) - Y -> MIN (X, 0)    MAX (X + Y, Y) - Y -> MAX (X, 0)  */
(for minmax (min max)
     maxmin (max min)
 (simplify
  (plus:c (minmax (minus@3 @0 @1) integer_zerop) @1)
  (if (ANY_INTEGRAL_TYPE_P (type)
       && gimple_match_no_overflow_p (MINUS_EXPR, type, @0, @1, @3))
   (minmax @0 @1)))
 (simplify
  (minus @0 (minmax (minus@3 @0 @1) integer_zerop))
  (if (ANY_INTEGRAL_TYPE_P (type)
       && gimple_match_no_overflow_p (MINUS_EXPR, type, @0, @1, @3))
   (maxmin @0 @1)))
 (simplify
  (minus (minmax:c (plus:c@3 @0 @1) @1) @1)
  (if (ANY_INTEGRAL_TYPE_P (type)
       && gimple_match_no_overflow_p (PLUS_EXPR, type, @0, @1, @3))
   (minmax @0 { build_zero_cst (type); }))))

/* Fold `1 >> a` into `a == 0` for scalar integral types. */
(simplify
 (rshift integer_onep @2)
 (if (INTEGRAL_TYPE_P (type))
  (convert (eq:boolean_type_node @2 { build_zero_cst (TREE_TYPE (@2)); }))))

/* Simplify (CST << x) & 1 to 0 if CST is even or to x == 0 if it is odd.  */
(simplify
 (bit_and (lshift INTEGER_CST@1 @0) integer_onep)
  (if ((wi::to_wide (@1) & 1) != 0)
   (convert (eq:boolean_type_node @0 { build_zero_cst (TREE_TYPE (@0)); }))
   { build_zero_cst (type); }))

/* Simplify ((C << x) & D) != 0 where C and D are power of two constants,
   either to false if D is smaller (unsigned comparison) than C, or to
   x == log2 (D) - log2 (C).  Similarly for right shifts.
   Note `(1 >> x)`, the & 1 has been removed so but will have been
   already folded above.  */
(for cmp (ne eq)
     icmp (eq ne)
 (simplify
  (cmp (bit_and (lshift integer_pow2p@1 @0) integer_pow2p@2) integer_zerop)
   (with { int c1 = wi::clz (wi::to_wide (@1));
	   int c2 = wi::clz (wi::to_wide (@2)); }
    (if (c1 < c2
	 /* If c1 - c2 isn't representable in TREE_TYPE (@0), it is also
	    never true, because for any valid x C << x will be smaller
	    than D.  See PR126476.  */
	 || !wi::fits_to_tree_p (wi::shwi (c1 - c2, HOST_BITS_PER_INT),
				 TREE_TYPE (@0)))
     { constant_boolean_node (cmp != NE_EXPR, type); }
     (icmp @0 { build_int_cst (TREE_TYPE (@0), c1 - c2); }))))
 (simplify
  (cmp (bit_and (rshift integer_pow2p@1 @0) integer_pow2p@2) integer_zerop)
   (if (tree_int_cst_sgn (@1) > 0)
    (with { int c1 = wi::clz (wi::to_wide (@1));
	    int c2 = wi::clz (wi::to_wide (@2)); }
     (if (c1 > c2
	  || !wi::fits_to_tree_p (wi::shwi (c2 - c1, HOST_BITS_PER_INT),
				  TREE_TYPE (@0)))
      { constant_boolean_node (cmp != NE_EXPR, type); }
      (icmp @0 { build_int_cst (TREE_TYPE (@0), c2 - c1); }))))))

/* (CST1 << A) == CST2 -> A == ctz (CST2) - ctz (CST1)
   (CST1 << A) != CST2 -> A != ctz (CST2) - ctz (CST1)
   if CST2 != 0.  */
(for cmp (ne eq)
 (simplify
  (cmp (lshift INTEGER_CST@0 @1) INTEGER_CST@2)
  (with { int cand = wi::ctz (wi::to_wide (@2)) - wi::ctz (wi::to_wide (@0)); }
   (if (cand < 0
	|| (!integer_zerop (@2)
	    && wi::lshift (wi::to_wide (@0), cand) != wi::to_wide (@2))
	|| !wi::fits_to_tree_p (wi::shwi (cand, HOST_BITS_PER_INT),
				TREE_TYPE (@1)))
    { constant_boolean_node (cmp == NE_EXPR, type); }
    (if (!integer_zerop (@2)
	 && wi::lshift (wi::to_wide (@0), cand) == wi::to_wide (@2))
     (cmp @1 { build_int_cst (TREE_TYPE (@1), cand); }))))))

/* Fold ((X + C1) & LOWMASK) ==/!= C2 into (X & LOWMASK) ==/!= C2 - C1.
   Only the low bits of the addition survive the mask, so the constant can
   move to the other side of the comparison and the addition goes away.  */
(for cmp (eq ne)
 (simplify
  (cmp (bit_and:s (plus:s @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
       && wi::popcount (wi::to_wide (@2) + 1) == 1)
   (with { wide_int mask = wi::to_wide (@2);
	   wide_int rhs = (wi::to_wide (@3) - wi::to_wide (@1)) & mask; }
    (if ((wi::to_wide (@3) & ~mask) == 0)
     (cmp (bit_and @0 @2)
	  { wide_int_to_tree (TREE_TYPE (@0), rhs); })
     { constant_boolean_node (cmp == NE_EXPR, type); })))))

/* Fold (X & HIGHMASK) >/<= C into X >/<= (C | ~HIGHMASK).  Clearing the
   low bits of X rounds it down, and a rounded value passes the comparison
   exactly when X passes it against the constant with those bits set.  */
(for cmp (gt le)
 (simplify
  (cmp (bit_and:s @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && bitmask_inv_cst_vector_p (@1))
   (with { wide_int c = wi::to_wide (@2) | wi::bit_not (wi::to_wide (@1)); }
    (cmp @0 { wide_int_to_tree (TREE_TYPE (@0), c); })))))

/* Fold ((X << C1) & C2) cmp C3 into (X & (C2 >> C1)) cmp (C3 >> C1)
	((X >> C1) & C2) cmp C3 into (X & (C2 << C1)) cmp (C3 << C1).  */
(for cmp (ne eq)
 (simplify
  (cmp (bit_and:s (lshift:s @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
  (if (tree_fits_shwi_p (@1)
       && tree_to_shwi (@1) > 0
       && tree_to_shwi (@1) < TYPE_PRECISION (TREE_TYPE (@0)))
    (if (tree_to_shwi (@1) > wi::ctz (wi::to_wide (@3)))
      { constant_boolean_node (cmp == NE_EXPR, type); }
      (with { wide_int c1 = wi::to_wide (@1);
	      wide_int c2 = wi::lrshift (wi::to_wide (@2), c1);
	      wide_int c3 = wi::lrshift (wi::to_wide (@3), c1); }
	(cmp (bit_and @0 { wide_int_to_tree (TREE_TYPE (@0), c2); })
	     { wide_int_to_tree (TREE_TYPE (@0), c3); })))))
 (simplify
  (cmp (bit_and:s (rshift:s @0 INTEGER_CST@1) INTEGER_CST@2) INTEGER_CST@3)
  (if (tree_fits_shwi_p (@1)
       && tree_to_shwi (@1) > 0
       && tree_to_shwi (@1) < TYPE_PRECISION (TREE_TYPE (@0)))
    (with { tree t0 = TREE_TYPE (@0);
	    unsigned int prec = TYPE_PRECISION (t0);
	    wide_int c1 = wi::to_wide (@1);
	    wide_int c2 = wi::to_wide (@2);
	    wide_int c3 = wi::to_wide (@3);
	    wide_int sb = wi::set_bit_in_zero (prec - 1, prec); }
      (if ((c2 & c3) != c3)
        { constant_boolean_node (cmp == NE_EXPR, type); }
	(if (TYPE_UNSIGNED (t0))
	  (if ((c3 & wi::arshift (sb, c1 - 1)) != 0)
            { constant_boolean_node (cmp == NE_EXPR, type); }
	    (cmp (bit_and @0 { wide_int_to_tree (t0, c2 << c1); })
		 { wide_int_to_tree (t0, c3 << c1); }))
	  (with { wide_int smask = wi::arshift (sb, c1); }
	    (switch
	      (if ((c2 & smask) == 0)
		(cmp (bit_and @0 { wide_int_to_tree (t0, c2 << c1); })
		     { wide_int_to_tree (t0, c3 << c1); }))
	      (if ((c3 & smask) == 0)
	        (cmp (bit_and @0 { wide_int_to_tree (t0, (c2 << c1) | sb); })
		     { wide_int_to_tree (t0, c3 << c1); }))
	      (if ((c2 & smask) != (c3 & smask))
		{ constant_boolean_node (cmp == NE_EXPR, type); })
	      (cmp (bit_and @0 { wide_int_to_tree (t0, (c2 << c1) | sb); })
		   { wide_int_to_tree (t0, (c3 << c1) | sb); })))))))))

/* Fold (X << C1) & C2 into (X << C1) & (C2 | ((1 << C1) - 1))
        (X >> C1) & C2 into (X >> C1) & (C2 | ~((type) -1 >> C1))
   if the new mask might be further optimized.  */
(for shift (lshift rshift)
 (simplify
  (bit_and (convert?:s@4 (shift:s@5 (convert1?@3 @0) INTEGER_CST@1))
           INTEGER_CST@2)
   (if (tree_nop_conversion_p (TREE_TYPE (@4), TREE_TYPE (@5))
	&& tree_fits_uhwi_p (@1)
	&& tree_to_uhwi (@1) > 0
	&& tree_to_uhwi (@1) < TYPE_PRECISION (type))
    (with
     {
       unsigned int shiftc = tree_to_uhwi (@1);
       wide_int mask = wi::to_wide (@2);
       wide_int newmask, zerobits = wi::zero (TYPE_PRECISION (type));
       tree shift_type = TREE_TYPE (@3);
       unsigned int prec;

       if (shift == LSHIFT_EXPR)
	 zerobits = wi::mask (shiftc, false, TYPE_PRECISION (type));
       else if (shift == RSHIFT_EXPR
		&& type_has_mode_precision_p (shift_type))
	 {
	   prec = TYPE_PRECISION (TREE_TYPE (@3));
	   tree arg00 = @0;
	   /* See if more bits can be proven as zero because of
	      zero extension.  */
	   if (@3 != @0
	       && TYPE_UNSIGNED (TREE_TYPE (@0)))
	     {
	       tree inner_type = TREE_TYPE (@0);
	       if (type_has_mode_precision_p (inner_type)
		   && TYPE_PRECISION (inner_type) < prec)
		 {
		   prec = TYPE_PRECISION (inner_type);
		   /* See if we can shorten the right shift.  */
		   if (shiftc < prec)
		     shift_type = inner_type;
		   /* Otherwise X >> C1 is all zeros, so we'll optimize
		      it into (X, 0) later on by making sure zerobits
		      is all ones.  */
		 }
	     }
	   zerobits = wi::minus_one (TYPE_PRECISION (type));
	   if (shiftc < prec)
	      zerobits = wi::mask (prec - shiftc, true, TYPE_PRECISION (type));
	   /* For arithmetic shift if sign bit could be set, zerobits
	      can contain actually sign bits, so no transformation is
	      possible, unless MASK masks them all away.  In that
	      case the shift needs to be converted into logical shift.  */
	   if (!TYPE_UNSIGNED (TREE_TYPE (@3))
	       && prec == TYPE_PRECISION (TREE_TYPE (@3)))
	     {
	       if ((mask & zerobits) == wi::zero (TYPE_PRECISION (type)))
		 shift_type = unsigned_type_for (TREE_TYPE (@3));
	       else
		 zerobits = wi::zero (TYPE_PRECISION (type));
	     }
	 }
     }
     /* ((X << 16) & 0xff00) is (X, 0).  */
     (if ((mask & zerobits) == mask)
      { build_int_cst (type, 0); }
      (with { newmask = (mask | zerobits);
	      /* Only do the transformation if NEWMASK is some integer mode's
		 mask, i.e.  an all-ones value (2^n - 1) where n is a power of
		 2 precision >= BITS_PER_UNIT.  We check this by testing
		 whether newmask + 1 is a power of two via wi::exact_log2.  */
	      int lg;
	      /* newmask + 1 overflows at full type precision (e.g.  128-bit
		 all-ones), so handle that case directly.  */
	      if (newmask == wi::minus_one (TYPE_PRECISION (type)))
		lg = TYPE_PRECISION (type);
	      else
		lg = wi::exact_log2 (newmask + 1);
	      unsigned int newmask_prec = lg >= 0 ? (unsigned int) lg : 0; }
       (if (newmask != mask
	    && lg > 0 && (newmask_prec & (newmask_prec - 1)) == 0
	    && newmask_prec >= BITS_PER_UNIT)
	  (with
	   { tree newmaskt = wide_int_to_tree (TREE_TYPE (@2), newmask); }
	   (if (!tree_int_cst_equal (newmaskt, @2))
	    (if (shift_type != TREE_TYPE (@3))
	     (bit_and (convert (shift:shift_type (convert @3) @1)) { newmaskt; })
	     (bit_and @4 { newmaskt; })))))))))))

/* ((1 << n) & M) != 0  -> n == log2 (M) */
(for cmp (ne eq)
       icmp (eq ne)
 (simplify
  (cmp
   (bit_and
    (nop_convert? (lshift integer_onep @0)) integer_pow2p@1) integer_zerop)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (with { int c = wi::exact_log2 (wi::to_wide (@1)); }
    (if (wi::fits_to_tree_p (wi::shwi (c, HOST_BITS_PER_INT), TREE_TYPE (@0)))
     (icmp @0 { wide_int_to_tree (TREE_TYPE (@0), c); }))))))

/* Fold (X {&,^,|} C2) << C1 into (X << C1) {&,^,|} (C2 << C1)
   (X {&,^,|} C2) >> C1 into (X >> C1) & (C2 >> C1).  */
(for shift (lshift rshift)
 (for bit_op (bit_and bit_xor bit_ior)
  (simplify
   (shift (convert?:s (bit_op:s @0 INTEGER_CST@2)) INTEGER_CST@1)
   (if (tree_nop_conversion_p (type, TREE_TYPE (@0)))
    (with { tree mask = int_const_binop (shift, fold_convert (type, @2), @1); }
     (if (mask)
      (bit_op (shift (convert @0) @1) { mask; })))))))

/* Fold ((X >> C1) & C2) * (1 << C1) into X & (C2 << C1), including cases where
   the & happens in a different type.  It is the conversion case that isn't
   a composition of other folds.

   Let the type of the * and >> be T1 and the type of the & be T2.
   The fold is valid if the conversion to T2 preserves all information;
   that is, if T2 is wider than T1 or drops no more than C1 bits from T1.
   In that case, the & might operate on bits that are dropped by the
   later conversion to T1 and the multiplication by (1 << C1), but those
   bits are also dropped by ANDing with C2 << C1 (converted to T1).

   If the conversion to T2 is not information-preserving, we have to be
   careful about the later conversion to T1 acting as a sign extension.
   We need either T2 to be unsigned or the top (sign) bit of C2 to be clear.
   That is equivalent to testing whether C2 is nonnegative.  */
(simplify
 (mult (convert? (bit_and (convert? (rshift @0 INTEGER_CST@1)) INTEGER_CST@2))
       INTEGER_CST@3)
 (if (tree_nop_conversion_p (type, TREE_TYPE (@0))
      && wi::ltu_p (wi::to_widest (@1), TYPE_PRECISION (type)))
  (with { unsigned int shift = tree_to_uhwi (@1);
	  unsigned int prec = TYPE_PRECISION (type); }
   (if ((prec <= TYPE_PRECISION (TREE_TYPE (@2)) + shift
	 || tree_int_cst_sgn (@2) >= 0)
	&& wi::to_wide (@3) == wi::set_bit_in_zero (shift, prec))
    (with { auto mask = wide_int::from (wi::to_wide (@2), prec, UNSIGNED); }
     (bit_and (convert @0) { wide_int_to_tree (type, mask << shift); }))))))

/* ~(~X >> Y) -> X >> Y (for arithmetic shift).  */
(simplify
 (bit_not (convert1?:s (rshift:s (convert2?@0 (bit_not @1)) @2)))
  (if (!TYPE_UNSIGNED (TREE_TYPE (@0))
       && (element_precision (TREE_TYPE (@0))
	   <= element_precision (TREE_TYPE (@1))
	   || !TYPE_UNSIGNED (TREE_TYPE (@1))))
   (with
    { tree shift_type = TREE_TYPE (@0); }
     (convert (rshift (convert:shift_type @1) @2)))))

/* ~(~X >>r Y) -> X >>r Y
   ~(~X <<r Y) -> X <<r Y */
(for rotate (lrotate rrotate)
 (simplify
  (bit_not (convert1?:s (rotate:s (convert2?@0 (bit_not @1)) @2)))
   (if ((element_precision (TREE_TYPE (@0))
	 <= element_precision (TREE_TYPE (@1))
	 || !TYPE_UNSIGNED (TREE_TYPE (@1)))
        && (element_precision (type) <= element_precision (TREE_TYPE (@0))
	    || !TYPE_UNSIGNED (TREE_TYPE (@0))))
    (with
     { tree rotate_type = TREE_TYPE (@0); }
      (convert (rotate (convert:rotate_type @1) @2))))))

(for cmp (eq ne)
 (for rotate (lrotate rrotate)
      invrot (rrotate lrotate)
  /* (X >>r Y) cmp (Z >>r Y) may simplify to X cmp Y. */
  (simplify
   (cmp (rotate @1 @0) (rotate @2 @0))
   (cmp @1 @2))
  /* (X >>r C1) cmp C2 may simplify to X cmp C3. */
  (simplify
   (cmp (rotate @0 INTEGER_CST@1) INTEGER_CST@2)
   (cmp @0 { const_binop (invrot, TREE_TYPE (@0), @2, @1); }))
  /* (X >>r Y) cmp C where C is 0 or ~0, may simplify to X cmp C.  */
  (simplify
   (cmp (rotate @0 @1) INTEGER_CST@2)
    (if (integer_zerop (@2) || integer_all_onesp (@2))
     (cmp @0 @2)))))

/* Narrow a lshift by constant.  */
(simplify
 (convert (lshift:s@0 @1 INTEGER_CST@2))
 (if (INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && !integer_zerop (@2)
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0)))
  (if (TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (@0))
       || wi::ltu_p (wi::to_wide (@2), TYPE_PRECISION (type)))
   (lshift (convert @1) @2)
   (if (wi::ltu_p (wi::to_wide (@2), TYPE_PRECISION (TREE_TYPE (@0))))
    { build_zero_cst (type); }))))

/* Simplifications of conversions.  */

/* Basic strip-useless-type-conversions / strip_nops.  */
(for cvt (convert view_convert float fix_trunc)
 (simplify
  (cvt @0)
  (if ((GIMPLE && useless_type_conversion_p (type, TREE_TYPE (@0)))
       || (GENERIC && type == TREE_TYPE (@0)))
   @0)))

/* Contract view-conversions.  */
(simplify
  (view_convert (view_convert @0))
  (view_convert @0))

/* Squash view_converts of BFRs if no precision is lost.  */
(simplify
  (view_convert (BIT_FIELD_REF@0 @1 @2 @3))
  (if (is_gimple_reg_type (type)
       && (!INTEGRAL_TYPE_P (type)
	   || type_has_mode_precision_p (type))
       && (!INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   || type_has_mode_precision_p (TREE_TYPE (@0))))
   (BIT_FIELD_REF:type @1 @2 @3)))

/* For integral conversions with the same precision or pointer
   conversions use a NOP_EXPR instead.  */
(simplify
  (view_convert @0)
  (if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
       && (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
       && TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (@0)))
   (convert @0)))

/* Strip inner integral conversions that do not change precision or size, or
   zero-extend while keeping the same size (for bool-to-char).
   However, keep this conversion if the result is an extended _BitInt,
   since it may rely on this conversion to extend properly.  */

(simplify
  (view_convert (convert@0 @1))
  (with {
    bool extended_bitint = false;
    if (BITINT_TYPE_P (TREE_TYPE (@0)))
      {
	struct bitint_info info;
	extended_bitint
	  = targetm.c.bitint_type_info (TYPE_PRECISION (TREE_TYPE (@0)),
							&info);
	extended_bitint = extended_bitint && info.extended;
      }
   }
  (if ((INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
       && (INTEGRAL_TYPE_P (TREE_TYPE (@1)) || POINTER_TYPE_P (TREE_TYPE (@1)))
       && !extended_bitint
       && TYPE_SIZE (TREE_TYPE (@0)) == TYPE_SIZE (TREE_TYPE (@1))
       && (TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (TREE_TYPE (@1))
	   || (TYPE_PRECISION (TREE_TYPE (@0)) > TYPE_PRECISION (TREE_TYPE (@1))
	       && TYPE_UNSIGNED (TREE_TYPE (@1)))))
   (view_convert @1))))

/* Simplify a view-converted empty or single-element constructor.  */
(simplify
  (view_convert CONSTRUCTOR@0)
  (with
   { tree ctor = (TREE_CODE (@0) == SSA_NAME
		  ? gimple_assign_rhs1 (SSA_NAME_DEF_STMT (@0)) : @0); }
   (switch
    (if (CONSTRUCTOR_NELTS (ctor) == 0)
     { build_zero_cst (type); })
    (if (CONSTRUCTOR_NELTS (ctor) == 1
	 && VECTOR_TYPE_P (TREE_TYPE (ctor))
	 && operand_equal_p (TYPE_SIZE (type),
			     TYPE_SIZE (TREE_TYPE
			       (CONSTRUCTOR_ELT (ctor, 0)->value))))
     (view_convert { CONSTRUCTOR_ELT (ctor, 0)->value; })))))

/* Re-association barriers around constants and other re-association
   barriers can be removed.  */
(simplify
 (paren CONSTANT_CLASS_P@0)
 @0)
(simplify
 (paren (paren@1 @0))
 @1)

/* Handle cases of two conversions in a row.  */
(for ocvt (convert float fix_trunc)
 (for icvt (convert float)
  (simplify
   (ocvt (icvt@1 @0))
   (with
    {
      tree inside_type = TREE_TYPE (@0);
      tree inter_type = TREE_TYPE (@1);
      int inside_int = ANY_INTEGRAL_TYPE_P (inside_type);
      int inside_ptr = POINTER_TYPE_P (inside_type);
      int inside_float = FLOAT_TYPE_P (inside_type);
      unsigned int inside_prec = element_precision (inside_type);
      int inside_unsignedp = TYPE_UNSIGNED (inside_type);
      int inter_int = ANY_INTEGRAL_TYPE_P (inter_type);
      int inter_ptr = POINTER_TYPE_P (inter_type);
      int inter_float = FLOAT_TYPE_P (inter_type);
      unsigned int inter_prec = element_precision (inter_type);
      int inter_unsignedp = TYPE_UNSIGNED (inter_type);
      int final_int = ANY_INTEGRAL_TYPE_P (type);
      int final_ptr = POINTER_TYPE_P (type);
      int final_float = FLOAT_TYPE_P (type);
      unsigned int final_prec = element_precision (type);
      int final_unsignedp = TYPE_UNSIGNED (type);
      int is_vec = VECTOR_TYPE_P (type);
      gcc_checking_assert (is_vec == VECTOR_TYPE_P (inside_type)
			   && is_vec == VECTOR_TYPE_P (inter_type));
    }
   (switch
    /* In addition to the cases of two conversions in a row
       handled below, if we are converting something to its own
       type via an object of identical or wider precision, neither
       conversion is needed.  */
    (if (((GIMPLE && useless_type_conversion_p (type, inside_type))
	  || (GENERIC
	      && TYPE_MAIN_VARIANT (type) == TYPE_MAIN_VARIANT (inside_type)))
	 && (((inter_int || inter_ptr) && final_int)
	     || (inter_float && final_float))
	 && inter_prec >= final_prec
	 && (! is_vec
	     || supportable_convert_operation (ocvt, type, TREE_TYPE (@0))))
     (ocvt @0))

    /* Likewise, if the intermediate and initial types are either both
       float or both integer, we don't need the middle conversion if the
       former is wider than the latter and doesn't change the signedness
       (for integers).  Avoid this if the final type is a pointer since
       then we sometimes need the middle conversion.  */
    (if (((inter_int && inside_int) || (inter_float && inside_float))
	 && (final_int || final_float)
	 && inter_prec >= inside_prec
	 && (inter_float || inter_unsignedp == inside_unsignedp)
	 && (! is_vec
	     || supportable_convert_operation (ocvt, type, TREE_TYPE (@0))))
     (ocvt @0))

    /* If we have a sign-extension of a zero-extended value, we can
       replace that by a single zero-extension.  Likewise if the
       final conversion does not change precision we can drop the
       intermediate conversion.  Similarly truncation of a sign-extension
       can be replaced by a single sign-extension.  */
    (if (inside_int && inter_int && final_int
	 && ((inside_prec < inter_prec && inter_prec < final_prec
	      && inside_unsignedp && !inter_unsignedp)
	     || final_prec == inter_prec
	     || (inside_prec < inter_prec && inter_prec > final_prec
		 && !inside_unsignedp && inter_unsignedp))
	 && (! is_vec
	     || supportable_convert_operation (ocvt, type, TREE_TYPE (@0))))
     (ocvt @0))

    /* Two conversions in a row are not needed unless:
	- some conversion is floating-point (overstrict for now), or
	- some conversion is a vector (overstrict for now), or
	- the intermediate type is narrower than both initial and
	  final, or
	- the intermediate type and innermost type differ in signedness,
	  and the outermost type is wider than the intermediate, or
	- the initial type is a pointer type and the precisions of the
	  intermediate and final types differ, or
	- the final type is a pointer type and the precisions of the
	  initial and intermediate types differ.  */
    (if (! inside_float && ! inter_float && ! final_float
	 && ! is_vec
	 && (inter_prec >= inside_prec || inter_prec >= final_prec)
	 && ! (inside_int && inter_int
	       && inter_unsignedp != inside_unsignedp
	       && inter_prec < final_prec)
	 && ((inter_unsignedp && inter_prec > inside_prec)
	     == (final_unsignedp && final_prec > inter_prec))
	 && ! (inside_ptr && inter_prec != final_prec)
	 && ! (final_ptr && inside_prec != inter_prec))
     (ocvt @0))

   /* `(outer:M)(inter:N) a:O`
      can be converted to `(outer:M) a`
      if M <= O && N >= O. No matter what signedness of the casts,
      as the final is either a truncation from the original or just
      a sign change of the type. */
   (if (inside_int && inter_int && final_int
        && final_prec <= inside_prec
	&& inter_prec >= inside_prec
	&& (! is_vec
	    || supportable_convert_operation (ocvt, type, TREE_TYPE (@0))))
    (convert @0))

    /* A truncation to an unsigned type (a zero-extension) should be
       canonicalized as bitwise and of a mask.  */
    (if (GIMPLE /* PR70366: doing this in GENERIC breaks -Wconversion.  */
	 && final_int && inter_int && inside_int
	 && ! is_vec
	 && final_prec >= inside_prec
	 && inside_prec > inter_prec
	 && inter_unsignedp)
     (convert (bit_and @0 { wide_int_to_tree
	                      (inside_type,
			       wi::mask (inter_prec, false,
					 TYPE_PRECISION (inside_type))); })))

    /* If we are converting an integer to a floating-point that can
       represent it exactly and back to an integer, we can skip the
       floating-point conversion.  PR66211, PR98984.  */
    (if (GIMPLE
	 && inside_int && inter_float && final_int
	 && ! is_vec
	 && (((unsigned) significand_size (TYPE_MODE (inter_type))
	      >= inside_prec - !inside_unsignedp)
	     || (! flag_trapping_math
		 && ((unsigned) significand_size (TYPE_MODE (inter_type))
		     >= final_prec))))
     (convert @0)))))))

/* (float_type)(integer_type) x -> trunc (x) if the type of x matches
   float_type.  Only do the transformation if we do not need to preserve
   trapping behaviour, so require !flag_trapping_math. */
#if GIMPLE
(simplify
   (float (fix_trunc @0))
   (if (!flag_trapping_math
	&& !HONOR_SIGNED_ZEROS (type)
	&& types_match (type, TREE_TYPE (@0))
	&& direct_internal_fn_supported_p (IFN_TRUNC, type,
					  OPTIMIZE_FOR_BOTH))
      (IFN_TRUNC @0)))
#endif

/* If we have a narrowing conversion to an integral type that is fed by a
   BIT_AND_EXPR, we might be able to remove the BIT_AND_EXPR if it merely
   masks off bits outside the final type (and nothing else).  */
(simplify
  (convert (bit_and @0 INTEGER_CST@1))
  (if (INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))
       && operand_equal_p (@1, build_low_bits_mask (TREE_TYPE (@1),
						    TYPE_PRECISION (type)), 0))
   (convert @0)))

#if GIMPLE
/* X / (1 << C) -> X /[ex] (1 << C) if the low C bits of X are clear.  */
(simplify
 (trunc_div with_possible_nonzero_bits@0 integer_pow2p@1)
 (if (INTEGRAL_TYPE_P (type)
      && !TYPE_UNSIGNED (type)
      && wi::multiple_of_p (get_nonzero_bits (@0), wi::to_wide (@1), SIGNED))
  (exact_div @0 @1)))
#endif

/* (X /[ex] A) * A -> X.  */
(simplify
  (mult (convert1? (exact_div @0 @@1)) (convert2? @1))
  (convert @0))

/* (X /[ex] C1) * (C1 * C2) -> X * C2.  */
(simplify
 (mult (convert? (exact_div @0 INTEGER_CST@1)) poly_int_tree_p@2)
 (with { poly_widest_int factor; }
  (if (multiple_p (wi::to_poly_widest (@2), wi::to_widest (@1), &factor))
   (mult (convert @0) { wide_int_to_tree (type, factor); }))))

/* Simplify (A / B) * B + (A % B) -> A.  */
(for div (trunc_div ceil_div floor_div round_div)
     mod (trunc_mod ceil_mod floor_mod round_mod)
  (simplify
   (plus:c (mult:c (div @0 @1) @1) (mod @0 @1))
   @0))

/* x / y * y == x -> x % y == 0, and likewise for !=.  */
(for cmp (eq ne)
 (simplify
  (cmp:c (mult:c (trunc_div:s @0 @1) @1) @0)
  (if (TREE_CODE (TREE_TYPE (@0)) != COMPLEX_TYPE
       && (!VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (@0)))
	   || !target_supports_op_p (TREE_TYPE (@0), TRUNC_DIV_EXPR,
				     optab_vector)
	   || target_supports_op_p (TREE_TYPE (@0), TRUNC_MOD_EXPR,
				    optab_vector)))
   (cmp (trunc_mod @0 @1) { build_zero_cst (TREE_TYPE (@0)); }))))

/* ((X /[ex] C1) +- C2) * (C1 * C3)  -->  (X * C3) +- (C1 * C2 * C3).  */
(for op (plus minus)
 (simplify
  (mult (convert1? (op (convert2? (exact_div @0 INTEGER_CST@1))
		       poly_int_tree_p@2))
	poly_int_tree_p@3)
  (with { poly_widest_int factor; }
   (if (tree_nop_conversion_p (type, TREE_TYPE (@2))
	&& tree_nop_conversion_p (TREE_TYPE (@0), TREE_TYPE (@2))
	&& multiple_p (wi::to_poly_widest (@3), wi::to_widest (@1), &factor))
    (with
      {
	wi::overflow_type overflow;
        wide_int mul;
      }
     (if (types_match (type, TREE_TYPE (@2))
	  && types_match (TREE_TYPE (@0), TREE_TYPE (@2))
	  && TREE_CODE (@2) == INTEGER_CST
	  && TREE_CODE (@3) == INTEGER_CST
	  && (mul = wi::mul (wi::to_wide (@2), wi::to_wide (@3),
			     TYPE_SIGN (type), &overflow),
	      !overflow)
	  && (TYPE_UNSIGNED (type)
	      /* Not using unsigned arithmetics is unsafe if factor
		 isn't 1 and if for op plus @0 and @2 could have different
		 sign or for op minus @0 and @2 could have the same sign.  */
	      || known_eq (factor, 1)
	      || (get_range_pos_neg (@0)
		  | (((op == PLUS_EXPR) ^ (tree_int_cst_sgn (@2) < 0))
		     ? 1 : 2)) != 3))
      (op (mult @0 { wide_int_to_tree (type, factor); })
	  { wide_int_to_tree (type, mul); })
      (with { tree utype = unsigned_type_for (type); }
       (convert (op (mult (convert:utype @0)
			  { wide_int_to_tree (utype, factor); })
		    (mult (convert:utype @3) (convert:utype @2)))))))))))

/* Canonicalization of binary operations.  */

/* Convert X + -C into X - C.  */
(simplify
 (plus @0 REAL_CST@1)
 (if (REAL_VALUE_NEGATIVE (TREE_REAL_CST (@1)))
  (with { tree tem = const_unop (NEGATE_EXPR, type, @1); }
   (if (!TREE_OVERFLOW (tem) || !flag_trapping_math)
    (minus @0 { tem; })))))

/* Convert x+x into x*2.  */
(simplify
 (plus @0 @0)
 (if (SCALAR_FLOAT_TYPE_P (type))
  (mult @0 { build_real (type, dconst2); })
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_PRECISION (type) > (2 - TYPE_UNSIGNED (type)))
   (mult @0 { build_int_cst (type, 2); }))))

/* 0 - X  ->  -X.  */
(simplify
 (minus integer_zerop @1)
 (negate @1))
(simplify
 (pointer_diff integer_zerop @1)
 (negate (convert @1)))

/* (0.0 - ARG1) can be transformed to -ARG1, if we don't honor NaNs
   and signed zeros or ARG1 is known to be non-zero.  Subtraction of
   NaN may signal or modify payload, but negation doesn't, so it is
   unsafe to apply this transformation for any kind of NaN.  When
   ARG1 is +0.0 or -0.0, the behaviour depends upon the rounding
   mode.  With the default rounding mode, (-0.0 - ARG1) is -ARG1,
   but (+0.0 - ARG1) is only -ARG1 if ARG1 cannot be +0.0.  */
(simplify
 (minus real_zerop@0 @1)
 (if (!tree_expr_maybe_nan_p (@1)
      && (!HONOR_SIGNED_ZEROS (type)
	  || tree_expr_nonzero_p (@1)
	  || (!flag_rounding_math
	      && real_negzerop (@0))))
  (negate @1)))

/* Transform x * -1 into -x.  */
(simplify
 (mult @0 integer_minus_onep)
 (negate @0))

/* Reassociate (X * CST) * Y to (X * Y) * CST.  This does not introduce
   signed overflow for CST != 0 && CST != -1.  */
(simplify
 (mult:c (mult:s@3 @0 INTEGER_CST@1) @2)
 (if (TREE_CODE (@2) != INTEGER_CST
      && single_use (@3)
      && !integer_zerop (@1) && !integer_minus_onep (@1))
  (mult (mult @0 @2) @1)))

/* True if we can easily extract the real and imaginary parts of a complex
   number.  */
(match compositional_complex
 (convert? (complex @0 @1)))

/* COMPLEX_EXPR and REALPART/IMAGPART_EXPR cancellations.  */
(simplify
 (complex (realpart @0) (imagpart @0))
 @0)
(simplify
 (realpart (complex @0 @1))
 @0)
(simplify
 (imagpart (complex @0 @1))
 @1)

/* Sometimes we only care about half of a complex expression.  */
(simplify
 (realpart (convert?:s (conj:s @0)))
 (convert (realpart @0)))
(simplify
 (imagpart (convert?:s (conj:s @0)))
 (convert (negate (imagpart @0))))
(for part (realpart imagpart)
 (for op (plus minus)
  (simplify
   (part (convert?:s@2 (op:s @0 @1)))
   (convert (op (part @0) (part @1))))))
(simplify
 (realpart (convert?@2 (CEXPI@1 @0)))
 (if (single_use (@1) && single_use (@2))
  (convert (COS @0))))
(simplify
 (imagpart (convert?@2 (CEXPI@1 @0)))
 (if (single_use (@1) && single_use (@2))
  (convert (SIN @0))))

/* conj(conj(x)) -> x  */
(simplify
 (conj (convert? (conj @0)))
 (if (tree_nop_conversion_p (TREE_TYPE (@0), type))
  (non_lvalue (convert @0))))

/* conj({x,y}) -> {x,-y}  */
(simplify
 (conj (convert?:s (complex:s @0 @1)))
 (with { tree itype = TREE_TYPE (type); }
  (complex (convert:itype @0) (negate (convert:itype @1)))))

/* BSWAP simplifications, transforms checked by gcc.dg/builtin-bswap-8.c.  */
(for bswap (BSWAP BITREVERSE)
 (simplify
  (bswap (bswap @0))
  @0)
 (simplify
  (bswap (bit_not (bswap @0)))
  (bit_not @0))
 (for bitop (bit_xor bit_ior bit_and)
  (simplify
   (bswap (bitop:c (bswap @0) @1))
   (bitop @0 (bswap @1))))
 (for cmp (eq ne)
  (simplify
   (cmp (bswap@2 @0) (bswap @1))
   (with { tree ctype = TREE_TYPE (@2); }
    (cmp (convert:ctype @0) (convert:ctype @1))))
  (simplify
   (cmp (bswap @0) INTEGER_CST@1)
   (with { tree ctype = TREE_TYPE (@1); }
    (cmp (convert:ctype @0) (bswap! @1))))))
(for bswap (BSWAP)
 /* (bswap(x) >> C1) & C2 can sometimes be simplified to (x >> C3) & C2.  */
 (simplify
  (bit_and (convert1? (rshift@0 (convert2? (bswap@4 @1)) INTEGER_CST@2))
	   INTEGER_CST@3)
   (if (BITS_PER_UNIT == 8
	&& tree_fits_uhwi_p (@2)
	&& tree_fits_uhwi_p (@3))
    (with
     {
      unsigned HOST_WIDE_INT prec = TYPE_PRECISION (TREE_TYPE (@4));
      unsigned HOST_WIDE_INT bits = tree_to_uhwi (@2);
      unsigned HOST_WIDE_INT mask = tree_to_uhwi (@3);
      unsigned HOST_WIDE_INT lo = bits & 7;
      unsigned HOST_WIDE_INT hi = bits - lo;
     }
     (if (bits < prec
	  && mask < (256u>>lo)
	  && bits < TYPE_PRECISION (TREE_TYPE(@0)))
      (with { unsigned HOST_WIDE_INT ns = (prec - (hi + 8)) + lo; }
       (if (ns == 0)
	(bit_and (convert @1) @3)
	(with
	 {
	  tree utype = unsigned_type_for (TREE_TYPE (@1));
	  tree nst = build_int_cst (integer_type_node, ns);
	 }
	 (bit_and (convert (rshift:utype (convert:utype @1) {nst;})) @3))))))))
 /* bswap(x) >> C1 can sometimes be simplified to (T)x >> C2.  */
 (simplify
  (rshift (convert? (bswap@2 @0)) INTEGER_CST@1)
   (if (BITS_PER_UNIT == 8
	&& CHAR_TYPE_SIZE == 8
	&& tree_fits_uhwi_p (@1))
    (with
     {
      unsigned HOST_WIDE_INT prec = TYPE_PRECISION (TREE_TYPE (@2));
      unsigned HOST_WIDE_INT bits = tree_to_uhwi (@1);
      /* If the bswap was extended before the original shift, this
	 byte (shift) has the sign of the extension, not the sign of
	 the original shift.  */
      tree st = TYPE_PRECISION (type) > prec ? TREE_TYPE (@2) : type;
     }
     /* Special case: logical right shift of sign-extended bswap.
	(unsigned)(short)bswap16(x)>>12 is (unsigned)((short)x<<8)>>12. */
     (if (TYPE_PRECISION (type) > prec
	  && !TYPE_UNSIGNED (TREE_TYPE (@2))
	  && TYPE_UNSIGNED (type)
	  && bits < prec && bits + 8 >= prec)
      (with { tree nst = build_int_cst (integer_type_node, prec - 8); }
       (rshift (convert (lshift:st (convert:st @0) {nst;})) @1))
      (if (bits + 8 == prec)
       (if (TYPE_UNSIGNED (st))
	(convert (convert:unsigned_char_type_node @0))
	(convert (convert:signed_char_type_node @0)))
       (if (bits < prec && bits + 8 > prec)
	(with
	 {
	  tree nst = build_int_cst (integer_type_node, bits & 7);
	  tree bt = TYPE_UNSIGNED (st) ? unsigned_char_type_node
				       : signed_char_type_node;
	 }
	 (convert (rshift:bt (convert:bt @0) {nst;})))))))))
 /* bswap(x) & C1 can sometimes be simplified to (x >> C2) & C1.  */
 (simplify
  (bit_and (convert? (bswap@2 @0)) INTEGER_CST@1)
   (if (BITS_PER_UNIT == 8
	&& tree_fits_uhwi_p (@1)
	&& tree_to_uhwi (@1) < 256)
    (with
     {
      unsigned HOST_WIDE_INT prec = TYPE_PRECISION (TREE_TYPE (@2));
      tree utype = unsigned_type_for (TREE_TYPE (@0));
      tree nst = build_int_cst (integer_type_node, prec - 8);
     }
     (bit_and (convert (rshift:utype (convert:utype @0) {nst;})) @1)))))


/* Combine COND_EXPRs and VEC_COND_EXPRs.  */

/* Simplify constant conditions.
   Only optimize constant conditions when the selected branch
   has the same type as the COND_EXPR.  This avoids optimizing
   away "c ? x : throw", where the throw has a void type.
   Note that we cannot throw away the fold-const.cc variant nor
   this one as we depend on doing this transform before possibly
   A ? B : B -> B triggers and the fold-const.cc one can optimize
   0 ? A : B to B even if A has side-effects.  Something
   genmatch cannot handle.  */
(simplify
 (cond INTEGER_CST@0 @1 @2)
 (if (integer_zerop (@0))
  (if (!VOID_TYPE_P (TREE_TYPE (@2)) || VOID_TYPE_P (type))
   @2)
  (if (!VOID_TYPE_P (TREE_TYPE (@1)) || VOID_TYPE_P (type))
   @1)))
(simplify
 (vec_cond VECTOR_CST@0 @1 @2)
 (if (integer_all_onesp (@0))
  @1
  (if (integer_zerop (@0))
   @2)))

/* Sink unary operations to branches, but only if we do fold both.  */
(for op (negate bit_not abs absu)
 (simplify
  (op (vec_cond:s @0 @1 @2))
  (vec_cond @0 (op!:type @1) (op!:type @2))))

/* Sink unary conversions to branches, but only if we do fold both
   and the target's truth type is the same as we already have.  */
(for op (convert float fix_trunc)
 (simplify
  (op (vec_cond:s @0 @1 @2))
  (if (VECTOR_TYPE_P (type)
       && types_match (TREE_TYPE (@0), truth_type_for (type))
       && (expand_vec_cond_expr_p (type, TREE_TYPE (@0))
	   || !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0))))
   (vec_cond @0 (op! @1) (op! @2)))))

/* Likewise for view_convert of nop_conversions. */
(simplify
 (view_convert (vec_cond:s @0 @1 @2))
 (if (VECTOR_TYPE_P (type) && VECTOR_TYPE_P (TREE_TYPE (@1))
      && known_eq (TYPE_VECTOR_SUBPARTS (type),
		   TYPE_VECTOR_SUBPARTS (TREE_TYPE (@1)))
      && tree_nop_conversion_p (TREE_TYPE (type), TREE_TYPE (TREE_TYPE (@1)))
      && (expand_vec_cond_expr_p (type, TREE_TYPE (@0))
	  || !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0))))
  (vec_cond @0 (view_convert! @1) (view_convert! @2))))

/* Sink binary operation to branches, but only if we can fold it.  */
(for op (tcc_comparison plus minus mult bit_and bit_ior bit_xor
	 lshift rshift rdiv trunc_div ceil_div floor_div round_div exact_div
	 trunc_mod ceil_mod floor_mod round_mod min max)
/* (c ? a : b) op (c ? d : e)  -->  c ? (a op d) : (b op e) */
 (simplify
  (op (vec_cond:s @0 @1 @2) (vec_cond:s @0 @3 @4))
  (if (VECTOR_TYPE_P (type)
       && (TREE_CODE_CLASS (op) != tcc_comparison
	   || types_match (type, TREE_TYPE (@1))
	   || expand_vec_cond_expr_p (type, TREE_TYPE (@0))
	   || (optimize_vectors_before_lowering_p ()
	       /* The following is optimistic on the side of non-support, we are
		  missing the legacy vcond{,u,eq} cases.  Do this only when
		  lowering will be able to fixup..  */
	       && !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0)))))
   (vec_cond @0 (op!:type @1 @3) (op!:type @2 @4))))

/* (c ? a : b) op d  -->  c ? (a op d) : (b op d) */
 (simplify
  (op (vec_cond:s @0 @1 @2) @3)
  (if (VECTOR_TYPE_P (type)
       && (TREE_CODE_CLASS (op) != tcc_comparison
	   || types_match (type, TREE_TYPE (@1))
	   || expand_vec_cond_expr_p (type, TREE_TYPE (@0))
	   || (optimize_vectors_before_lowering_p ()
	       && !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0)))))
   (vec_cond @0 (op!:type @1 @3) (op!:type @2 @3))))
 (simplify
  (op @3 (vec_cond:s @0 @1 @2))
  (if (VECTOR_TYPE_P (type)
       && (TREE_CODE_CLASS (op) != tcc_comparison
	   || types_match (type, TREE_TYPE (@1))
	   || expand_vec_cond_expr_p (type, TREE_TYPE (@0))
	   || (optimize_vectors_before_lowering_p ()
	       && !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0)))))
   (vec_cond @0 (op!:type @3 @1) (op!:type @3 @2)))))

/* (@0 ? @2 : @3) lop (@1 ? @2 : @3)  -->  (@0 lop @1) ? @2 : @3.  */
(for lop (bit_and bit_ior bit_xor)
   (simplify
   (lop
      (vec_cond @0 integer_minus_onep@2 integer_zerop@3)
      (vec_cond @1 @2 @3))
      (if (target_supports_op_p (TREE_TYPE (@0), lop, optab_vector))
   (vec_cond (lop @0 @1) @2 @3))))

/* If we have a "if a bit is not set, set it" case,
   just set the bit all the time (PR 64567).  Note that
   this does not work if we're checking for more than one
   bit, e.g. (a & 5 ? a | 5 : a) will fail for a = 1 (we
   would return 5 instead of 1).  */
(simplify
 (cond (eq (bit_and @0 INTEGER_CST@1) integer_zerop)
       (bit_ior@2 @0 INTEGER_CST@1) @0)
  (if (wi::popcount (wi::to_wide (@1)) == 1)
   @2))

/* A clear bit version of the above: "if a bitmask is
   set, clear it". In this case always clear the bitmask
   (see PR 64567).  */
(simplify
 (cond (ne (bit_and @0 INTEGER_CST@1) integer_zerop)
       (bit_and@3 @0 INTEGER_CST@2) @0)
  (if (wi::to_wide (@2) == ~wi::to_wide (@1))
   @3))

/* PR124667: Similar to PR64567 above but with the bit given
   by an (1 << val) lshift.  Note that this shift pattern
   guarantees that this is a single bit operation.

   The same pattern, if using '!=' instead of '==', will
   result in just 'A'.

   (A & (1 << val)) == 0 ? A |= (1 << val) : A => A |= (1 << val)
   (A & (1 << val)) != 0 ? A |= (1 << val) : A => A;  */
(for neeq (eq ne)
 (simplify
  (cond
   (neeq (bit_and:c @0 (lshift@1 integer_onep @2)) integer_zerop)
	 (bit_ior:c@3 @0 @1)
	 @0)
    (if (neeq == EQ_EXPR)
     @3
     @0))

/* PR124667: NE variant of the previous match with switched true/false
   legs:

   (A & (1 << val)) != 0 ? A : A |= (1 << val) => A |= (1 << val)
   (A & (1 << val)) == 0 ? A : A |= (1 << val) => A;  */
 (simplify
  (cond
   (neeq (bit_and:c @0 (lshift@1 integer_onep @2)) integer_zerop)
	 @0
	 (bit_ior:c@3 @0 @1))
    (if (neeq == NE_EXPR)
     @3
     @0))

/* PR124667: a bitmask set simplification that is possible as
   long as the whole bitmask is being checked.  Switching the
   comparison from '==' to '!=' results in 'A':

   (A & B) == B ? A : A |= B  => A |= B
   (A & B) != B ? A : A |= B  => A;  */
 (simplify
  (cond
   (neeq:c (bit_and @0 @1) @1)
	   @0
	   (bit_ior@2 @0 @1))
    (if (neeq == EQ_EXPR)
     @2
     @0))

/* PR124667: similar from above with true/false legs switched:

   (A & B) != B ? A |= B : A => A | B.
   (A & B) == B ? A |= B : A => A;  */
(simplify
 (cond
  (neeq:c (bit_and @0 @1) @1)
	  (bit_ior@2 @0 @1)
	  @0)
   (if (neeq == NE_EXPR)
    @2
    @0)))

/* PR124667: these also reduces to just "A":
   ((A >> val) & 1) == 0 ?  A : (A | (1 << val))
   ((A >> val) & 1) != 0 ?  (A | (1 << val)) : A;

   For both cases the zero_one patterns in lines 4860-4890
   will turn both into the following 'original':

   intD.7 bitshiftD.4489 = 1 << bitD.4485;
   arrD.4488 = arrD.4488 >> bitD.4485 & 1) * bitshiftD.4489 | arrD.4488;

   And it'll appear back in phiopt as:

   _1 = arrD.4482;
   _2 = _1 >> bit_7;
   _3 = _2 & 1;
   _4 = _3 << bit_7;
   _6 = _1 | _4;

   I.e. it'll drop the 'mult' and lshift the zero_one result from the
   bit check.  The 3 middle stmts can be reduced to:

   _3 = 1 << bit_7;
   _4 = _1 & _3;

   Resulting in:

   _1 = arrD.4482;
   _3 = 1 << bit_7;
   _4 = _1 & _3
   _6 = _1 | _4;

   Which will reduce to just _1.  Simplifying the middle stmts has the
   advantage of also benefitting other optimizations as well.  Note
   that this transformation is valid with all CST masks, not just
   '1'.  */
(simplify
  (lshift
    (bit_and:c (rshift @0 @1) INTEGER_CST@2)
     @1)
  (bit_and @0 (lshift @2 @1)))

#if GIMPLE
(match (nop_atomic_bit_test_and_p @0 @1 @4)
 (bit_and (convert?@4 (ATOMIC_FETCH_OR_XOR_N @2 INTEGER_CST@0 @3))
	   INTEGER_CST@1)
 (with {
	 int ibit = tree_log2 (@0);
	 int ibit2 = tree_log2 (@1);
       }
  (if (ibit == ibit2
      && ibit >= 0
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))))))

(match (nop_atomic_bit_test_and_p @0 @1 @3)
 (bit_and (convert?@3 (SYNC_FETCH_OR_XOR_N @2 INTEGER_CST@0))
	  INTEGER_CST@1)
 (with {
	 int ibit = tree_log2 (@0);
	 int ibit2 = tree_log2 (@1);
       }
  (if (ibit == ibit2
      && ibit >= 0
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))))))

(match (nop_atomic_bit_test_and_p @0 @0 @4)
 (bit_and:c
  (convert1?@4
   (ATOMIC_FETCH_OR_XOR_N @2 (nop_convert? (lshift@0 integer_onep@5 @6)) @3))
  (convert2? @0))
 (if (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0)))))

(match (nop_atomic_bit_test_and_p @0 @0 @4)
 (bit_and:c
  (convert1?@4
   (SYNC_FETCH_OR_XOR_N @2 (nop_convert? (lshift@0 integer_onep@3 @5))))
  (convert2? @0))
 (if (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0)))))

(match (nop_atomic_bit_test_and_p @0 @1 @3)
 (bit_and@4 (convert?@3 (ATOMIC_FETCH_AND_N @2 INTEGER_CST@0 @5))
	    INTEGER_CST@1)
 (with {
	 int ibit = wi::exact_log2 (wi::zext (wi::bit_not (wi::to_wide (@0)),
					      TYPE_PRECISION(type)));
	 int ibit2 = tree_log2 (@1);
       }
  (if (ibit == ibit2
      && ibit >= 0
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))))))

(match (nop_atomic_bit_test_and_p @0 @1 @3)
 (bit_and@4
  (convert?@3 (SYNC_FETCH_AND_AND_N @2 INTEGER_CST@0))
  INTEGER_CST@1)
 (with {
	 int ibit = wi::exact_log2 (wi::zext (wi::bit_not (wi::to_wide (@0)),
					      TYPE_PRECISION(type)));
	 int ibit2 = tree_log2 (@1);
       }
  (if (ibit == ibit2
      && ibit >= 0
      && TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@0))))))

(match (nop_atomic_bit_test_and_p @4 @0 @3)
 (bit_and:c
  (convert1?@3
   (ATOMIC_FETCH_AND_N @2 (nop_convert?@4 (bit_not (lshift@0 integer_onep@6 @7))) @5))
  (convert2? @0))
 (if (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@4)))))

(match (nop_atomic_bit_test_and_p @4 @0 @3)
 (bit_and:c
  (convert1?@3
   (SYNC_FETCH_AND_AND_N @2 (nop_convert?@4 (bit_not (lshift@0 integer_onep@6 @7)))))
  (convert2? @0))
  (if (TYPE_PRECISION (type) <= TYPE_PRECISION (TREE_TYPE (@4)))))

#endif

/* (v ? w : 0) ? a : b is just (v & w) ? a : b
   Currently disabled after pass lvec because ARM understands
   VEC_COND_EXPR<v==w,-1,0> but not a plain v==w fed to BIT_IOR_EXPR.  */
#if GIMPLE
/* These can only be done in gimple as fold likes to convert:
   (CMP) & N into (CMP) ? N : 0
   and we try to match the same pattern again and again. */
(simplify
 (vec_cond (vec_cond:s @0 @3 integer_zerop) @1 @2)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @3))
  (vec_cond (bit_and @0 @3) @1 @2)))
(simplify
 (vec_cond (vec_cond:s @0 integer_all_onesp @3) @1 @2)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @3))
  (vec_cond (bit_ior @0 @3) @1 @2)))
(simplify
 (vec_cond (vec_cond:s @0 integer_zerop @3) @1 @2)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @3))
  (vec_cond (bit_ior @0 (bit_not @3)) @2 @1)))
(simplify
 (vec_cond (vec_cond:s @0 @3 integer_all_onesp) @1 @2)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @3))
  (vec_cond (bit_and @0 (bit_not @3)) @2 @1)))

/*  ((VCE (a ? -1 : 0)) < 0) ? c : d is just
    (VCE (a ? (VCE c) : (VCE d))) when TYPE_PRECISION of the
    component type of the outer vec_cond is greater equal the inner one.  */
(simplify
 (vec_cond
  (lt (view_convert@5 (vec_cond@6 @4 integer_all_onesp integer_zerop))
       integer_zerop) @2 @3)
 (if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (@5))
      && !TYPE_UNSIGNED (TREE_TYPE (@5))
      && VECTOR_TYPE_P (TREE_TYPE (@6))
      && VECTOR_TYPE_P (type)
      && tree_int_cst_le (TYPE_SIZE (TREE_TYPE (type)),
			  TYPE_SIZE (TREE_TYPE (TREE_TYPE (@6))))
      && TYPE_SIZE (type) == TYPE_SIZE (TREE_TYPE (@6)))
  (with { tree vtype = TREE_TYPE (@6);}
   (view_convert:type
    (vec_cond @4 (view_convert:vtype @2) (view_convert:vtype @3))))))

/* c1 ? c2 ? a : b : b  -->  (c1 & c2) ? a : b  */
(simplify
 (vec_cond @0 (vec_cond:s @1 @2 @3) @3)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @1))
  (vec_cond (bit_and @0 @1) @2 @3)))
(simplify
 (vec_cond @0 @2 (vec_cond:s @1 @2 @3))
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @1))
  (vec_cond (bit_ior @0 @1) @2 @3)))
(simplify
 (vec_cond @0 (vec_cond:s @1 @2 @3) @2)
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @1))
  (vec_cond (bit_ior (bit_not @0) @1) @2 @3)))
(simplify
 (vec_cond @0 @3 (vec_cond:s @1 @2 @3))
 (if (optimize_vectors_before_lowering_p () && types_match (@0, @1))
  (vec_cond (bit_and (bit_not @0) @1) @2 @3)))
#endif

/* (a ? x : y) != (b ? x : y) --> (a^b & (x != y)) ? TRUE  : FALSE */
/* (a ? x : y) == (b ? x : y) --> (a^b & (x != y)) ? FALSE : TRUE  */
/* (a ? x : y) != (b ? y : x) --> (a^b | (x == y)) ? FALSE : TRUE  */
/* (a ? x : y) == (b ? y : x) --> (a^b | (x == y)) ? TRUE  : FALSE */
/* These are only valid if x and y don't have NaNs. */
(for cnd (cond vec_cond)
 (for eqne (eq ne)
  (simplify
   (eqne (cnd @0 @1 @2) (cnd @3 @1 @2))
    (if (!HONOR_NANS (@1)
	 && types_match (TREE_TYPE (@0), TREE_TYPE (@3))
         && types_match (type, TREE_TYPE (@0))
	 && expr_no_side_effects_p (@1)
	 && expr_no_side_effects_p (@2))
     (cnd (bit_and (bit_xor @0 @3) (ne:type @1 @2))
      { constant_boolean_node (eqne == NE_EXPR, type); }
      { constant_boolean_node (eqne != NE_EXPR, type); })))
  (simplify
   (eqne (cnd @0 @1 @2) (cnd @3 @2 @1))
    (if (!HONOR_NANS (@1)
	 && types_match (TREE_TYPE (@0), TREE_TYPE (@3))
         && types_match (type, TREE_TYPE (@0))
	 && expr_no_side_effects_p (@1)
	 && expr_no_side_effects_p (@2))
     (cnd (bit_ior (bit_xor @0 @3) (eq:type @1 @2))
      { constant_boolean_node (eqne != NE_EXPR, type); }
      { constant_boolean_node (eqne == NE_EXPR, type); })))))

/* Canonicalize mask ? { 0, ... } : { -1, ...} to ~mask if the mask
   types are compatible.  */
(simplify
 (vec_cond @0 VECTOR_CST@1 VECTOR_CST@2)
 (if (VECTOR_BOOLEAN_TYPE_P (type)
      && types_match (type, TREE_TYPE (@0)))
  (if (integer_zerop (@1) && integer_all_onesp (@2))
   (bit_not @0)
   (if (integer_all_onesp (@1) && integer_zerop (@2))
    @0))))

/* A few simplifications of "a ? CST1 : CST2". */
/* NOTE: Only do this on gimple as the if-chain-to-switch
   optimization depends on the gimple to have if statements in it. */
#if GIMPLE
(simplify
 (cond @0 INTEGER_CST@1 INTEGER_CST@2)
 (switch
  (if (integer_zerop (@2))
   (switch
    /* a ? 1 : 0 -> a if 0 and 1 are integral types.  */
    (if (integer_onep (@1))
     (convert (convert:boolean_type_node @0)))
    /* a ? -1 : 0 -> -a.  */
    (if (INTEGRAL_TYPE_P (type) && integer_all_onesp (@1))
     (if (TYPE_PRECISION (type) == 1)
      /* For signed 1-bit precision just cast bool to the type.  */
      (convert (convert:boolean_type_node @0))
      (if (TREE_CODE (type) == BOOLEAN_TYPE)
       (with {
	  tree intt = build_nonstandard_integer_type (TYPE_PRECISION (type),
						      TYPE_UNSIGNED (type));
	}
	(convert (negate (convert:intt (convert:boolean_type_node @0)))))
       (negate (convert:type (convert:boolean_type_node @0))))))
    /* a ? powerof2cst : 0 -> a << (log2(powerof2cst)) */
    (if (INTEGRAL_TYPE_P (type) && integer_pow2p (@1))
     (with {
       tree shift = build_int_cst (integer_type_node, tree_log2 (@1));
      }
      (lshift (convert (convert:boolean_type_node @0)) { shift; })))))
  (if (integer_zerop (@1))
   (switch
    /* a ? 0 : 1 -> !a.  */
    (if (integer_onep (@2))
     (convert (bit_xor (convert:boolean_type_node @0) { boolean_true_node; })))
    /* a ? 0 : -1 -> -(!a).  */
    (if (INTEGRAL_TYPE_P (type) && integer_all_onesp (@2))
     (if (TYPE_PRECISION (type) == 1)
      /* For signed 1-bit precision just cast bool to the type.  */
      (convert (bit_xor (convert:boolean_type_node @0) { boolean_true_node; }))
      (if (TREE_CODE (type) == BOOLEAN_TYPE)
       (with {
	  tree intt = build_nonstandard_integer_type (TYPE_PRECISION (type),
						      TYPE_UNSIGNED (type));
	}
	(convert (negate (convert:intt (bit_xor (convert:boolean_type_node @0)
						{ boolean_true_node; })))))
       (negate (convert:type (bit_xor (convert:boolean_type_node @0)
				      { boolean_true_node; }))))))
    /* a ? 0 : powerof2cst -> (!a) << (log2(powerof2cst)) */
    (if (INTEGRAL_TYPE_P (type) && integer_pow2p (@2))
     (with {
       tree shift = build_int_cst (integer_type_node, tree_log2 (@2));
      }
      (lshift (convert (bit_xor (convert:boolean_type_node @0)
				{ boolean_true_node; })) { shift; })))))))

/* Recognize a 2-arg PHI of the form
     PHI<base + pow2, base>
   guarded by an unsigned compare on (@0, @1).  @1 is captured without
   further structure; the long-mul fold in tree-ssa-forwprop.cc
   classifies it to drive a PHI-form carry summand.  The consumer
   treats the carry as strict @0 > @1, compares must encode that
   same condition:
     When the true edge selects (base + pow2), the
       compare itself must be @0 > @1.
     When the false edge selects (base + pow2), the
       compare itself must be @0 <= @1 or rather the invese of @0 > @1.  */
(if (INTEGRAL_TYPE_P (type)
     && TYPE_UNSIGNED (type)
     && type_has_mode_precision_p (type))
 (match (cond_carry_add @0 @1 @2 @3)
  (cond^ (gt:c @0 @1) (plus @2 integer_pow2p@3) @2))
 (match (cond_carry_add @0 @1 @2 @3)
  (cond^ (le:c @0 @1) @2 (plus @2 integer_pow2p@3))))

/* (a > 1) ? 0 : (cast)a is the same as (cast)(a == 1)
   for unsigned types. */
(simplify
 (cond (gt @0 integer_onep@1) integer_zerop (convert? @2))
 (if (TYPE_UNSIGNED (TREE_TYPE (@0))
      && bitwise_equal_p (@0, @2))
  (convert (eq @0 @1))
 )
)

/* (a <= 1) & (cast)a is the same as (cast)(a == 1)
   for unsigned types. */
(simplify
 (bit_and:c (convert1? (le @0 integer_onep@1)) (convert2? @2))
 (if (TYPE_UNSIGNED (TREE_TYPE (@0))
      && bitwise_equal_p (@0, @2))
  (convert (eq @0 @1))
 )
)

/* `(a == CST) & a` can be simplified to `0` or `(a == CST)` depending
   on the first bit of the CST.  */
(simplify
 (bit_and:c (convert@2 (eq @0 INTEGER_CST@1)) (convert? @0))
 (if ((wi::to_wide (@1) & 1) != 0)
  @2
  { build_zero_cst (type); }))

/* Optimize
   # x_5 in range [cst1, cst2] where cst2 = cst1 + 1
   x_5 == cstN ? cst4 : cst3
   # op is == or != and N is 1 or 2
   to r_6 = x_5 + (min (cst3, cst4) - cst1) or
   r_6 = (min (cst3, cst4) + cst1) - x_5 depending on op, N and which
   of cst3 and cst4 is smaller.
   This was originally done by two_value_replacement in phiopt (PR 88676).  */
(for eqne (ne eq)
 (simplify
  (cond (eqne@4 SSA_NAME@0 INTEGER_CST@1) INTEGER_CST@2 INTEGER_CST@3)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (type)
       && (wi::to_widest (@2) + 1 == wi::to_widest (@3)
           || wi::to_widest (@2) == wi::to_widest (@3) + 1))
   (with {
     int_range_max r;
     if (!gimple_match_range_of_expr (r, @0, @4))
       r.set_varying (TREE_TYPE (@0));

     wide_int min = r.lower_bound ();
     wide_int max = r.upper_bound ();
    }
    (if (min + 1 == max
	 && (wi::to_wide (@1) == min
	     || wi::to_wide (@1) == max))
     (with {
       tree arg0 = @2, arg1 = @3;
       tree type1;
       if ((eqne == EQ_EXPR) ^ (wi::to_wide (@1) == min))
         std::swap (arg0, arg1);
       if (TYPE_PRECISION (TREE_TYPE (@0)) > TYPE_PRECISION (type)
           // Prefer unsigned type if the two has the same precision.
           || (TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (type)
		&& !TYPE_UNSIGNED (type)))
	 type1 = TREE_TYPE (@0);
       else
	 type1 = type;
       auto prec = TYPE_PRECISION (type1);
       auto unsign = TYPE_UNSIGNED (type1);
       // For boolean types, use an integer type for the plus/minus.
       if (TREE_CODE (type1) == BOOLEAN_TYPE)
	type1 = build_nonstandard_integer_type (prec, unsign);
       min = wide_int::from (min, prec,
			     TYPE_SIGN (TREE_TYPE (@0)));
       wide_int a = wide_int::from (wi::to_wide (arg0), prec,
				    TYPE_SIGN (type));
       enum tree_code code;
       wi::overflow_type ovf;
       if (tree_int_cst_lt (arg0, arg1))
	 {
	   code = PLUS_EXPR;
	   a -= min;
	   if (!unsign)
	     {
	       /* lhs is known to be in range [min, min+1] and we want to add a
		  to it.  Check if that operation can overflow for those 2 values
		  and if yes, force unsigned type.  */
	       wi::add (min + (wi::neg_p (a) ? 0 : 1), a, SIGNED, &ovf);
	       if (ovf)
		 type1 = unsigned_type_for (type1);
	     }
	 }
       else
	 {
	   code = MINUS_EXPR;
	   a += min;
	   if (!unsign)
	     {
	       /* lhs is known to be in range [min, min+1] and we want to subtract
		  it from a.  Check if that operation can overflow for those 2
		  values and if yes, force unsigned type.  */
	       wi::sub (a, min + (wi::neg_p (min) ? 0 : 1), SIGNED, &ovf);
	       if (ovf)
		type1 = unsigned_type_for (type1);
	     }
	 }
       tree arg = wide_int_to_tree (type1, a);
      }
      (if (code == PLUS_EXPR)
       (convert (plus (convert:type1 @0) { arg; }))
       (convert (minus { arg; } (convert:type1 @0))))))))))
#endif

/* X & C1 ? (X + -C1) : (X | C1) -> X ^ C1
   X & C1 ? (X & ~C1) : (X | C1) -> X ^ C1
   when C1 has a single bit set.  */
(for op (plus bit_and)
 (simplify
  (cond (ne (bit_and @0 INTEGER_CST@1) integer_zerop)
	     (op @0 INTEGER_CST@2) (bit_ior @0 @1))
   (with {
     auto c1 = wi::to_wide (@1);
     auto c2 = wi::to_wide (@2);
    }
    (if (wi::popcount (c1) == 1
	  && ((op == PLUS_EXPR && wi::eq_p (wi::neg (c2), c1))
	       || (op == BIT_AND_EXPR && wi::eq_p (wi::bit_not (c2), c1))))
	  (bit_xor @0 @1)))))

(simplify
 (convert (cond@0 @1 INTEGER_CST@2 INTEGER_CST@3))
 (if (INTEGRAL_TYPE_P (type)
      && INTEGRAL_TYPE_P (TREE_TYPE (@0)))
  (cond @1 (convert @2) (convert @3))))

#if GIMPLE
/* PR123967: if (cond) A | CST1 : A & ~CST1

   Make both bitops unconditional: (A & ~CST1) | (cond * CST1).  */
(for cmp (simple_comparison)
 (simplify
  (cond (cmp@4 @5 @6) (bit_ior @0 INTEGER_CST@1) (bit_and@3 @0 INTEGER_CST@2))
   (if (INTEGRAL_TYPE_P (type)
	&& INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& (TYPE_UNSIGNED (TREE_TYPE (@1)) || tree_int_cst_sgn (@1) > 0)
	&& TYPE_PRECISION (type) <= BITS_PER_WORD
	&& wi::to_wide (@1) == ~wi::to_wide (@2))
    (bit_ior @3 (mult (convert:type @4) (convert:type @1))))))
#endif

/* Simplification moved from fold_cond_expr_with_comparison.  It may also
   be extended.  */
/* This pattern implements two kinds simplification:

   Case 1)
   (cond (cmp (convert1? x) c1) (convert2? x) c2) -> (minmax (x c)) if:
     1) Conversions are type widening from smaller type.
     2) Const c1 equals to c2 after canonicalizing comparison.
     3) Comparison has tree code LT, LE, GT or GE.
   This specific pattern is needed when (cmp (convert x) c) may not
   be simplified by comparison patterns because of multiple uses of
   x.  It also makes sense here because simplifying across multiple
   referred var is always benefitial for complicated cases.

   Case 2)
   (cond (eq (convert1? x) c1) (convert2? x) c2) -> (cond (eq x c1) c1 c2).  */
(for cmp (lt le gt ge eq ne)
 (simplify
  (cond (cmp (convert1? @1) INTEGER_CST@3) (convert2? @1) INTEGER_CST@2)
  (with
   {
     tree from_type = TREE_TYPE (@1);
     tree c1_type = TREE_TYPE (@3), c2_type = TREE_TYPE (@2);
     enum tree_code code = ERROR_MARK;

     /* `((signed)a) < 0` should be converted back into
        `a >= (unsigned)SIGNED_TYPE_MIN`.
	`((signed)a) >= 0` should be converted back into
	`a < (unsigned)SIGNED_TYPE_MIN`. */
     if (integer_zerop (@3)
         && INTEGRAL_TYPE_P (from_type)
         && (cmp == GE_EXPR || cmp == LT_EXPR)
	 && TYPE_UNSIGNED (from_type)
	 && !TYPE_UNSIGNED (c1_type)
	 && TYPE_PRECISION (from_type) == TYPE_PRECISION (c1_type)
	 && int_fits_type_p (@2, from_type)
	 && (types_match (c2_type, from_type)
	     || (TYPE_PRECISION (c2_type) > TYPE_PRECISION (from_type)
		 && (TYPE_UNSIGNED (from_type)
		     || TYPE_SIGN (c2_type) == TYPE_SIGN (from_type)))))
	{
	  tree_code ncmp = cmp == GE_EXPR ? LE_EXPR : GT_EXPR;
	  widest_int c1 = wi::mask<widest_int>(TYPE_PRECISION (c1_type) - 1, 0);
	  code = minmax_from_comparison (ncmp, @1, c1, wi::to_widest (@2));
	}

     if (code == ERROR_MARK
         && INTEGRAL_TYPE_P (from_type)
	 && int_fits_type_p (@2, from_type)
	 && (types_match (c1_type, from_type)
	     || (TYPE_PRECISION (c1_type) > TYPE_PRECISION (from_type)
		 && (TYPE_UNSIGNED (from_type)
		     || TYPE_SIGN (c1_type) == TYPE_SIGN (from_type))))
	 && (types_match (c2_type, from_type)
	     || (TYPE_PRECISION (c2_type) > TYPE_PRECISION (from_type)
		 && (TYPE_UNSIGNED (from_type)
		     || TYPE_SIGN (c2_type) == TYPE_SIGN (from_type)))))
       {
	 if (cmp != EQ_EXPR)
	   code = minmax_from_comparison (cmp, @1, @3, @1, @2);
	 /* Can do A == C1 ? A : C2  ->  A == C1 ? C1 : C2?  */
	 else if (int_fits_type_p (@3, from_type))
	   code = EQ_EXPR;
       }
   }
   (if (code == MAX_EXPR)
    (convert (max @1 (convert @2)))
    (if (code == MIN_EXPR)
     (convert (min @1 (convert @2)))
     (if (code == EQ_EXPR)
      (convert (cond (eq @1 (convert @3))
		     (convert:from_type @3) (convert:from_type @2)))))))))

/* (cond (cmp (convert? x) c1) (op x c2) c3) -> (op (minmax x c1) c2) if:

     1) OP is PLUS or MINUS.
     2) CMP is LT, LE, GT or GE.
     3) C3 == (C1 op C2), and computation doesn't have undefined behavior.

   This pattern also handles special cases like:

     A) Operand x is a unsigned to signed type conversion and c1 is
	integer zero.  In this case,
	  (signed type)x  < 0  <=>  x  > MAX_VAL(signed type)
	  (signed type)x >= 0  <=>  x <= MAX_VAL(signed type)
     B) Const c1 may not equal to (C3 op' C2).  In this case we also
	check equality for (c1+1) and (c1-1) by adjusting comparison
	code.

   TODO: Though signed type is handled by this pattern, it cannot be
   simplified at the moment because C standard requires additional
   type promotion.  In order to match&simplify it here, the IR needs
   to be cleaned up by other optimizers, i.e, VRP.  */
(for op (plus minus)
 (for cmp (lt le gt ge)
  (simplify
   (cond (cmp (convert? @X) INTEGER_CST@1) (op @X INTEGER_CST@2) INTEGER_CST@3)
   (with { tree from_type = TREE_TYPE (@X), to_type = TREE_TYPE (@1); }
    (if (types_match (from_type, to_type)
	 /* Check if it is special case A).  */
	 || (TYPE_UNSIGNED (from_type)
	     && !TYPE_UNSIGNED (to_type)
	     && TYPE_PRECISION (from_type) == TYPE_PRECISION (to_type)
	     && integer_zerop (@1)
	     && (cmp == LT_EXPR || cmp == GE_EXPR)))
     (with
      {
	wi::overflow_type overflow = wi::OVF_NONE;
	enum tree_code code, cmp_code = cmp;
	wide_int real_c1;
	wide_int c1 = wi::to_wide (@1);
	wide_int c2 = wi::to_wide (@2);
	wide_int c3 = wi::to_wide (@3);
	signop sgn = TYPE_SIGN (from_type);

	/* Handle special case A), given x of unsigned type:
	    ((signed type)x  < 0) <=> (x  > MAX_VAL(signed type))
	    ((signed type)x >= 0) <=> (x <= MAX_VAL(signed type))  */
	if (!types_match (from_type, to_type))
	  {
	    if (cmp_code == LT_EXPR)
	      cmp_code = GT_EXPR;
	    if (cmp_code == GE_EXPR)
	      cmp_code = LE_EXPR;
	    c1 = wi::max_value (to_type);
	  }
	/* To simplify this pattern, we require c3 = (c1 op c2).  Here we
	   compute (c3 op' c2) and check if it equals to c1 with op' being
	   the inverted operator of op.  Make sure overflow doesn't happen
	   if it is undefined.  */
	if (op == PLUS_EXPR)
	  real_c1 = wi::sub (c3, c2, sgn, &overflow);
	else
	  real_c1 = wi::add (c3, c2, sgn, &overflow);

	code = cmp_code;
	if (!overflow || !TYPE_OVERFLOW_UNDEFINED (from_type))
	  {
	    /* Check if c1 equals to real_c1.  Boundary condition is handled
	       by adjusting comparison operation if necessary.  */
	    if (!wi::cmp (wi::sub (real_c1, 1, sgn, &overflow), c1, sgn)
		&& !overflow)
	      {
		/* X <= Y - 1 equals to X < Y.  */
		if (cmp_code == LE_EXPR)
		  code = LT_EXPR;
		/* X > Y - 1 equals to X >= Y.  */
		if (cmp_code == GT_EXPR)
		  code = GE_EXPR;
	      }
	    if (!wi::cmp (wi::add (real_c1, 1, sgn, &overflow), c1, sgn)
		&& !overflow)
	      {
		/* X < Y + 1 equals to X <= Y.  */
		if (cmp_code == LT_EXPR)
		  code = LE_EXPR;
		/* X >= Y + 1 equals to X > Y.  */
		if (cmp_code == GE_EXPR)
		  code = GT_EXPR;
	      }
	    if (code != cmp_code || !wi::cmp (real_c1, c1, sgn))
	      {
		if (cmp_code == LT_EXPR || cmp_code == LE_EXPR)
		  code = MIN_EXPR;
		if (cmp_code == GT_EXPR || cmp_code == GE_EXPR)
		  code = MAX_EXPR;
	      }
	  }
      }
      (if (code == MAX_EXPR)
       (op (max @X { wide_int_to_tree (from_type, real_c1); })
	   { wide_int_to_tree (from_type, c2); })
       (if (code == MIN_EXPR)
	(op (min @X { wide_int_to_tree (from_type, real_c1); })
	    { wide_int_to_tree (from_type, c2); })))))))))

#if GIMPLE
/* A >= B ? A : B -> max (A, B) and friends.  The code is still
   in fold_cond_expr_with_comparison for GENERIC folding with
   some extra constraints.  */
(for cmp (eq ne le lt unle unlt ge gt unge ungt uneq ltgt)
 (simplify
  (cond (cmp:c (nop_convert1?@c0 @0) (nop_convert2?@c1 @1))
        (convert3? @0) (convert4? @1))
  (if (!HONOR_SIGNED_ZEROS (type)
       && (/* Allow widening conversions of the compare operands as data.  */
	   (INTEGRAL_TYPE_P (type)
	    && types_match (TREE_TYPE (@c0), TREE_TYPE (@0))
	    && types_match (TREE_TYPE (@c1), TREE_TYPE (@1))
	    && TYPE_PRECISION (TREE_TYPE (@0)) <= TYPE_PRECISION (type)
	    && TYPE_PRECISION (TREE_TYPE (@1)) <= TYPE_PRECISION (type))
	   /* Or sign conversions for the comparison.  */
	   || (types_match (type, TREE_TYPE (@0))
	       && types_match (type, TREE_TYPE (@1)))))
   (switch
    (if (cmp == EQ_EXPR)
     (if (VECTOR_TYPE_P (type))
      (view_convert @c1)
      (convert @c1)))
    (if (cmp == NE_EXPR)
     (if (VECTOR_TYPE_P (type))
      (view_convert @c0)
      (convert @c0)))
    (if (cmp == LE_EXPR || cmp == UNLE_EXPR || cmp == LT_EXPR || cmp == UNLT_EXPR)
     (if (!HONOR_NANS (type))
      (if (VECTOR_TYPE_P (type))
       (view_convert (min @c0 @c1))
       (convert (min @c0 @c1)))))
    (if (cmp == GE_EXPR || cmp == UNGE_EXPR || cmp == GT_EXPR || cmp == UNGT_EXPR)
     (if (!HONOR_NANS (type))
      (if (VECTOR_TYPE_P (type))
       (view_convert (max @c0 @c1))
       (convert (max @c0 @c1)))))
    (if (cmp == UNEQ_EXPR)
     (if (!HONOR_NANS (type))
      (if (VECTOR_TYPE_P (type))
       (view_convert @c1)
       (convert @c1))))
    (if (cmp == LTGT_EXPR)
     (if (!HONOR_NANS (type))
      (if (VECTOR_TYPE_P (type))
       (view_convert @c0)
       (convert @c0))))))))

/* This is for VEC_COND_EXPR
   Optimize A < B ? A : B to MIN (A, B)
	    A > B ? A : B to MAX (A, B).  */
(for cmp (lt le ungt unge gt ge unlt unle)
     minmax (min min min min max max max max)
     MINMAX (MIN_EXPR MIN_EXPR MIN_EXPR MIN_EXPR MAX_EXPR MAX_EXPR MAX_EXPR MAX_EXPR)
 (simplify
  (vec_cond (cmp @0 @1) @0 @1)
   (if (VECTOR_INTEGER_TYPE_P (type)
       && target_supports_op_p (type, MINMAX, optab_vector))
    (minmax @0 @1))))

(for cmp (lt le ungt unge gt ge unlt unle)
     minmax (max max max max min min min min)
     MINMAX (MAX_EXPR MAX_EXPR MAX_EXPR MAX_EXPR MIN_EXPR MIN_EXPR MIN_EXPR MIN_EXPR)
 (simplify
  (vec_cond (cmp @0 @1) @1 @0)
   (if (VECTOR_INTEGER_TYPE_P (type)
       && target_supports_op_p (type, MINMAX, optab_vector))
    (minmax @0 @1))))

/* Comparison canonicalization changes X > C to X >= C + 1 and
   X < C to X <= C - 1.  */
(for cmp (ge le)
     minmax (max min)
     MINMAX (MAX_EXPR MIN_EXPR)
 (simplify
  (vec_cond (cmp @0 VECTOR_CST@1) @0 VECTOR_CST@2)
  (with
   {
     tree diff = (minmax == MAX_EXPR
		  ? uniform_vector_difference_p (@1, @2)
		  : uniform_vector_difference_p (@2, @1));
   }
   (if (VECTOR_INTEGER_TYPE_P (type)
	&& target_supports_op_p (type, MINMAX, optab_vector)
	&& diff
	&& integer_onep (diff))
    (minmax @0 @2)))))

(for cmp (lt gt)
     minmax (max min)
     MINMAX (MAX_EXPR MIN_EXPR)
 (simplify
  (vec_cond (cmp @0 VECTOR_CST@1) VECTOR_CST@2 @0)
  (with
   {
     tree diff = (minmax == MAX_EXPR
		  ? uniform_vector_difference_p (@1, @2)
		  : uniform_vector_difference_p (@2, @1));
   }
   (if (VECTOR_INTEGER_TYPE_P (type)
	&& target_supports_op_p (type, MINMAX, optab_vector)
	&& diff
	&& integer_onep (diff))
    (minmax @0 @2)))))

/* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
   and x < 0 ? 1 : 0 into (unsigned) x >> 31.  */
(simplify
  (vec_cond (lt @0 integer_zerop) integer_all_onesp integer_zerop)
   (if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && tree_nop_conversion_p (type, TREE_TYPE (@0))
       && target_supports_op_p (TREE_TYPE (@0), RSHIFT_EXPR, optab_scalar))
    (with
      {
       unsigned int prec = element_precision (TREE_TYPE (@0));
      }
    (view_convert
      (rshift @0 { build_int_cst (integer_type_node, prec - 1);})))))

(simplify
  (vec_cond (lt @0 integer_zerop) integer_onep integer_zerop)
   (if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && tree_nop_conversion_p (type, TREE_TYPE (@0)))
    (with
     {
       unsigned int prec = element_precision (TREE_TYPE (@0));
       tree utype = unsigned_type_for (TREE_TYPE (@0));
     }
     (if (target_supports_op_p (utype, RSHIFT_EXPR, optab_scalar))
      (view_convert
	(rshift (view_convert:utype @0)
		{ build_int_cst (integer_type_node, prec - 1);}))))))
#endif

(for cnd (cond vec_cond)
 /* (a != b) ? (a - b) : 0 -> (a - b) */
 /* (a != b) ? (a ^ b) : 0 -> (a ^ b) */
 (for op (minus bit_xor)
  (simplify
   (cnd (ne:c @0 @1) (op@2 @0 @1) integer_zerop)
   @2))
 /* (a != b) ? (a & b) : a -> (a & b) */
 /* (a != b) ? (a | b) : a -> (a | b) */
 /* (a != b) ? min(a,b) : a -> min(a,b) */
 /* (a != b) ? max(a,b) : a -> max(a,b) */
 (for op (bit_and bit_ior min max)
  (simplify
   (cnd (ne:c @0 @1) (op:c@2 @0 @1) @0)
   @2))
 /* (a != b) ? (a * b) : (a * a) -> (a * b) */
 /* (a != b) ? (a + b) : (a + a) -> (a + b) */
 (for op (mult plus)
  (simplify
   (cnd (ne:c @0 @1) (op@2 @0 @1) (op @0 @0))
   (if (ANY_INTEGRAL_TYPE_P (type))
    @2)))
 /* (a != b) ? (a + b) : (2 * a) -> (a + b) */
 (simplify
  (cnd (ne:c @0 @1) (plus:c@2 @0 @1) (mult @0 uniform_integer_cst_p@3))
  (if (wi::to_wide (uniform_integer_cst_p (@3)) == 2)
   @2))
)

#if GIMPLE
/* Given two integers a and b:
   (a != b) & ((a|b) != 0) -> (a != b)
   (a == b) | ((a|b) == 0) -> (a == b)

   (a == b) & ((a|b) == 0) -> ((a|b) == 0)
   (a != b) | ((a|b) != 0) -> ((a|b) != 0)

   (a != b) & ((a|b) == 0) -> false
   (a == b) | ((a|b) != 0) -> true

   (a == 0) | ((a | b) == 0) -> (a == 0)
   (a != 0) & ((a | b) != 0) -> (a != 0)  */
(for bitop (bit_and bit_ior)
     neeq  (ne      eq)
     eqne  (eq      ne)
 (simplify
  (bitop:c (neeq @0 @1) (neeq (bit_ior @0 @1) integer_zerop))
  (neeq @0 @1))
 (simplify
  (bitop:c (eqne @0 @1) (eqne (bit_ior@2 @0 @1) integer_zerop@3))
  (eqne @2 @3))
 (simplify
  (bitop (neeq @0 @1) (eqne (bit_ior @0 @1) integer_zerop))
  { constant_boolean_node (bitop == BIT_IOR_EXPR, type); })
 (simplify
  (bitop:c (neeq@2 @0 integer_zerop) (neeq (bit_ior:c @0 @1) integer_zerop))
  @2))
#endif

/* These was part of minmax phiopt.  */
/* Optimize (a CMP b) ? minmax<a, c> : minmax<b, c>
   to minmax<min/max<a, b>, c> */
(for minmax (min max)
 (for cmp (lt le gt ge ne)
  (simplify
   (cond (cmp:c @1 @3) (minmax:c @1 @4) (minmax:c @2 @4))
   (with
    {
      tree_code code = minmax_from_comparison (cmp, @1, @3, @1, @2);
    }
    (if (code == MIN_EXPR)
     (minmax (min @1 @2) @4)
     (if (code == MAX_EXPR)
      (minmax (max @1 @2) @4)))))))

/* Optimize (a CMP CST1) ? max<a,CST2> : a */
(for cmp    (gt  ge  lt  le)
     minmax (min min max max)
 (simplify
  (cond (cmp:c @0 @1) (minmax:c@2 @0 @3) @4)
   (with
    {
      tree_code code = minmax_from_comparison (cmp, @0, @1, @0, @4);
    }
    (if ((cmp == LT_EXPR || cmp == LE_EXPR)
	 && code == MIN_EXPR
         && integer_nonzerop (fold_build2 (LE_EXPR, boolean_type_node, @3, @4)))
     (min @2 @4)
     (if ((cmp == GT_EXPR || cmp == GE_EXPR)
	  && code == MAX_EXPR
          && integer_nonzerop (fold_build2 (GE_EXPR, boolean_type_node, @3, @4)))
      (max @2 @4))))))

/* Optimize (((signed)a CMP 0) ? max<a,CST2> : CST3 */
(for cmp    (lt  ge)
     minmax (min max)
 (simplify
  (cond (cmp:c (nop_convert @0) integer_zerop@1) (minmax@2 @0 INTEGER_CST@3) INTEGER_CST@4)
   (if (!TYPE_UNSIGNED (TREE_TYPE (@1))
        && TYPE_UNSIGNED (TREE_TYPE (@0)))
    (with
     {
       tree_code code;
       /* ((signed)a) < 0 -> a > SIGNED_MAX */
       /* ((signed)a) >= 0 -> a <= SIGNED_MAX */
       widest_int c1 = wi::mask<widest_int>(TYPE_PRECISION (type) - 1, 0);
       tree_code ncmp = cmp == GE_EXPR ? LE_EXPR : GT_EXPR;
       code = minmax_from_comparison (ncmp, @0, c1, wi::to_widest (@4));
     }
     (if (ncmp == LE_EXPR
	  && code == MIN_EXPR
	  && wi::le_p (wi::to_wide (@3),
		       wi::to_wide (@4),
		       TYPE_SIGN (type)))
      (min @2 @4)
      (if (ncmp == GT_EXPR
	   && code == MAX_EXPR
	   && wi::ge_p (wi::to_wide (@3),
		        wi::to_wide (@4),
		        TYPE_SIGN (type)))
       (max @2 @4)))))))

/* Optimize signed < 0 ? positive : MIN<signed, positive> to
   MIN<(unsigned)signed, (unsigned)positive>. */
(simplify
 (cond (lt @0 integer_zerop)
  tree_expr_nonnegative_p@1
  (min:c @0 @1))
 (if (INTEGRAL_TYPE_P (type)
      && !TYPE_UNSIGNED (type))
  (with { tree utype = unsigned_type_for (type); }
   (convert (min (convert:utype @0) (convert:utype @1))))))

#if GIMPLE
/* These patterns should be after min/max detection as simplifications
   of `(type)(zero_one ==/!= 0)` to `(type)(zero_one)`
   and `(type)(zero_one^1)` are not done yet.  See PR 110637.
   Even without those, reaching min/max/and/ior faster is better.  */
(simplify
 (cond @0 zero_one_valued_p@1 zero_one_valued_p@2)
 (switch
  /* bool0 ? bool1 : 0 -> bool0 & bool1 */
  (if (integer_zerop (@2))
   (bit_and (convert @0) @1))
  /* bool0 ? 0 : bool2 -> (bool0^1) & bool2 */
  (if (integer_zerop (@1))
   (bit_and (bit_xor (convert @0) { build_one_cst (type); } ) @2))
  /* bool0 ? 1 : bool2 -> bool0 | bool2 */
  (if (integer_onep (@1))
   (bit_ior (convert @0) @2))
  /* bool0 ? bool1 : 1 -> (bool0^1) | bool1 */
  (if (integer_onep (@2))
   (bit_ior (bit_xor (convert @0) @2) @1))
 )
)
#endif

/* X != C1 ? -X : C2 simplifies to -X when -C1 == C2.  Additionally,
   when C1 is the minimum signed value (e.g.  INT_MIN), -X would be
   undefined for signed types, so emit (signed)(-(unsigned)X) instead.  */
(simplify
 (cond (ne @0 INTEGER_CST@1) (negate@3 @0) INTEGER_CST@2)
 (if (!TYPE_SATURATING (type)
      && wi::eq_p (wi::neg (wi::to_wide (@1)), wi::to_wide (@2)))
  (if (TYPE_OVERFLOW_WRAPS (type)
       || !wi::only_sign_bit_p (wi::to_wide (@1)))
   @3
   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
    (convert (negate (convert:utype @0)))))))

/* X != C1 ? ~X : C2 simplifies to ~X when ~C1 == C2.  */
(simplify
 (cond (ne @0 INTEGER_CST@1) (bit_not@3 @0) INTEGER_CST@2)
 (if (wi::eq_p (wi::bit_not (wi::to_wide (@1)), wi::to_wide (@2)))
  @3))

/* X != C1 ? abs(X) : C2 simplifies to abs(x) when abs(C1) == C2. */
(for op (abs absu)
 (simplify
  (cond (ne @0 INTEGER_CST@1) (op@3 @0) INTEGER_CST@2)
  (if (wi::abs (wi::to_wide (@1)) == wi::to_wide (@2))
   (if (op != ABSU_EXPR && wi::only_sign_bit_p (wi::to_wide (@1)))
    (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
     (convert (absu:utype @0)))
    @3))))

/* X >  Positive ? X : ABS(X) -> ABS(X) */
/* X >= Positive ? X : ABS(X) -> ABS(X) */
/* X == Positive ? X : ABS(X) -> ABS(X) */
(for cmp (eq gt ge)
 (simplify
  (cond (cmp:c @0 tree_expr_nonnegative_p@1) @0 (abs@3 @0))
  (if (INTEGRAL_TYPE_P (type))
   @3)))

/* X == Positive ? Positive : ABS(X) -> ABS(X) */
(simplify
 (cond (eq:c @0 tree_expr_nonnegative_p@1) @1 (abs@3 @0))
 (if (INTEGRAL_TYPE_P (type))
  @3))

/* A > B ? ABS(A) : B -> MAX(A, B) when B is non-negative.
   When A > B >= 0, A is positive so ABS(A) = A = MAX(A, B).
   When A <= B, the result is B = MAX(A, B).  */
(for cmp (gt ge)
 (simplify
  (cond (cmp:c @0 tree_expr_nonnegative_p@1) (abs @0) @1)
  (if (INTEGRAL_TYPE_P (type))
   (max @0 @1))))

/* (X + 1) > Y ? -X : 1 simplifies to X >= Y ? -X : 1 when
   X is unsigned, as when X + 1 overflows, X is -1, so -X == 1.  */
(simplify
 (cond (gt (plus @0 integer_onep) @1) (negate @0) integer_onep@2)
 (if (TYPE_UNSIGNED (type))
  (cond (ge @0 @1) (negate @0) @2)))

(for cnd (cond vec_cond)
 /* A ? B : (A ? X : C) -> A ? B : C.  */
 (simplify
  (cnd @0 (cnd @0 @1 @2) @3)
  (cnd @0 @1 @3))
 (simplify
  (cnd @0 @1 (cnd @0 @2 @3))
  (cnd @0 @1 @3))
 /* A ? B : (!A ? C : X) -> A ? B : C.  */
 /* ???  This matches embedded conditions open-coded because genmatch
    would generate matching code for conditions in separate stmts only.
    The following is still important to merge then and else arm cases
    from if-conversion.  */
 (simplify
  (cnd @0 @1 (cnd @2 @3 @4))
  (if (inverse_conditions_p (@0, @2))
   (cnd @0 @1 @3)))
 (simplify
  (cnd @0 (cnd @1 @2 @3) @4)
  (if (inverse_conditions_p (@0, @1))
   (cnd @0 @3 @4)))

 /* A ? B : B -> B.  */
 (simplify
  (cnd @0 @1 @1)
  @1)

 /* !A ? B : C -> A ? C : B.  */
 (simplify
  (cnd (logical_inverted_value truth_valued_p@0) @1 @2)
  /* For CONDs, don't handle signed values here. */
  (if (cnd == VEC_COND_EXPR
       || TYPE_UNSIGNED (TREE_TYPE (@0)))
   (cnd @0 @2 @1)))

 /* A == CST ? A : CST -> CST.  */
 (simplify
  (cnd (eq @0 CONSTANT_CLASS_P@1) (nop_convert? @0) CONSTANT_CLASS_P@2)
  (if (ANY_INTEGRAL_TYPE_P (type)
       && operand_equal_p (@1, @2))
   (convert @1)))

 /* A != CST ? CST : A -> CST.  */
 (simplify
  (cnd (ne @0 CONSTANT_CLASS_P@1) CONSTANT_CLASS_P@2 (nop_convert? @0))
  (if (ANY_INTEGRAL_TYPE_P (type)
       && operand_equal_p (@1, @2))
   (convert @1))))

/* abs/negative simplifications moved from fold_cond_expr_with_comparison.

   None of these transformations work for modes with signed
   zeros.  If A is +/-0, the first two transformations will
   change the sign of the result (from +0 to -0, or vice
   versa).  The last four will fix the sign of the result,
   even though the original expressions could be positive or
   negative, depending on the sign of A.

   Note that all these transformations are correct if A is
   NaN, since the two alternatives (A and -A) are also NaNs.  */

(for cnd (cond vec_cond)
 /* A == 0 ? A : -A    same as -A */
 (for cmp (eq uneq)
  (simplify
   (cnd (cmp @0 zerop) @2 (negate@1 @2))
    (if (!HONOR_SIGNED_ZEROS (type)
	 && bitwise_equal_p (@0, @2))
     @1))
  (simplify
   (cnd (cmp @0 zerop) zerop (negate@1 @2))
    (if (!HONOR_SIGNED_ZEROS (type)
	 && bitwise_equal_p (@0, @2))
     @1))
 )
 /* A != 0 ? A : -A    same as A */
 (for cmp (ne ltgt)
  (simplify
   (cnd (cmp @0 zerop) @1 (negate @1))
    (if (!HONOR_SIGNED_ZEROS (type)
	 && bitwise_equal_p (@0, @1))
     @1))
  (simplify
   (cnd (cmp @0 zerop) @1 integer_zerop)
    (if (!HONOR_SIGNED_ZEROS (type)
	 && bitwise_equal_p (@0, @1))
     @1))
 )
 /* (type)A >=/> 0 ? A : -A    same as abs (A) */
 (for cmp (ge gt)
  (simplify
   (cnd (cmp (convert?@0 @1) zerop) @2 (negate @2))
    (if (!HONOR_SIGNED_ZEROS (TREE_TYPE (@1))
	 /* Support SEXT of @0 only.  */
	 && !TYPE_UNSIGNED (TREE_TYPE (@1))
	 && element_precision (@1)
	      <= element_precision (@0)
	 && bitwise_equal_p (@1, @2)
	 && (!VECTOR_TYPE_P (type)
	     || target_supports_op_p (type, ABS_EXPR, optab_vector)))
    (if (TYPE_UNSIGNED (TREE_TYPE (@2)))
      (with {
	tree stype = signed_type_for (TREE_TYPE (@2));
       }
      (if (types_match (@0, stype))
       (absu @0)
       (absu (convert:stype @2))))
      (abs @2)))))
 /* (type)A <=/< 0 ? A : -A    same as -abs (A) */
 (for cmp (le lt)
  (simplify
   (cnd (cmp (convert?@0 @1) zerop) @2 (negate @2))
    (if (!HONOR_SIGNED_ZEROS (TREE_TYPE (@1))
	 /* Support SEXT of @0 only.  */
	 && !TYPE_UNSIGNED (TREE_TYPE (@1))
	 && element_precision (@1)
	      <= element_precision (@0)
	 && bitwise_equal_p (@1, @2)
	 && (!VECTOR_TYPE_P (type)
	     || (target_supports_op_p (type, ABS_EXPR, optab_vector)
		 && target_supports_op_p (type, NEGATE_EXPR, optab_vector))))
      (if ((ANY_INTEGRAL_TYPE_P (TREE_TYPE (@2))
	      && !TYPE_OVERFLOW_WRAPS (TREE_TYPE (@2)))
	    || TYPE_UNSIGNED (TREE_TYPE (@2)))
	(with {
	  tree stype = signed_type_for (TREE_TYPE (@2));
	  tree utype = unsigned_type_for (TREE_TYPE (@2));
	 }
	(if (types_match (@0, stype))
	 (convert (negate (absu:utype @0)))
	 (convert (negate (absu:utype (convert:stype @2))))))
	(convert (negate (abs @2))))))
 )
 /* A </<= 0 ? -A : A    same as abs (A)
    A >/>= 0 ? -A : A    same as -abs (A)
    For integers these are reached by inverting the comparison of the
    forms above, but a floating-point comparison that may trap on NaNs
    cannot be inverted.  ABS does not trap, so the ordered compare may
    be removed only when this operand cannot be a NaN or the invalid
    exception is not observable.  */
 (for cmp (lt le)
  (simplify
   (cnd (cmp @0 real_zerop) (negate @0) @0)
    (if (!HONOR_SIGNED_ZEROS (type)
	 && !(tree_expr_maybe_nan_p (@0) && flag_trapping_math)
	 && (!VECTOR_TYPE_P (type)
	     || target_supports_op_p (type, ABS_EXPR, optab_vector)))
     (abs @0))))
 (for cmp (gt ge)
  (simplify
   (cnd (cmp @0 real_zerop) (negate @0) @0)
    (if (!HONOR_SIGNED_ZEROS (type)
	 && !(tree_expr_maybe_nan_p (@0) && flag_trapping_math)
	 && (!VECTOR_TYPE_P (type)
	     || (target_supports_op_p (type, ABS_EXPR, optab_vector)
		 && target_supports_op_p (type, NEGATE_EXPR, optab_vector))))
     (negate (abs @0)))))

 /* (A - B) == 0 ? (A - B) : (B - A)    same as (B - A) */
 (for cmp (eq uneq)
  (simplify
   (cnd (cmp (minus@0 @1 @2) zerop) @0 (minus@3 @2 @1))
   (if (!HONOR_SIGNED_ZEROS (type))
    @3))
  (simplify
   (cnd (cmp (minus@0 @1 @2) integer_zerop) integer_zerop (minus@3 @2 @1))
   @3)
 )
 /* (A - B) != 0 ? (A - B) : (B - A)    same as (A - B) */
 (for cmp (ne ltgt)
  (simplify
   (cnd (cmp (minus@0 @1 @2) zerop) @0 (minus @2 @1))
   (if (!HONOR_SIGNED_ZEROS (type))
    @0))
  (simplify
   (cnd (cmp (minus@0 @1 @2) integer_zerop) @0 integer_zerop)
   @0)
 )
 /* (A - B) >=/> 0 ? (A - B) : (B - A)    same as abs (A - B) */
 (for cmp (ge gt)
  (simplify
   (cnd (cmp (minus@0 @1 @2) zerop) @0 (minus @2 @1))
   (if (!HONOR_SIGNED_ZEROS (type)
	&& !TYPE_UNSIGNED (type)
	&& (!VECTOR_TYPE_P (type)
	    || target_supports_op_p (type, ABS_EXPR, optab_vector)))
    (abs @0))))
 /* (A - B) <=/< 0 ? (A - B) : (B - A)    same as -abs (A - B) */
 (for cmp (le lt)
  (simplify
   (cnd (cmp (minus@0 @1 @2) zerop) @0 (minus @2 @1))
   (if (!HONOR_SIGNED_ZEROS (type)
	&& !TYPE_UNSIGNED (type))
    (if (ANY_INTEGRAL_TYPE_P (type)
	 && !TYPE_OVERFLOW_WRAPS (type)
	 && (!VECTOR_TYPE_P (type)
	     || (target_supports_op_p (type, ABS_EXPR, optab_vector)
		 && target_supports_op_p (type, NEGATE_EXPR, optab_vector))))
     (with {
        tree utype = unsigned_type_for (type);
      }
      (convert (negate (absu:utype @0))))
      (negate (abs @0)))))
 )
)

/* X >=/> 0 ? Y + X : Y - X
   X <=/< 0 ? Y - X : Y + X
     same as Y + abs (X).

   X >=/> 0 ? Y - X : Y + X
   X <=/< 0 ? Y + X : Y - X
     same as Y - abs (X).

   Build the addition or subtraction in an unsigned type for integral
   types so the replacement does not introduce signed overflow.  */

#if GIMPLE
(for cmp (ge gt)
 (simplify
  (cond (cmp @0 integer_zerop) (plus:c @0 @1) (minus @1 @0))
  (if (!TYPE_UNSIGNED (type)
       && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with { tree utype = unsigned_type_for (type); }
    (convert (plus:utype (convert:utype @1)
			 (absu:utype @0))))))
 (simplify
  (cond (cmp @0 integer_zerop) (minus @1 @0) (plus:c @0 @1))
  (if (!TYPE_UNSIGNED (type)
       && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with { tree utype = unsigned_type_for (type); }
    (convert (minus:utype (convert:utype @1)
			  (absu:utype @0)))))))
(for cmp (le lt)
 (simplify
  (cond (cmp @0 integer_zerop) (minus @1 @0) (plus:c @0 @1))
  (if (!TYPE_UNSIGNED (type)
       && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with { tree utype = unsigned_type_for (type); }
    (convert (plus:utype (convert:utype @1)
			 (absu:utype @0))))))
 (simplify
  (cond (cmp @0 integer_zerop) (plus:c @0 @1) (minus @1 @0))
  (if (!TYPE_UNSIGNED (type)
       && !TYPE_OVERFLOW_TRAPS (type)
       && !TYPE_OVERFLOW_SANITIZED (type))
   (with { tree utype = unsigned_type_for (type); }
    (convert (minus:utype (convert:utype @1)
			  (absu:utype @0)))))))
#endif

/* -(type)!A -> (type)A - 1.  */
(simplify
 (negate (convert?:s (logical_inverted_value:s @0)))
 (if (INTEGRAL_TYPE_P (type)
      && TREE_CODE (type) != BOOLEAN_TYPE
      && TYPE_PRECISION (type) > 1
      && TREE_CODE (@0) == SSA_NAME
      && ssa_name_has_boolean_range (@0))
  (plus (convert:type @0) { build_all_ones_cst (type); })))

/* A + (B vcmp C ? 1 : 0) -> A - (B vcmp C ? -1 : 0), since vector comparisons
   return all -1 or all 0 results.  */
/* ??? We could instead convert all instances of the vec_cond to negate,
   but that isn't necessarily a win on its own.  */
(simplify
 (plus:c @3 (view_convert? (vec_cond:s @0 integer_each_onep@1 integer_zerop@2)))
 (if (VECTOR_TYPE_P (type)
      && known_eq (TYPE_VECTOR_SUBPARTS (type),
		   TYPE_VECTOR_SUBPARTS (TREE_TYPE (@1)))
      && (TYPE_MODE (TREE_TYPE (type))
          == TYPE_MODE (TREE_TYPE (TREE_TYPE (@1)))))
  (minus @3 (view_convert (vec_cond @0 (negate @1) @2)))))

/* ... likewise A - (B vcmp C ? 1 : 0) -> A + (B vcmp C ? -1 : 0).  */
(simplify
 (minus @3 (view_convert? (vec_cond:s @0 integer_each_onep@1 integer_zerop@2)))
 (if (VECTOR_TYPE_P (type)
      && known_eq (TYPE_VECTOR_SUBPARTS (type),
		   TYPE_VECTOR_SUBPARTS (TREE_TYPE (@1)))
      && (TYPE_MODE (TREE_TYPE (type))
          == TYPE_MODE (TREE_TYPE (TREE_TYPE (@1)))))
  (plus @3 (view_convert (vec_cond @0 (negate @1) @2)))))


/* Simplifications of comparisons.  */

/* See if we can reduce the magnitude of a constant involved in a
   comparison by changing the comparison code.  This is a canonicalization
   formerly done by maybe_canonicalize_comparison_1.  */
(for cmp  (le gt)
     acmp (lt ge)
 (simplify
  (cmp @0 uniform_integer_cst_p@1)
  (with { tree cst = uniform_integer_cst_p (@1); }
   (if (tree_int_cst_sgn (cst) == -1)
     (acmp @0 { build_uniform_cst (TREE_TYPE (@1),
				   wide_int_to_tree (TREE_TYPE (cst),
						     wi::to_wide (cst)
						     + 1)); })))))
(for cmp  (ge lt)
     acmp (gt le)
 (simplify
  (cmp @0 uniform_integer_cst_p@1)
  (with { tree cst = uniform_integer_cst_p (@1); }
   (if (tree_int_cst_sgn (cst) == 1)
    (acmp @0 { build_uniform_cst (TREE_TYPE (@1),
				  wide_int_to_tree (TREE_TYPE (cst),
				  wi::to_wide (cst) - 1)); })))))

/* We can simplify a logical negation of a comparison to the
   inverted comparison.  As we cannot compute an expression
   operator using invert_tree_comparison we have to simulate
   that with expression code iteration.  */
(for cmp (tcc_comparison)
 /* Ideally we'd like to combine the following two patterns
    and handle some more cases by using
      (logical_inverted_value (cmp @0 @1))
    here but for that genmatch would need to "inline" that.
    For now implement what forward_propagate_comparison did.  */
 (simplify
  (bit_not (cmp @0 @1))
  (if (VECTOR_TYPE_P (type)
       || (INTEGRAL_TYPE_P (type) && TYPE_PRECISION (type) == 1))
   /* Comparison inversion may be impossible for trapping math,
      invert_tree_comparison will tell us.  */
   (with rescmpcode (tcc_comparison)
     {
       enum tree_code rescmpcode;
       rescmpcode = invert_tree_comparison (cmp, HONOR_NANS (@0));
     }
    (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
     (rescmpcode @0 @1)))))
 (simplify
  (bit_xor (cmp @0 @1) integer_truep)
  (with rescmpcode (tcc_comparison)
     {
       enum tree_code rescmpcode;
       rescmpcode = invert_tree_comparison (cmp, HONOR_NANS (@0));
     }
    (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
     (rescmpcode @0 @1))))
 /* ((cast)cmp) - 1 -> -(cast)icmp . */
 (simplify
  (plus (convert? (cmp@2 @0 @1)) integer_minus_onep)
  (if (TYPE_PRECISION (type) > 1
       && INTEGRAL_TYPE_P (TREE_TYPE (@2)) && TYPE_PRECISION (TREE_TYPE (@2)) == 1)
   /* Comparison inversion may be impossible for trapping math,
      invert_tree_comparison will tell us.  */
   (with rescmpcode (tcc_comparison)
     {
       enum tree_code rescmpcode;
       rescmpcode = invert_tree_comparison (cmp, HONOR_NANS (@0));
       tree cmptype = TREE_TYPE (@2);
     }
    (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
     (negate (convert (rescmpcode:cmptype @0 @1)))))))
 /* The following bits are handled by fold_binary_op_with_conditional_arg.  */
 (simplify
  (ne (cmp@2 @0 @1) integer_zerop)
  (if (types_match (type, TREE_TYPE (@2)))
   (cmp @0 @1)))
 (simplify
  (eq (cmp@2 @0 @1) integer_truep)
  (if (types_match (type, TREE_TYPE (@2)))
   (cmp @0 @1)))
 (simplify
  (ne (cmp@2 @0 @1) integer_truep)
  (if (types_match (type, TREE_TYPE (@2)))
   (with rescmpcode (tcc_comparison)
     {
       enum tree_code rescmpcode;
       rescmpcode = invert_tree_comparison (cmp, HONOR_NANS (@0));
     }
    (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
     (rescmpcode @0 @1)))))
 (simplify
  (eq (cmp@2 @0 @1) integer_zerop)
  (if (types_match (type, TREE_TYPE (@2)))
   (with rescmpcode (tcc_comparison)
     {
       enum tree_code rescmpcode;
       rescmpcode = invert_tree_comparison (cmp, HONOR_NANS (@0));
     }
    (if (TREE_CODE_CLASS (rescmpcode) == tcc_comparison)
     (rescmpcode @0 @1))))))

/* Transform comparisons of the form X - Y CMP 0 to X CMP Y.
   ??? The transformation is valid for the other operators if overflow
   is undefined for the type, but performing it here badly interacts
   with the transformation in fold_cond_expr_with_comparison which
   attempts to synthetize ABS_EXPR.  */
(for cmp (eq ne)
 (for sub (minus pointer_diff)
  (simplify
   (cmp (sub@2 @0 @1) integer_zerop)
   (if (single_use (@2))
    (cmp @0 @1)))))

/* Fold ((X CMP1 Y) - (X CMP2 Y)) REL 0 into a single comparison of X and Y,
   e.g., ((X > Y) - (X < Y)) < 0 to X < Y.  This cannot be merged into the
   generic X - Y CMP 0 rule above as it would prevent ABS_EXPR synthesis in
   fold_cond_expr_with_comparison.  */
(for rel (lt le gt ge)
 (for cmp1 (gt ge)
      cmp2 (lt le)
  (simplify
   (rel (minus@2 (convert? (cmp1:c @0 @1))
		 (convert? (cmp2:c @0 @1))) integer_zerop)
    (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	 && !TYPE_UNSIGNED (TREE_TYPE (@2))
	 && TYPE_PRECISION (TREE_TYPE (@2)) > 1)
     (rel @0 @1)))))

/* Simplify (x < 0) ^ (y < 0) to (x ^ y) < 0 and
   (x >= 0) ^ (y >= 0) to (x ^ y) < 0.  */
(for cmp (lt ge)
 (simplify
  (bit_xor (cmp:s @0 integer_zerop) (cmp:s @1 integer_zerop))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& !TYPE_UNSIGNED (TREE_TYPE (@0))
	&& types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
    (lt (bit_xor @0 @1) { build_zero_cst (TREE_TYPE (@0)); }))))
/* Simplify (x < 0) ^ (y >= 0) to (x ^ y) >= 0 and
   (x >= 0) ^ (y < 0) to (x ^ y) >= 0.  */
(simplify
 (bit_xor:c (lt:s @0 integer_zerop) (ge:s @1 integer_zerop))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
   (ge (bit_xor @0 @1) { build_zero_cst (TREE_TYPE (@0)); })))

/* Transform comparisons of the form X * C1 CMP 0 to X CMP 0 in the
   signed arithmetic case.  That form is created by the compiler
   often enough for folding it to be of value.  One example is in
   computing loop trip counts after Operator Strength Reduction.  */
(for cmp (simple_comparison)
     scmp (swapped_simple_comparison)
 (simplify
  (cmp (mult@3 @0 INTEGER_CST@1) integer_zerop@2)
  /* Handle unfolded multiplication by zero.  */
  (if (integer_zerop (@1))
   (cmp @1 @2)
   (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	&& single_use (@3))
    /* If @1 is negative we swap the sense of the comparison.  */
    (if (tree_int_cst_sgn (@1) < 0)
     (scmp @0 @2)
     (cmp @0 @2))))))

/* For integral types with undefined overflow fold
   x * C1 == C2 into x == C2 / C1 or false.
   If overflow wraps and C1 is odd, simplify to x == C2 / C1 in the ring
   Z / 2^n Z.  */
(for cmp (eq ne)
 (simplify
  (cmp (mult @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
       && wi::to_wide (@1) != 0)
   (with { widest_int quot; }
    (if (wi::multiple_of_p (wi::to_widest (@2), wi::to_widest (@1),
			    TYPE_SIGN (TREE_TYPE (@0)), &quot))
     (cmp @0 { wide_int_to_tree (TREE_TYPE (@0), quot); })
     { constant_boolean_node (cmp == NE_EXPR, type); }))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0))
	&& (wi::bit_and (wi::to_wide (@1), 1) == 1))
    (cmp @0
     {
       tree itype = TREE_TYPE (@0);
       int p = TYPE_PRECISION (itype);
       wide_int m = wi::one (p + 1) << p;
       wide_int a = wide_int::from (wi::to_wide (@1), p + 1, UNSIGNED);
       wide_int i = wide_int::from (wi::mod_inv (a, m),
				    p, TYPE_SIGN (itype));
       wide_int_to_tree (itype, wi::mul (i, wi::to_wide (@2)));
     })))))

/* abs (X) >= X -> true.  */
/* abs (X) < X -> false.  */
(for cmp (ge lt)
 (simplify
  (cmp:c (abs @0) @0)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))
   { constant_boolean_node (cmp == GE_EXPR, type); })))

/* abs (X) >  X -> X < 0.  */
/* abs (X) != X -> X < 0.  */
/* abs (X) <= X -> X >= 0.  */
/* abs (X) == X -> X >= 0.  */
/* When signed overflow wraps, abs (INT_MIN) == INT_MIN.  */
(for cmp (gt ne le eq)
     out (lt lt ge ge)
 (simplify
  (cmp:c (abs @0) @0)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))
   (switch
    (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
     (out @0 { build_zero_cst (TREE_TYPE (@0)); }))
    (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
     (with
      {
        tree utype = unsigned_type_for (TREE_TYPE (@0));
        wide_int min = wi::min_value (TREE_TYPE (@0));
      }
      (if (cmp == LE_EXPR || cmp == EQ_EXPR)
       (lt (convert:utype @0) { wide_int_to_tree (utype, min + 1); })
       (gt (convert:utype @0) { wide_int_to_tree (utype, min); }))))))))

/* (unsigned) abs (X) <= (unsigned) X -> true.  */
/* (unsigned) abs (X) > (unsigned) X -> false.  */
(for cmp (le gt)
 (simplify
  (cmp:c (convert@1 (abs @0)) (convert@2 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_UNSIGNED (TREE_TYPE (@1))
       && types_match (TREE_TYPE (@1), TREE_TYPE (@2))
       && element_precision (TREE_TYPE (@0)) == element_precision (TREE_TYPE (@1))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))
   { constant_boolean_node (cmp == LE_EXPR, type); })))

/* (unsigned) abs (X) <  (unsigned) X -> X < 0.  */
/* (unsigned) abs (X) != (unsigned) X -> X < 0.  */
/* (unsigned) abs (X) >= (unsigned) X -> X >= 0.  */
/* (unsigned) abs (X) == (unsigned) X -> X >= 0.  */
/* TODO: Add wrapping-overflow variants for these four cases.  */
(for cmp (lt ne ge eq)
     out (lt lt ge ge)
 (simplify
  (cmp:c (convert@1 (abs @0)) (convert@2 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && TYPE_UNSIGNED (TREE_TYPE (@1))
       && types_match (TREE_TYPE (@1), TREE_TYPE (@2))
       && element_precision (TREE_TYPE (@0)) == element_precision (TREE_TYPE (@1))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
   (out @0 { build_zero_cst (TREE_TYPE (@0)); }))))

/* Simplify comparison of something with itself.  For IEEE
   floating-point, we can only do some of these simplifications.  */
(for cmp (eq ge le)
 (simplify
  (cmp @0 @0)
  (if (! FLOAT_TYPE_P (TREE_TYPE (@0))
       || ! tree_expr_maybe_nan_p (@0))
   { constant_boolean_node (true, type); }
   (if (cmp != EQ_EXPR
	/* With -ftrapping-math conversion to EQ loses an exception.  */
	&& (! FLOAT_TYPE_P (TREE_TYPE (@0))
	    || ! flag_trapping_math))
    (eq @0 @0)))))
(for cmp (ne gt lt)
 (simplify
  (cmp @0 @0)
  (if (cmp != NE_EXPR
       || ! FLOAT_TYPE_P (TREE_TYPE (@0))
       || ! tree_expr_maybe_nan_p (@0))
   { constant_boolean_node (false, type); })))
(for cmp (unle unge uneq)
 (simplify
  (cmp @0 @0)
  { constant_boolean_node (true, type); }))
(for cmp (unlt ungt)
 (simplify
  (cmp @0 @0)
  (unordered @0 @0)))
(simplify
 (ltgt @0 @0)
 (if (!flag_trapping_math || !tree_expr_maybe_nan_p (@0))
  { constant_boolean_node (false, type); }))

/* x == ~x -> false */
/* x != ~x -> true */
(for cmp (eq ne)
 (simplify
  (cmp:c (convert? @0) (convert? (maybe_bit_not @1)))
  (with { bool wascmp; }
   (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1))
        && bitwise_inverted_equal_p (@0, @1, wascmp))
    { constant_boolean_node (cmp == NE_EXPR, type); }))))

/* Fold ~X op ~Y as Y op X.  */
(for cmp (simple_comparison)
 (simplify
  (cmp (nop_convert1?@4 (bit_not@2 @0)) (nop_convert2? (bit_not@3 @1)))
  (if (single_use (@2) && single_use (@3))
   (with { tree otype = TREE_TYPE (@4); }
    (cmp (convert:otype @1) (convert:otype @0))))))

/* Fold ~X op C as X op' ~C, where op' is the swapped comparison.  */
(for cmp (simple_comparison)
     scmp (swapped_simple_comparison)
 (simplify
  (cmp (nop_convert? (bit_not@2 @0)) CONSTANT_CLASS_P@1)
  (if (single_use (@2)
       && (TREE_CODE (@1) == INTEGER_CST || TREE_CODE (@1) == VECTOR_CST))
   (with { tree otype = TREE_TYPE (@1); }
    (scmp (convert:otype @0) (bit_not @1))))))

(for cmp (simple_comparison)
 (simplify
  (cmp @0 REAL_CST@1)
  /* IEEE doesn't distinguish +0 and -0 in comparisons.  */
  (switch
   /* a CMP (-0) -> a CMP 0  */
   (if (REAL_VALUE_MINUS_ZERO (TREE_REAL_CST (@1)))
    (cmp @0 { build_real (TREE_TYPE (@1), dconst0); }))
   /* (-0) CMP b -> 0 CMP b.  */
   (if (TREE_CODE (@0) == REAL_CST
	&& REAL_VALUE_MINUS_ZERO (TREE_REAL_CST (@0)))
    (cmp { build_real (TREE_TYPE (@0), dconst0); } @1))
   /* x != NaN is always true, other ops are always false.  */
   (if (REAL_VALUE_ISNAN (TREE_REAL_CST (@1))
	&& (cmp == EQ_EXPR || cmp == NE_EXPR || !flag_trapping_math)
	&& !tree_expr_signaling_nan_p (@1)
	&& !tree_expr_maybe_signaling_nan_p (@0))
    { constant_boolean_node (cmp == NE_EXPR, type); })
   /* NaN != y is always true, other ops are always false.  */
   (if (TREE_CODE (@0) == REAL_CST
	&& REAL_VALUE_ISNAN (TREE_REAL_CST (@0))
	&& (cmp == EQ_EXPR || cmp == NE_EXPR || !flag_trapping_math)
        && !tree_expr_signaling_nan_p (@0)
        && !tree_expr_signaling_nan_p (@1))
    { constant_boolean_node (cmp == NE_EXPR, type); })
   /* Fold comparisons against infinity.  */
   (if (REAL_VALUE_ISINF (TREE_REAL_CST (@1))
	&& MODE_HAS_INFINITIES (TYPE_MODE (TREE_TYPE (@1))))
    (with
     {
       REAL_VALUE_TYPE max;
       enum tree_code code = cmp;
       bool neg = REAL_VALUE_NEGATIVE (TREE_REAL_CST (@1));
       if (neg)
         code = swap_tree_comparison (code);
     }
     (switch
      /* x > +Inf is always false, if we ignore NaNs or exceptions.  */
      (if (code == GT_EXPR
	   && !(HONOR_NANS (@0) && flag_trapping_math))
       { constant_boolean_node (false, type); })
      (if (code == LE_EXPR)
       /* x <= +Inf is always true, if we don't care about NaNs.  */
       (if (! HONOR_NANS (@0))
	{ constant_boolean_node (true, type); }
	/* x <= +Inf is the same as x == x, i.e. !isnan(x), but this loses
	   an "invalid" exception.  */
	(if (!flag_trapping_math)
	 (eq @0 @0))))
      /* x == +Inf and x >= +Inf are always equal to x > DBL_MAX, but
	 for == this introduces an exception for x a NaN.  */
      (if ((code == EQ_EXPR && !(HONOR_NANS (@0) && flag_trapping_math))
	   || code == GE_EXPR)
       (with { real_maxval (&max, neg, TYPE_MODE (TREE_TYPE (@0))); }
	(if (neg)
	 (lt @0 { build_real (TREE_TYPE (@0), max); })
	 (gt @0 { build_real (TREE_TYPE (@0), max); }))))
      /* x < +Inf is always equal to x <= DBL_MAX.  */
      (if (code == LT_EXPR)
       (with { real_maxval (&max, neg, TYPE_MODE (TREE_TYPE (@0))); }
	(if (neg)
	 (ge @0 { build_real (TREE_TYPE (@0), max); })
	 (le @0 { build_real (TREE_TYPE (@0), max); }))))
      /* x != +Inf is always equal to !(x > DBL_MAX), but this introduces
	 an exception for x a NaN so use an unordered comparison.  */
      (if (code == NE_EXPR)
       (with { real_maxval (&max, neg, TYPE_MODE (TREE_TYPE (@0))); }
	(if (! HONOR_NANS (@0))
	 (if (neg)
	  (ge @0 { build_real (TREE_TYPE (@0), max); })
	  (le @0 { build_real (TREE_TYPE (@0), max); }))
	 (if (neg)
	  (unge @0 { build_real (TREE_TYPE (@0), max); })
	  (unle @0 { build_real (TREE_TYPE (@0), max); }))))))))))

 /* If this is a comparison of a real constant with a PLUS_EXPR
    or a MINUS_EXPR of a real constant, we can convert it into a
    comparison with a revised real constant as long as no overflow
    occurs when unsafe_math_optimizations are enabled.  */
 (if (flag_unsafe_math_optimizations)
  (for op (plus minus)
   (simplify
    (cmp (op @0 REAL_CST@1) REAL_CST@2)
    (with
     {
       tree tem = const_binop (op == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR,
			       TREE_TYPE (@1), @2, @1);
     }
     (if (tem && !TREE_OVERFLOW (tem))
      (cmp @0 { tem; }))))))

 /* Likewise, we can simplify a comparison of a real constant with
    a MINUS_EXPR whose first operand is also a real constant, i.e.
    (c1 - x) < c2 becomes x > c1-c2.  Reordering is allowed on
    floating-point types only if -fassociative-math is set.  */
 (if (flag_associative_math)
  (simplify
   (cmp (minus REAL_CST@0 @1) REAL_CST@2)
   (with { tree tem = const_binop (MINUS_EXPR, TREE_TYPE (@1), @0, @2); }
    (if (tem && !TREE_OVERFLOW (tem))
     (cmp { tem; } @1)))))

 /* Fold comparisons against built-in math functions.  */
 (if (flag_unsafe_math_optimizations && ! flag_errno_math)
  (for sq (SQRT)
   (simplify
    (cmp (sq @0) REAL_CST@1)
    (switch
     (if (REAL_VALUE_NEGATIVE (TREE_REAL_CST (@1)))
      (switch
       /* sqrt(x) < y is always false, if y is negative.  */
       (if (cmp == EQ_EXPR || cmp == LT_EXPR || cmp == LE_EXPR)
	{ constant_boolean_node (false, type); })
       /* sqrt(x) > y is always true, if y is negative and we
	  don't care about NaNs, i.e. negative values of x.  */
       (if (cmp == NE_EXPR || !HONOR_NANS (@0))
	{ constant_boolean_node (true, type); })
       /* sqrt(x) > y is the same as x >= 0, if y is negative.  */
       (ge @0 { build_real (TREE_TYPE (@0), dconst0); })))
     (if (real_equal (TREE_REAL_CST_PTR (@1), &dconst0))
      (switch
       /* sqrt(x) < 0 is always false.  */
       (if (cmp == LT_EXPR)
	{ constant_boolean_node (false, type); })
       /* sqrt(x) >= 0 is always true if we don't care about NaNs.  */
       (if (cmp == GE_EXPR && !HONOR_NANS (@0))
	{ constant_boolean_node (true, type); })
       /* sqrt(x) <= 0 -> x == 0.  */
       (if (cmp == LE_EXPR)
	(eq @0 @1))
       /* Otherwise sqrt(x) cmp 0 -> x cmp 0.  Here cmp can be >=, >,
          == or !=.  In the last case:

	    (sqrt(x) != 0) == (NaN != 0) == true == (x != 0)

	  if x is negative or NaN.  Due to -funsafe-math-optimizations,
	  the results for other x follow from natural arithmetic.  */
       (cmp @0 @1)))
     (if ((cmp == LT_EXPR
	   || cmp == LE_EXPR
	   || cmp == GT_EXPR
	   || cmp == GE_EXPR)
	  && !REAL_VALUE_ISNAN (TREE_REAL_CST (@1))
	  /* Give up for -frounding-math.  */
	  && !HONOR_SIGN_DEPENDENT_ROUNDING (TREE_TYPE (@0)))
      (with
       {
	 REAL_VALUE_TYPE c2;
	 enum tree_code ncmp = cmp;
	 const real_format *fmt
	   = REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (@0)));
	 real_arithmetic (&c2, MULT_EXPR,
			  &TREE_REAL_CST (@1), &TREE_REAL_CST (@1));
	 real_convert (&c2, fmt, &c2);
	 /* See PR91734: if c2 is inexact and sqrt(c2) < c (or sqrt(c2) >= c),
	    then change LT_EXPR into LE_EXPR or GE_EXPR into GT_EXPR.  */
	 if (!REAL_VALUE_ISINF (c2))
	   {
	     tree c3 = fold_const_call (CFN_SQRT, TREE_TYPE (@0),
					build_real (TREE_TYPE (@0), c2));
	     if (c3 == NULL_TREE || TREE_CODE (c3) != REAL_CST)
	       ncmp = ERROR_MARK;
	     else if ((cmp == LT_EXPR || cmp == GE_EXPR)
		      && real_less (&TREE_REAL_CST (c3), &TREE_REAL_CST (@1)))
	       ncmp = cmp == LT_EXPR ? LE_EXPR : GT_EXPR;
	     else if ((cmp == LE_EXPR || cmp == GT_EXPR)
		      && real_less (&TREE_REAL_CST (@1), &TREE_REAL_CST (c3)))
	       ncmp = cmp == LE_EXPR ? LT_EXPR : GE_EXPR;
	     else
	       {
		 /* With rounding to even, sqrt of up to 3 different values
		    gives the same normal result, so in some cases c2 needs
		    to be adjusted.  */
		 REAL_VALUE_TYPE c2alt, tow;
		 if (cmp == LT_EXPR || cmp == GE_EXPR)
		   tow = dconst0;
		 else
		   tow = dconstinf;
		 real_nextafter (&c2alt, fmt, &c2, &tow);
		 real_convert (&c2alt, fmt, &c2alt);
		 if (REAL_VALUE_ISINF (c2alt))
		   ncmp = ERROR_MARK;
		 else
		   {
		     c3 = fold_const_call (CFN_SQRT, TREE_TYPE (@0),
					   build_real (TREE_TYPE (@0), c2alt));
		     if (c3 == NULL_TREE || TREE_CODE (c3) != REAL_CST)
		       ncmp = ERROR_MARK;
		     else if (real_equal (&TREE_REAL_CST (c3),
					  &TREE_REAL_CST (@1)))
		       c2 = c2alt;
		   }
	       }
	   }
       }
       (if (cmp == GT_EXPR || cmp == GE_EXPR)
	(if (REAL_VALUE_ISINF (c2))
	 /* sqrt(x) > y is x == +Inf, when y is very large.  */
	 (if (HONOR_INFINITIES (@0))
	  (eq @0 { build_real (TREE_TYPE (@0), c2); })
	  { constant_boolean_node (false, type); })
	 /* sqrt(x) > c is the same as x > c*c.  */
	 (if (ncmp != ERROR_MARK)
	  (if (ncmp == GE_EXPR)
	   (ge @0 { build_real (TREE_TYPE (@0), c2); })
	   (gt @0 { build_real (TREE_TYPE (@0), c2); }))))
	/* else if (cmp == LT_EXPR || cmp == LE_EXPR)  */
	(if (REAL_VALUE_ISINF (c2))
	 (switch
	  /* sqrt(x) < y is always true, when y is a very large
	     value and we don't care about NaNs or Infinities.  */
	  (if (! HONOR_NANS (@0) && ! HONOR_INFINITIES (@0))
	   { constant_boolean_node (true, type); })
	  /* sqrt(x) < y is x != +Inf when y is very large and we
	     don't care about NaNs.  */
	  (if (! HONOR_NANS (@0))
	   (ne @0 { build_real (TREE_TYPE (@0), c2); }))
	  /* sqrt(x) < y is x >= 0 when y is very large and we
	     don't care about Infinities.  */
	  (if (! HONOR_INFINITIES (@0))
	   (ge @0 { build_real (TREE_TYPE (@0), dconst0); }))
	  /* sqrt(x) < y is x >= 0 && x != +Inf, when y is large.  */
	  (if (GENERIC)
	   (truth_andif
	    (ge @0 { build_real (TREE_TYPE (@0), dconst0); })
	    (ne @0 { build_real (TREE_TYPE (@0), c2); }))))
	 /* sqrt(x) < c is the same as x < c*c, if we ignore NaNs.  */
	 (if (ncmp != ERROR_MARK && ! HONOR_NANS (@0))
	  (if (ncmp == LT_EXPR)
	   (lt @0 { build_real (TREE_TYPE (@0), c2); })
	   (le @0 { build_real (TREE_TYPE (@0), c2); }))
	  /* sqrt(x) < c is the same as x >= 0 && x < c*c.  */
	  (if (ncmp != ERROR_MARK && GENERIC)
	   (if (ncmp == LT_EXPR)
	    (truth_andif
	     (ge @0 { build_real (TREE_TYPE (@0), dconst0); })
	     (lt @0 { build_real (TREE_TYPE (@0), c2); }))
	    (truth_andif
	     (ge @0 { build_real (TREE_TYPE (@0), dconst0); })
	     (le @0 { build_real (TREE_TYPE (@0), c2); })))))))))))
   /* Transform sqrt(x) cmp sqrt(y) -> x cmp y.  */
   (simplify
    (cmp (sq @0) (sq @1))
      (if (! HONOR_NANS (@0))
	(cmp @0 @1))))))

/* Optimize various special cases of (FTYPE) N CMP (FTYPE) M.  */
(for cmp  (lt le eq ne ge gt unordered ordered unlt unle ungt unge uneq ltgt)
     icmp (lt le eq ne ge gt unordered ordered lt   le   gt   ge   eq   ne)
 (simplify
  (cmp (float@0 @1) (float@3 @2))
   (if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (@0))
	&& ! DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@0)))
    (with
     {
       format_helper fmt (REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (@0))));
       tree type1 = TREE_TYPE (@1);
       bool type1_signed_p = TYPE_SIGN (type1) == SIGNED;
       bool type1_ok = fmt.can_represent_integral_type_p (type1);
       tree type2 = TREE_TYPE (@2);
       bool type2_signed_p = TYPE_SIGN (type2) == SIGNED;
       bool type2_ok = fmt.can_represent_integral_type_p (type2);
#if GIMPLE
       int_range_max vr1, vr2;
       if (!type1_ok
	   && gimple_match_range_of_expr (vr1, @1, @0))
	 type1_ok = fmt.can_represent_range_value_p (&vr1);
       if (!type2_ok
	   && gimple_match_range_of_expr (vr2, @2, @3))
	 type2_ok = fmt.can_represent_range_value_p (&vr2);
#endif
     }
     (if (type1_ok && type2_ok)
      (if (cmp == ORDERED_EXPR || cmp == UNORDERED_EXPR)
       { constant_boolean_node (cmp == ORDERED_EXPR, type); }
       (if (TYPE_PRECISION (type1) > TYPE_PRECISION (type2)
            && type1_signed_p >= type2_signed_p)
        (icmp @1 (convert @2))
        (if (TYPE_PRECISION (type1) < TYPE_PRECISION (type2)
             && type1_signed_p <= type2_signed_p)
         (icmp (convert:type2 @1) @2)
         (if (TYPE_PRECISION (type1) == TYPE_PRECISION (type2)
              && type1_signed_p == type2_signed_p)
	  (icmp @1 (convert @2)))))))))))

/* Optimize various special cases of (FTYPE) N CMP CST.  */
(for cmp  (lt le eq ne ge gt)
     icmp (le le eq ne ge ge)
 (simplify
  (cmp (float@2 @0) REAL_CST@1)
   (if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (@1))
	&& ! DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@1)))
    (with
     {
       tree itype = TREE_TYPE (@0);
       format_helper fmt (REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (@1))));
       bool value_ok = fmt.can_represent_integral_type_p (itype);
       const REAL_VALUE_TYPE *cst = TREE_REAL_CST_PTR (@1);
       /* Be careful to preserve any potential exceptions due to NaNs.
	  qNaNs are ok in == or != context.  */
       bool exception_p = real_isnan (cst)
			  && flag_trapping_math
			  && ((cmp != EQ_EXPR && cmp != NE_EXPR)
			      || (cst->signalling
				  && HONOR_SNANS (TREE_TYPE (@1))));
#if GIMPLE
       int_range_max vr;
       if (gimple_match_range_of_expr (vr, @0, @2))
	 value_ok = fmt.can_represent_range_value_p (&vr);
#endif
       /* Conversion may raise FPE_INEXACT with -ftrapping-math.  */
       if (flag_trapping_math && !value_ok)
	 exception_p = true;
     }

     (if (! exception_p)
      (with
       {
	 signop isign = TYPE_SIGN (itype);
	 wide_int imin_val = wi::min_value (itype);
	 wide_int imax_val = wi::max_value (itype);
#if GIMPLE
	 if (!vr.undefined_p ())
	   {
	     imin_val = vr.lower_bound ();
	     imax_val = vr.upper_bound ();
	   }
#endif
	 REAL_VALUE_TYPE imin, imax;
	 real_from_integer (&imin, fmt, imin_val, isign);
	 real_from_integer (&imax, fmt, imax_val, isign);

	 REAL_VALUE_TYPE icst;
	 if (cmp == GT_EXPR || cmp == GE_EXPR)
	   real_ceil (&icst, fmt, cst);
	 else if (cmp == LT_EXPR || cmp == LE_EXPR)
	   real_floor (&icst, fmt, cst);
	 else
	   real_trunc (&icst, fmt, cst);

	 bool cst_int_p = real_isfinite (cst) && real_identical (&icst, cst);

	 bool overflow_p = false;
	 wide_int icst_val
	   = real_to_integer (&icst, &overflow_p, TYPE_PRECISION (itype));
	 /* Double check that icst is representable in itype.  */
	 if (isign == SIGNED && !overflow_p)
	   {
	     REAL_VALUE_TYPE check;
	     real_from_integer (&check, fmt, icst_val, isign);
	     if (!real_identical (&check, &icst))
	       overflow_p = true;
	   }
       }
       (switch
	/* Optimize cases when CST is outside of ITYPE's range.  */
	(if (real_compare (LT_EXPR, cst, &imin))
	 { constant_boolean_node (cmp == GT_EXPR || cmp == GE_EXPR || cmp == NE_EXPR,
				  type); })
	(if (real_compare (GT_EXPR, cst, &imax))
	 { constant_boolean_node (cmp == LT_EXPR || cmp == LE_EXPR || cmp == NE_EXPR,
				  type); })
	/* When CST is fractional, infinity or NaN, then optimize
	    (FTYPE) N == CST -> 0
	    (FTYPE) N != CST -> 1.  */
	(if (!cst_int_p
	     && (cmp == EQ_EXPR || cmp == NE_EXPR))
	 { constant_boolean_node (cmp == NE_EXPR, type); })
	/* Remove cast if CST is an integer representable by ITYPE,
	   and it is safe to do so.  */
	(if (cst_int_p
	     && !overflow_p
	     && fold_cmp_float_cst_p (imin_val, imax_val, cmp, &icst, fmt,
				      icst_val, isign))
	 (cmp @0 { wide_int_to_tree (itype, icst_val); })
	)
	/* Otherwise replace with sensible integer constant.  */
	(if (!cst_int_p
	     && !overflow_p
	     && fold_cmp_float_cst_p (imin_val, imax_val, icmp, &icst, fmt,
				      icst_val, isign))
	 (icmp @0 { wide_int_to_tree (itype, icst_val); })))))))))

/* Fold A /[ex] B CMP C to A CMP B * C.  */
(for cmp (eq ne)
 (simplify
  (cmp (exact_div @0 @1) INTEGER_CST@2)
  (if (!integer_zerop (@1))
   (if (wi::to_wide (@2) == 0)
    (cmp @0 @2)
    (if (TREE_CODE (@1) == INTEGER_CST)
     (with
      {
	wi::overflow_type ovf;
	wide_int prod = wi::mul (wi::to_wide (@2), wi::to_wide (@1),
				 TYPE_SIGN (TREE_TYPE (@1)), &ovf);
      }
      (if (ovf)
       { constant_boolean_node (cmp == NE_EXPR, type); }
       (cmp @0 { wide_int_to_tree (TREE_TYPE (@0), prod); }))))))))
(for cmp (lt le gt ge)
 (simplify
  (cmp (exact_div @0 INTEGER_CST@1) INTEGER_CST@2)
  (if (wi::gt_p (wi::to_wide (@1), 0, TYPE_SIGN (TREE_TYPE (@1))))
   (with
    {
      wi::overflow_type ovf;
      wide_int prod = wi::mul (wi::to_wide (@2), wi::to_wide (@1),
			       TYPE_SIGN (TREE_TYPE (@1)), &ovf);
    }
    (if (ovf)
     { constant_boolean_node (wi::lt_p (wi::to_wide (@2), 0,
					TYPE_SIGN (TREE_TYPE (@2)))
			      != (cmp == LT_EXPR || cmp == LE_EXPR), type); }
     (cmp @0 { wide_int_to_tree (TREE_TYPE (@0), prod); }))))))

/* Fold (size_t)(A /[ex] B) CMP C to (size_t)A CMP (size_t)B * C or A CMP' 0.

   For small C (less than max/B), this is (size_t)A CMP (size_t)B * C.
   For large C (more than min/B+2^size), this is also true, with the
   multiplication computed modulo 2^size.
   For intermediate C, this just tests the sign of A.  */
(for cmp  (lt le gt ge)
     cmp2 (ge ge lt lt)
 (simplify
  (cmp (convert (exact_div @0 INTEGER_CST@1)) INTEGER_CST@2)
  (if (tree_nop_conversion_p (TREE_TYPE (@0), TREE_TYPE (@2))
       && TYPE_UNSIGNED (TREE_TYPE (@2)) && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && wi::gt_p (wi::to_wide (@1), 0, TYPE_SIGN (TREE_TYPE (@1))))
   (with
    {
      tree utype = TREE_TYPE (@2);
      wide_int denom = wi::to_wide (@1);
      wide_int right = wi::to_wide (@2);
      wide_int smax = wi::sdiv_trunc (wi::max_value (TREE_TYPE (@0)), denom);
      wide_int smin = wi::sdiv_trunc (wi::min_value (TREE_TYPE (@0)), denom);
      bool small = wi::leu_p (right, smax);
      bool large = wi::geu_p (right, smin);
    }
    (if (small || large)
     (cmp (convert:utype @0) (mult @2 (convert @1)))
     (cmp2 @0 { build_zero_cst (TREE_TYPE (@0)); }))))))

/* Unordered tests if either argument is a NaN.  */
(for bitop (bit_ior   bit_and)
     op    (unordered ordered)
 (simplify
  (bitop (op @0 @0) (op @1 @1))
  (if (types_match (@0, @1))
   (op @0 @1)))
 (simplify
  (bitop:c (op @0 @0) (op:c@2 @0 @1))
  @2))

/* A & (2**N - 1) <= 2**K - 1 -> A & (2**N - 2**K) == 0
   A & (2**N - 1) >  2**K - 1 -> A & (2**N - 2**K) != 0

   Note that comparisons
     A & (2**N - 1) <  2**K   -> A & (2**N - 2**K) == 0
     A & (2**N - 1) >= 2**K   -> A & (2**N - 2**K) != 0
   will be canonicalized to above so there's no need to
   consider them here.
 */

(for cmp (le gt)
     eqcmp (eq ne)
 (simplify
  (cmp (bit_and@0 @1 INTEGER_CST@2) INTEGER_CST@3)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))
   (with
    {
     tree ty = TREE_TYPE (@0);
     unsigned prec = TYPE_PRECISION (ty);
     wide_int mask = wi::to_wide (@2, prec);
     wide_int rhs = wi::to_wide (@3, prec);
     signop sgn = TYPE_SIGN (ty);
    }
    (if ((mask & (mask + 1)) == 0 && wi::gt_p (rhs, 0, sgn)
	 && (rhs & (rhs + 1)) == 0 && wi::ge_p (mask, rhs, sgn))
      (eqcmp (bit_and @1 { wide_int_to_tree (ty, mask - rhs); })
	     { build_zero_cst (ty); }))))))

/* -A CMP -B -> B CMP A.  */
(for cmp (tcc_comparison)
     scmp (swapped_tcc_comparison)
 (simplify
  (cmp (negate @0) (negate @1))
  (if (FLOAT_TYPE_P (TREE_TYPE (@0))
       || (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && (cmp == EQ_EXPR
	       || cmp == NE_EXPR
	       || TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))))
   (scmp @0 @1)))
 (simplify
  (cmp (negate @0) CONSTANT_CLASS_P@1)
  (if (FLOAT_TYPE_P (TREE_TYPE (@0))
       || (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && (cmp == EQ_EXPR
	       || cmp == NE_EXPR
	       || TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))))
   (with { tree tem = const_unop (NEGATE_EXPR, TREE_TYPE (@0), @1); }
    (if (tem && !TREE_OVERFLOW (tem))
     (scmp @0 { tem; }))))))

/* Convert ABS[U]_EXPR<x> == 0 or ABS[U]_EXPR<x> != 0 to x == 0 or x != 0.  */
(for op (abs absu)
 (for eqne (eq ne)
  (simplify
   (eqne (op @0) zerop@1)
   (eqne @0 { build_zero_cst (TREE_TYPE (@0)); }))))

#if GIMPLE
/* ABS_EXPR <X> > C -> (unsigned) (X + C) > 2 * C, and the same for <= and
   for ABSU_EXPR.  */
(for op (abs absu)
 (for cmp (gt le)
  (simplify
   (cmp (op@2 @0) INTEGER_CST@1)
   (if ((op == ABSU_EXPR || TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
	&& !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
	&& single_use (@2)
	&& wi::ge_p (wi::to_wide (@1), 0, SIGNED)
	&& !tree_int_cst_equal (@1, TYPE_MAX_VALUE (TREE_TYPE (@0))))
    (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
     (cmp (plus (convert:utype @0) (convert:utype @1))
	  { wide_int_to_tree (utype, wi::to_wide (@1) * 2); }))))))
#endif

/* From fold_sign_changed_comparison and fold_widened_comparison.
   FIXME: the lack of symmetry is disturbing.  */
(for cmp (simple_comparison)
 (simplify
  (cmp (convert@0 @00) (convert?@1 @10))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       /* Disable this optimization if we're casting a function pointer
	  type on targets that require function pointer canonicalization.  */
       && !(targetm.have_canonicalize_funcptr_for_compare ()
	    && ((POINTER_TYPE_P (TREE_TYPE (@00))
		 && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (@00))))
		|| (POINTER_TYPE_P (TREE_TYPE (@10))
		    && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (@10))))))
       && single_use (@0))
   (if (TYPE_PRECISION (TREE_TYPE (@00)) == TYPE_PRECISION (TREE_TYPE (@0))
	&& (TREE_CODE (@10) == INTEGER_CST
	    || @1 != @10)
	&& (TYPE_UNSIGNED (TREE_TYPE (@00)) == TYPE_UNSIGNED (TREE_TYPE (@0))
	    || cmp == NE_EXPR
	    || cmp == EQ_EXPR)
	&& !POINTER_TYPE_P (TREE_TYPE (@00))
	/* (int)bool:32 != (int)uint is not the same as
	   bool:32 != (bool:32)uint since boolean types only have two valid
	   values independent of their precision.  */
	&& (TREE_CODE (TREE_TYPE (@00)) != BOOLEAN_TYPE
	    || TREE_CODE (TREE_TYPE (@10)) == BOOLEAN_TYPE))
    /* ???  The special-casing of INTEGER_CST conversion was in the original
       code and here to avoid a spurious overflow flag on the resulting
       constant which fold_convert produces.  */
    (if (TREE_CODE (@1) == INTEGER_CST)
     (cmp @00 { force_fit_type (TREE_TYPE (@00),
				wide_int::from (wi::to_wide (@1),
						MAX (TYPE_PRECISION (TREE_TYPE (@1)),
						     TYPE_PRECISION (TREE_TYPE (@00))),
						TYPE_SIGN (TREE_TYPE (@1))),
				0, TREE_OVERFLOW (@1)); })
     (cmp @00 (convert @1)))

    (if (TYPE_PRECISION (TREE_TYPE (@0)) > TYPE_PRECISION (TREE_TYPE (@00)))
     /* If possible, express the comparison in the shorter mode.  */
     (if ((cmp == EQ_EXPR || cmp == NE_EXPR
	   || TYPE_UNSIGNED (TREE_TYPE (@0)) == TYPE_UNSIGNED (TREE_TYPE (@00))
	   || (!TYPE_UNSIGNED (TREE_TYPE (@0))
	       && TYPE_UNSIGNED (TREE_TYPE (@00))))
	  && (types_match (TREE_TYPE (@10), TREE_TYPE (@00))
	      || ((TYPE_PRECISION (TREE_TYPE (@00))
		   >= TYPE_PRECISION (TREE_TYPE (@10)))
		  && (TYPE_UNSIGNED (TREE_TYPE (@00))
		      == TYPE_UNSIGNED (TREE_TYPE (@10))))
	      || (TREE_CODE (@1) == INTEGER_CST
		  && INTEGRAL_TYPE_P (TREE_TYPE (@00))
		  && int_fits_type_p (@1, TREE_TYPE (@00)))))
      (cmp @00 (convert @10))
      (if (TREE_CODE (@1) == INTEGER_CST
	   && INTEGRAL_TYPE_P (TREE_TYPE (@00))
	   && !int_fits_type_p (@1, TREE_TYPE (@00)))
       (with
	{
	  tree min = lower_bound_in_type (TREE_TYPE (@10), TREE_TYPE (@00));
	  tree max = upper_bound_in_type (TREE_TYPE (@10), TREE_TYPE (@00));
	  bool above = integer_nonzerop (const_binop (LT_EXPR, type, max, @1));
	  bool below = integer_nonzerop (const_binop (LT_EXPR, type, @1, min));
	}
	(if (above || below)
	 (if (cmp == EQ_EXPR || cmp == NE_EXPR)
	  { constant_boolean_node (cmp != EQ_EXPR, type); }
	  (if (cmp == LT_EXPR || cmp == LE_EXPR)
	   { constant_boolean_node (above, type); }
	   (if (cmp == GT_EXPR || cmp == GE_EXPR)
	    { constant_boolean_node (!above, type); })))))))

    /* For eq/ne with narrowing conversion:
       (T)(X) == (T)(Y) -> (T)(X ^ Y) == 0  */
    (if (INTEGRAL_TYPE_P (TREE_TYPE (@00))
	 && (TYPE_PRECISION (TREE_TYPE (@0))
             < TYPE_PRECISION (TREE_TYPE (@00)))
         && (cmp == EQ_EXPR || cmp == NE_EXPR)
         && types_match (TREE_TYPE (@00),
                         TREE_TYPE (@10)))
     (with { tree itype = TREE_TYPE (@0); }
      (cmp (convert:itype (bit_xor @00 @10))
           { build_zero_cst (itype); })))))

   /* Fold (double)float1 CMP (double)float2 into float1 CMP float2.  */
   (if (FLOAT_TYPE_P (TREE_TYPE (@00))
	&& (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@0))
	    == DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@00)))
	&& (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@0))
	    == DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@10))))
    (with
     {
       tree type1 = TREE_TYPE (@10);
       if (TREE_CODE (@10) == REAL_CST && !DECIMAL_FLOAT_TYPE_P (type1))
	 {
	   REAL_VALUE_TYPE orig = TREE_REAL_CST (@10);
	   if (TYPE_PRECISION (type1) > TYPE_PRECISION (float_type_node)
	       && exact_real_truncate (TYPE_MODE (float_type_node), &orig))
	     type1 = float_type_node;
	   if (TYPE_PRECISION (type1) > TYPE_PRECISION (double_type_node)
	       && exact_real_truncate (TYPE_MODE (double_type_node), &orig))
	     type1 = double_type_node;
	 }
       tree newtype
	 = (element_precision (TREE_TYPE (@00)) > element_precision (type1)
	    ? TREE_TYPE (@00) : type1);
     }
     (if (element_precision (TREE_TYPE (@0)) > element_precision (newtype)
	  && (!VECTOR_TYPE_P (type) || is_truth_type_for (newtype, type)))
      (cmp (convert:newtype @00) (convert:newtype @10))))))))


(for cmp (eq ne)
 (simplify
  /* ADDRs are canonicalized to 2nd place.  */
  (cmp SSA_NAME@1 addr@0)
  (with
   {
     poly_int64 off; tree base;
     tree addr = (TREE_CODE (@0) == SSA_NAME
		  ? gimple_assign_rhs1 (SSA_NAME_DEF_STMT (@0)) : @0);
   }
   /* A local variable can never be pointed to by
      the default SSA name of an incoming parameter.  */
   (if (SSA_NAME_IS_DEFAULT_DEF (@1)
	&& TREE_CODE (SSA_NAME_VAR (@1)) == PARM_DECL
	&& (base = get_base_address (TREE_OPERAND (addr, 0)))
	&& TREE_CODE (base) == VAR_DECL
	&& auto_var_in_fn_p (base, current_function_decl))
    (if (cmp == NE_EXPR)
     { constant_boolean_node (true, type); }
     { constant_boolean_node (false, type); })
    /* If the address is based on @1 decide using the offset.  */
    (if ((base = get_addr_base_and_unit_offset (TREE_OPERAND (addr, 0), &off))
	 && TREE_CODE (base) == MEM_REF
	 && TREE_OPERAND (base, 0) == @1)
     (with { off += mem_ref_offset (base).force_shwi (); }
      (if (known_ne (off, 0))
       { constant_boolean_node (cmp == NE_EXPR, type); }
       (if (known_eq (off, 0))
        { constant_boolean_node (cmp == EQ_EXPR, type); }))))))))

/* Equality compare simplifications from fold_binary  */
(for cmp (eq ne)

 /* If we have (A | C) == D where C & ~D != 0, convert this into 0.
    Similarly for NE_EXPR.  */
 (simplify
  (cmp (convert?@3 (bit_ior @0 INTEGER_CST@1)) INTEGER_CST@2)
  (if (tree_nop_conversion_p (TREE_TYPE (@3), TREE_TYPE (@0))
       && wi::bit_and_not (wi::to_wide (@1), wi::to_wide (@2)) != 0)
   { constant_boolean_node (cmp == NE_EXPR, type); }))

 /* (X ^ Y) == 0 becomes X == Y, and (X ^ Y) != 0 becomes X != Y.  */
 (simplify
  (cmp (bit_xor @0 @1) integer_zerop)
  (cmp @0 @1))

 /* (X ^ Y) == Y becomes X == 0.
    Likewise (X ^ Y) == X becomes Y == 0.  */
 (simplify
  (cmp:c (bit_xor:c @0 @1) @0)
  (cmp @1 { build_zero_cst (TREE_TYPE (@1)); }))

 /* (X & Y) == X becomes (X & ~Y) == 0.  */
 (simplify
  (cmp:c (bit_and:c @0 @1) @0)
  (cmp (bit_and @0 (bit_not! @1)) { build_zero_cst (TREE_TYPE (@0)); }))
 (simplify
  (cmp:c (convert@3 (bit_and (convert@2 @0) INTEGER_CST@1)) (convert @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@2))
       && INTEGRAL_TYPE_P (TREE_TYPE (@3))
       && TYPE_PRECISION (TREE_TYPE (@2)) == TYPE_PRECISION (TREE_TYPE (@0))
       && TYPE_PRECISION (TREE_TYPE (@3)) > TYPE_PRECISION (TREE_TYPE (@2))
       && !wi::neg_p (wi::to_wide (@1)))
   (cmp (bit_and @0 (convert (bit_not @1)))
	{ build_zero_cst (TREE_TYPE (@0)); })))

 /* (X | Y) == Y becomes (X & ~Y) == 0.  */
 (simplify
  (cmp:c (bit_ior:c @0 @1) @1)
  (cmp (bit_and @0 (bit_not! @1)) { build_zero_cst (TREE_TYPE (@0)); }))

 /* (X ^ C1) op C2 can be rewritten as X op (C1 ^ C2).  */
 (simplify
  (cmp (convert?@3 (bit_xor @0 INTEGER_CST@1)) INTEGER_CST@2)
  (if (tree_nop_conversion_p (TREE_TYPE (@3), TREE_TYPE (@0)))
   (cmp @0 (bit_xor @1 (convert @2)))))

 (simplify
  (cmp (nop_convert? @0) integer_zerop)
  (if (tree_expr_nonzero_p (@0))
   { constant_boolean_node (cmp == NE_EXPR, type); }))

 /* (X & C) op (Y & C) into (X ^ Y) & C op 0.  */
 (simplify
  (cmp (bit_and:cs @0 @2) (bit_and:cs @1 @2))
  (cmp (bit_and (bit_xor @0 @1) @2)
       { build_zero_cst (TREE_TYPE (@2)); }))

 /* (X | C) op (Y | C) into (X ^ Y) & ~C op 0, the dual of the rule above.
    The two values agree on the bits C forces, so only the bits outside C
    can differ.  */
 (simplify
  (cmp (bit_ior:cs @0 @2) (bit_ior:cs @1 @2))
  (cmp (bit_and (bit_xor @0 @1) (bit_not! @2))
       { build_zero_cst (TREE_TYPE (@2)); })))

/* (X < 0) != (Y < 0) into (X ^ Y) < 0.
   (X >= 0) != (Y >= 0) into (X ^ Y) < 0.
   (X < 0) == (Y < 0) into (X ^ Y) >= 0.
   (X >= 0) == (Y >= 0) into (X ^ Y) >= 0.  */
(for cmp (eq ne)
     ncmp (ge lt)
 (for sgncmp (ge lt)
  (simplify
   (cmp (sgncmp @0 integer_zerop@2) (sgncmp @1 integer_zerop))
   (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& !TYPE_UNSIGNED (TREE_TYPE (@0))
	&& types_match (@0, @1))
    (ncmp (bit_xor @0 @1) @2)))))
/* (X < 0) == (Y >= 0) into (X ^ Y) < 0.
   (X < 0) != (Y >= 0) into (X ^ Y) >= 0.  */
(for cmp (eq ne)
     ncmp (lt ge)
 (simplify
  (cmp:c (lt @0 integer_zerop@2) (ge @1 integer_zerop))
   (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& !TYPE_UNSIGNED (TREE_TYPE (@0))
	&& types_match (@0, @1))
    (ncmp (bit_xor @0 @1) @2))))

/* If we have (A & C) == C where C is a power of 2, convert this into
   (A & C) != 0.  Similarly for NE_EXPR.  */
(for cmp (eq ne)
     icmp (ne eq)
 (simplify
  (cmp (bit_and@2 @0 integer_pow2p@1) @1)
  (icmp @2 { build_zero_cst (TREE_TYPE (@0)); })))

/* If we have (A | C) == A where C is a power of 2, convert this into
   (A & C) != 0.  Similarly for NE_EXPR.  */
(for cmp (eq ne)
     icmp (ne eq)
 (simplify
  (cmp:c (bit_ior @0 integer_pow2p@1) @0)
  (icmp (bit_and @0 @1) { build_zero_cst (TREE_TYPE (@0)); })))

(for cmp (ge lt)
/* x < 0 ? ~y : y into (x >> (prec-1)) ^ y. */
/* x >= 0 ? ~y : y into ~((x >> (prec-1)) ^ y). */
 (simplify
  (cond (cmp @0 integer_zerop) (bit_not @1) @1)
   (if (INTEGRAL_TYPE_P (type)
	&& INTEGRAL_TYPE_P (TREE_TYPE (@0))
        && !TYPE_UNSIGNED (TREE_TYPE (@0))
        && TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (type))
    (with
     {
       tree shifter = build_int_cst (integer_type_node, TYPE_PRECISION (type) - 1);
     }
    (if (cmp == LT_EXPR)
     (bit_xor (convert (rshift @0 {shifter;})) @1)
     (bit_not (bit_xor (convert (rshift @0 {shifter;})) @1))))))
/* x < 0 ? y : ~y into ~((x >> (prec-1)) ^ y). */
/* x >= 0 ? y : ~y into (x >> (prec-1)) ^ y. */
 (simplify
  (cond (cmp @0 integer_zerop) @1 (bit_not @1))
   (if (INTEGRAL_TYPE_P (type)
	&& INTEGRAL_TYPE_P (TREE_TYPE (@0))
        && !TYPE_UNSIGNED (TREE_TYPE (@0))
        && TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (type))
    (with
     {
       tree shifter = build_int_cst (integer_type_node, TYPE_PRECISION (type) - 1);
     }
    (if (cmp == GE_EXPR)
     (bit_xor (convert (rshift @0 {shifter;})) @1)
     (bit_not (bit_xor (convert (rshift @0 {shifter;})) @1)))))))

/* If we have (A & C) != 0 ? D : 0 where C and D are powers of 2,
   convert this into a shift followed by ANDing with D.  */
(simplify
 (cond
  (ne (bit_and @0 integer_pow2p@1) integer_zerop)
  INTEGER_CST@2 integer_zerop)
 (if (!POINTER_TYPE_P (type) && integer_pow2p (@2))
  (with {
     int shift = (wi::exact_log2 (wi::to_wide (@2))
		  - wi::exact_log2 (wi::to_wide (@1)));
   }
   (if (shift > 0)
    (bit_and
     (lshift (convert @0) { build_int_cst (integer_type_node, shift); }) @2)
    (bit_and
     (convert (rshift @0 { build_int_cst (integer_type_node, -shift); }))
     @2)))))

/* If we have (A & C) != 0 where C is the sign bit of A, convert
   this into A < 0.  Similarly for (A & C) == 0 into A >= 0.  */
(for cmp (eq ne)
     ncmp (ge lt)
 (simplify
  (cmp (bit_and (convert?@2 @0) integer_pow2p@1) integer_zerop)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && type_has_mode_precision_p (TREE_TYPE (@0))
       && element_precision (@2) >= element_precision (@0)
       && wi::only_sign_bit_p (wi::to_wide (@1), element_precision (@0)))
   (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
    (ncmp (convert:stype @0) { build_zero_cst (stype); })))))

/* If we have A < 0 ? C : 0 where C is a power of 2, convert
   this into a right shift or sign extension followed by ANDing with C.  */
(simplify
 (cond
  (lt @0 integer_zerop)
  INTEGER_CST@1 integer_zerop)
 (if (integer_pow2p (@1)
      && !TYPE_UNSIGNED (TREE_TYPE (@0)))
  (with {
    int shift = element_precision (@0) - wi::exact_log2 (wi::to_wide (@1)) - 1;
   }
   (if (shift >= 0)
    (bit_and
     (convert (rshift @0 { build_int_cst (integer_type_node, shift); }))
     @1)
    /* Otherwise ctype must be wider than TREE_TYPE (@0) and pure
       sign extension followed by AND with C will achieve the effect.  */
    (bit_and (convert @0) @1)))))

/* (A | (convert?)(A != 0)) EQ|NE 0 -> A EQ|NE 0 (PR114969)

   The NE variant of this pattern is produced by the C++ code:

   (bool)(std::max (ulong_A, 0) | (ulong_A ? 1 : 0)).  */
(for cmp (eq ne)
 (simplify
  (cmp (bit_ior:c @0 (convert? (ne @0 integer_zerop))) integer_zerop)
  (cmp @0 { build_zero_cst (TREE_TYPE (@0)); })))

/* When the addresses are not directly of decls compare base and offset.
   This implements some remaining parts of fold_comparison address
   comparisons but still no complete part of it.  Still it is good
   enough to make fold_stmt not regress when not dispatching to fold_binary.  */
(for cmp (simple_comparison)
 (simplify
  (cmp (convert1?@2 addr@0) (convert2? addr@1))
  (with
   {
     poly_int64 off0, off1;
     tree base0, base1;
     int equal = address_compare (cmp, TREE_TYPE (@2), @0, @1, base0, base1,
				  off0, off1, GENERIC);
   }
   (if (equal == 1)
    (switch
     (if (cmp == EQ_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
      { constant_boolean_node (known_eq (off0, off1), type); })
     (if (cmp == NE_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
      { constant_boolean_node (known_ne (off0, off1), type); })
     (if (cmp == LT_EXPR && (known_lt (off0, off1) || known_ge (off0, off1)))
      { constant_boolean_node (known_lt (off0, off1), type); })
     (if (cmp == LE_EXPR && (known_le (off0, off1) || known_gt (off0, off1)))
      { constant_boolean_node (known_le (off0, off1), type); })
     (if (cmp == GE_EXPR && (known_ge (off0, off1) || known_lt (off0, off1)))
      { constant_boolean_node (known_ge (off0, off1), type); })
     (if (cmp == GT_EXPR && (known_gt (off0, off1) || known_le (off0, off1)))
      { constant_boolean_node (known_gt (off0, off1), type); }))
    (if (equal == 0)
     (switch
      (if (cmp == EQ_EXPR)
       { constant_boolean_node (false, type); })
      (if (cmp == NE_EXPR)
       { constant_boolean_node (true, type); })))))))

#if GIMPLE
/* a?~t:t -> (-(a))^t */
(simplify
 (cond @0 @1 @2)
 (with { bool wascmp; }
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_UNSIGNED (TREE_TYPE (@0))
       && bitwise_inverted_equal_p (@1, @2, wascmp)
       && (!wascmp || TYPE_PRECISION (type) == 1))
   (if ((!TYPE_UNSIGNED (type) && TREE_CODE (type) == BOOLEAN_TYPE)
	|| TYPE_PRECISION (type) == 1)
    (bit_xor (convert:type @0) @2)
    (bit_xor (negate (convert:type @0)) @2)))))
#endif

/* Simplify pointer equality compares using PTA.  */
(for neeq (ne eq)
 (simplify
  (neeq @0 @1)
  (if (POINTER_TYPE_P (TREE_TYPE (@0))
       && ptrs_compare_unequal (@0, @1))
   { constant_boolean_node (neeq != EQ_EXPR, type); })))

/* PR70920: Transform (intptr_t)x eq/ne CST to x eq/ne (typeof x) CST.
   and (typeof ptr_cst) x eq/ne ptr_cst to x eq/ne (typeof x) CST.
   Disable the transform if either operand is pointer to function.
   This broke pr22051-2.c for arm where function pointer
   canonicalizaion is not wanted.  */

(for cmp (ne eq)
 (simplify
  (cmp (convert @0) INTEGER_CST@1)
  (if (((POINTER_TYPE_P (TREE_TYPE (@0))
	 && !FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (@0)))
	 && INTEGRAL_TYPE_P (TREE_TYPE (@1))
	 /* Don't perform this optimization in GENERIC if @0 has reference
	    type when sanitizing.  See PR101210.  */
	 && !(GENERIC
	      && TREE_CODE (TREE_TYPE (@0)) == REFERENCE_TYPE
	      && (flag_sanitize & (SANITIZE_NULL | SANITIZE_ALIGNMENT))))
	|| (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	    && POINTER_TYPE_P (TREE_TYPE (@1))
	    && !FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (@1)))))
       && TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (TREE_TYPE (@1)))
   (cmp @0 (convert @1)))))

/* (a & -a) < 0 -> a == INT_MIN
   (a & -a) >= 0 -> a != INT_MIN.  */
(match (negate_maybe_without_ud @0)
 (negate @0))
(match (negate_maybe_without_ud @0)
 (convert (negate (nop_convert @0)))
 (if (types_match (type, @0))))

(for cmp (lt ge)
     eqcmp (eq ne)
 (simplify
  (cmp (nop_convert?@0 (bit_and:c (negate_maybe_without_ud @1) @1))
       integer_zerop)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && !TYPE_UNSIGNED (TREE_TYPE (@0))
       && !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@1)))
   (eqcmp @1 { wide_int_to_tree (TREE_TYPE (@1),
 wi::min_value (TYPE_PRECISION (TREE_TYPE (@1)), SIGNED)); }))))

/* Non-equality compare simplifications from fold_binary  */
(for cmp (lt gt le ge)
 /* Comparisons with the highest or lowest possible integer of
    the specified precision will have known values.  */
 (simplify
  (cmp (convert?@2 @0) uniform_integer_cst_p@1)
  (if ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
	|| POINTER_TYPE_P (TREE_TYPE (@1))
	|| VECTOR_INTEGER_TYPE_P (TREE_TYPE (@1)))
       && tree_nop_conversion_p (TREE_TYPE (@2), TREE_TYPE (@0)))
   (with
    {
      tree cst = uniform_integer_cst_p (@1);
      tree arg1_type = TREE_TYPE (cst);
      unsigned int prec = TYPE_PRECISION (arg1_type);
      wide_int max = wi::max_value (arg1_type);
      wide_int signed_max = wi::max_value (prec, SIGNED);
      wide_int min = wi::min_value (arg1_type);
    }
    (switch
     (if (wi::to_wide (cst) == max)
      (switch
       (if (cmp == GT_EXPR)
	{ constant_boolean_node (false, type); })
       (if (cmp == GE_EXPR)
	(eq @2 @1))
       (if (cmp == LE_EXPR)
	{ constant_boolean_node (true, type); })
       (if (cmp == LT_EXPR)
	(ne @2 @1))))
     (if (wi::to_wide (cst) == min)
      (switch
       (if (cmp == LT_EXPR)
        { constant_boolean_node (false, type); })
       (if (cmp == LE_EXPR)
        (eq @2 @1))
       (if (cmp == GE_EXPR)
        { constant_boolean_node (true, type); })
       (if (cmp == GT_EXPR)
        (ne @2 @1))))
     (if (wi::to_wide (cst) == max - 1)
      (switch
       (if (cmp == GT_EXPR)
	(eq @2 { build_uniform_cst (TREE_TYPE (@1),
				    wide_int_to_tree (TREE_TYPE (cst),
						      wi::to_wide (cst)
						      + 1)); }))
       (if (cmp == LE_EXPR)
	(ne @2 { build_uniform_cst (TREE_TYPE (@1),
				    wide_int_to_tree (TREE_TYPE (cst),
						      wi::to_wide (cst)
						      + 1)); }))))
     (if (wi::to_wide (cst) == min + 1)
      (switch
       (if (cmp == GE_EXPR)
        (ne @2 { build_uniform_cst (TREE_TYPE (@1),
				    wide_int_to_tree (TREE_TYPE (cst),
						      wi::to_wide (cst)
						      - 1)); }))
       (if (cmp == LT_EXPR)
        (eq @2 { build_uniform_cst (TREE_TYPE (@1),
				    wide_int_to_tree (TREE_TYPE (cst),
						      wi::to_wide (cst)
						      - 1)); }))))
     (if (wi::to_wide (cst) == signed_max
	  && TYPE_UNSIGNED (arg1_type)
	  && TYPE_MODE (arg1_type) != BLKmode
	  /* We will flip the signedness of the comparison operator
	     associated with the mode of @1, so the sign bit is
	     specified by this mode.  Check that @1 is the signed
	     max associated with this sign bit.  */
	  && prec == GET_MODE_PRECISION (SCALAR_INT_TYPE_MODE (arg1_type))
	  /* signed_type does not work on pointer types.  */
	  && INTEGRAL_TYPE_P (arg1_type))
      /* The following case also applies to X < signed_max+1
	 and X >= signed_max+1 because previous transformations.  */
      (if (cmp == LE_EXPR || cmp == GT_EXPR)
       (with { tree st = signed_type_for (TREE_TYPE (@1)); }
       	(switch
	 (if (cst == @1 && cmp == LE_EXPR)
	  (ge (convert:st @0) { build_zero_cst (st); }))
	 (if (cst == @1 && cmp == GT_EXPR)
	  (lt (convert:st @0) { build_zero_cst (st); }))
	 (if (cmp == LE_EXPR)
	  (ge (view_convert:st @0) { build_zero_cst (st); }))
	 (if (cmp == GT_EXPR)
	  (lt (view_convert:st @0) { build_zero_cst (st); })))))))))))

/* For nonnegative x, x < (typeof x)(x != 0) is always false.  */
(simplify
 (lt:c @0 (convert (ne tree_expr_nonnegative_p@0 integer_zerop)))
 { constant_boolean_node (false, type); })

/* x != (typeof x)(x == CST) -> CST == 0 ? 1 : (CST == 1 ? (x!=0&&x!=1) : x != 0) */
/* x != (typeof x)(x != CST) -> CST == 1 ? 1 : (CST == 0 ? (x!=0&&x!=1) : x != 1) */
/* x == (typeof x)(x == CST) -> CST == 0 ? 0 : (CST == 1 ? (x==0||x==1) : x == 0) */
/* x == (typeof x)(x != CST) -> CST == 1 ? 0 : (CST == 0 ? (x==0||x==1) : x == 1) */
(for outer (ne eq)
 (for inner (ne eq)
  (simplify
   (outer:c @0 (convert (inner @0 INTEGER_CST@1)))
   (with {
     bool cst1 = integer_onep (@1);
     bool cst0 = integer_zerop (@1);
     bool innereq = inner == EQ_EXPR;
     bool outereq = outer == EQ_EXPR;
    }
    (switch
     (if (innereq ? cst0 : cst1)
      { constant_boolean_node (!outereq, type); })
     (if (innereq ? cst1 : cst0)
      (with {
        tree utype = unsigned_type_for (TREE_TYPE (@0));
        tree ucst1 = build_one_cst (utype);
       }
       (if (!outereq)
        (gt (convert:utype @0) { ucst1; })
        (le (convert:utype @0) { ucst1; })
       )
      )
     )
     (with {
       tree value = build_int_cst (TREE_TYPE (@0), !innereq);
      }
      (if (outereq)
       (eq @0 { value; })
       (ne @0 { value; })
      )
     )
    )
   )
  )
 )
)

(for cmp (unordered ordered unlt unle ungt unge uneq ltgt)
 /* If the second operand is NaN, the result is constant.  */
 (simplify
  (cmp @0 REAL_CST@1)
  (if (REAL_VALUE_ISNAN (TREE_REAL_CST (@1))
       && (cmp != LTGT_EXPR || ! flag_trapping_math))
   { constant_boolean_node (cmp != ORDERED_EXPR && cmp != LTGT_EXPR,
			    type); })))

/* Fold UNORDERED if either operand must be NaN, or neither can be.  */
(simplify
  (unordered @0 @1)
  (switch
    (if (tree_expr_nan_p (@0) || tree_expr_nan_p (@1))
	{ constant_boolean_node (true, type); })
    (if (!tree_expr_maybe_nan_p (@0) && !tree_expr_maybe_nan_p (@1))
	{ constant_boolean_node (false, type); })))

/* Fold ORDERED if either operand must be NaN, or neither can be.  */
(simplify
  (ordered @0 @1)
  (switch
    (if (tree_expr_nan_p (@0) || tree_expr_nan_p (@1))
	{ constant_boolean_node (false, type); })
    (if (!tree_expr_maybe_nan_p (@0) && !tree_expr_maybe_nan_p (@1))
	{ constant_boolean_node (true, type); })))

/* bool_var != 0 becomes bool_var.  */
(simplify
 (ne @0 integer_zerop)
 (if (TREE_CODE (TREE_TYPE (@0)) == BOOLEAN_TYPE
      && types_match (type, TREE_TYPE (@0)))
  (non_lvalue @0)))
/* bool_var == 1 becomes bool_var.  */
(simplify
 (eq @0 integer_onep)
 (if (TREE_CODE (TREE_TYPE (@0)) == BOOLEAN_TYPE
      && types_match (type, TREE_TYPE (@0)))
  (non_lvalue @0)))
/* Do not handle
   bool_var == 0 becomes !bool_var or
   bool_var != 1 becomes !bool_var
   here because that only is good in assignment context as long
   as we require a tcc_comparison in GIMPLE_CONDs where we'd
   replace if (x == 0) with tem = ~x; if (tem != 0) which is
   clearly less optimal and which we'll transform again in forwprop.  */

#if GIMPLE
/* From fold_binary_op_with_conditional_arg handle the case of
   rewriting (a ? b : c) > d to a ? (b > d) : (c > d) when the
   compares simplify.
   This should be after the boolean comparison simplification so
   that it can remove the outer comparison before appling it to
   the inner condtional operands.  */
(for cmp (simple_comparison)
 (simplify
  (cmp:c (cond @0 @1 @2) @3)
  /* Do not move possibly trapping operations into the conditional as this
     pessimizes code and causes gimplification issues when applied late.  */
  (if (!FLOAT_TYPE_P (TREE_TYPE (@3))
       || !operation_could_trap_p (cmp, true, false, @3))
   (cond @0 (cmp!:type @1 @3) (cmp!:type @2 @3)))))

/* Similar to above:
   (c ? a : b) op d  ->  c ? (a op d) : (b op d)
   But with non-vector binary ops, most of them non-commutative.  */
(for op (plus minus mult bit_and bit_ior bit_xor
	 lshift rshift rdiv trunc_div ceil_div floor_div round_div exact_div
	 trunc_mod ceil_mod floor_mod round_mod min max pointer_diff
	 lrotate rrotate mult_highpart)
 (simplify
  (op (cond @0 @1 @2) @3)
  (cond @0 (op!:type @1 @3) (op!:type @2 @3)))

 /* Support the variant
    d op (c ? a : b)  ->  c ? (d op a) : (d op b)  */
 (simplify
  (op @3 (cond @0 @1 @2))
  (cond @0 (op!:type @3 @1) (op!:type @3 @2))))
#endif

/* Transform comparisons of the form (X & Y) CMP 0 to X CMP2 Z
   where ~Y + 1 == pow2 and Z = ~Y.  */
(for cst (VECTOR_CST INTEGER_CST)
 (for cmp (eq ne)
      icmp (le gt)
  (simplify
   (cmp (bit_and:c@2 @0 cst@1) integer_zerop)
    (with { tree csts = bitmask_inv_cst_vector_p (@1); }
     (if (csts && (VECTOR_TYPE_P (TREE_TYPE (@1)) || single_use (@2)))
      (with { auto optab = VECTOR_TYPE_P (TREE_TYPE (@1))
			 ? optab_vector : optab_default;
	      tree utype = unsigned_type_for (TREE_TYPE (@1)); }
       (if (target_supports_op_p (utype, icmp, optab)
	    || (optimize_vectors_before_lowering_p ()
		&& (!target_supports_op_p (type, cmp, optab)
		    || !target_supports_op_p (type, BIT_AND_EXPR, optab))))
	(if (TYPE_UNSIGNED (TREE_TYPE (@1)))
	 (icmp @0 { csts; })
	 (icmp (view_convert:utype @0) { csts; })))))))))

/* When one argument is a constant, overflow detection can be simplified.
   Currently restricted to single use so as not to interfere too much with
   ADD_OVERFLOW detection in tree-ssa-math-opts.cc.
   CONVERT?(CONVERT?(A) + CST) CMP A  ->  A CMP' CST' */
(for cmp (lt le ge gt)
     out (gt gt le le)
 (simplify
  (cmp:c (convert?@3 (plus@2 (convert?@4 @0) INTEGER_CST@1)) @0)
  (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@2))
       && types_match (TREE_TYPE (@0), TREE_TYPE (@3))
       && tree_nop_conversion_p (TREE_TYPE (@4), TREE_TYPE (@0))
       && wi::to_wide (@1) != 0
       && single_use (@2))
   (with {
     unsigned int prec = TYPE_PRECISION (TREE_TYPE (@0));
     signop sign = TYPE_SIGN (TREE_TYPE (@0));
    }
    (out @0 { wide_int_to_tree (TREE_TYPE (@0),
			        wi::max_value (prec, sign)
				- wi::to_wide (@1)); })))))

/* To detect overflow in unsigned A - B, A < B is simpler than A - B > A.
   However, the detection logic for SUB_OVERFLOW in tree-ssa-math-opts.cc
   expects the long form, so we restrict the transformation for now.  */
(for cmp (gt le)
 (simplify
  (cmp:c (minus@2 @0 @1) @0)
  (if (single_use (@2)
       && ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_UNSIGNED (TREE_TYPE (@0)))
   (cmp @1 @0))))

/* x SAT_ADD 0 == x.  */
(simplify
 (IFN_SAT_ADD:c @0 integer_zerop)
  @0)

/* Optimize A - B + -1 >= A into B >= A for unsigned comparisons.  */
(for cmp (ge lt)
 (simplify
  (cmp:c (plus (minus @0 @1) integer_minus_onep) @0)
   (if (ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& TYPE_UNSIGNED (TREE_TYPE (@0)))
    (cmp @1 @0))))

/* Testing for overflow is unnecessary if we already know the result.  */
/* A - B > A  */
(for cmp (gt le)
     out (ne eq)
 (simplify
  (cmp:c (realpart (IFN_SUB_OVERFLOW@2 @0 @1)) @0)
  (if (TYPE_UNSIGNED (TREE_TYPE (@0))
       && types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
   (out (imagpart @2) { build_zero_cst (TREE_TYPE (@0)); }))))
/* A + B < A  */
(for cmp (lt ge)
     out (ne eq)
 (simplify
  (cmp:c (realpart (IFN_ADD_OVERFLOW:c@2 @0 @1)) @0)
  (if (TYPE_UNSIGNED (TREE_TYPE (@0))
       && types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
   (out (imagpart @2) { build_zero_cst (TREE_TYPE (@0)); }))))

/* For unsigned operands, -1 / B < A checks whether A * B would overflow.
   Simplify it to __builtin_mul_overflow (A, B, <unused>).  */
(for cmp (lt ge)
     out (ne eq)
 (simplify
  (cmp:c (trunc_div:s integer_all_onesp @1) @0)
  (if (TYPE_UNSIGNED (TREE_TYPE (@0)) && !VECTOR_TYPE_P (TREE_TYPE (@0)))
   (with { tree t = TREE_TYPE (@0), cpx = build_complex_type (t); }
    (out (imagpart (IFN_MUL_OVERFLOW:cpx @0 @1)) { build_zero_cst (t); })))))

/* Similarly, for unsigned operands, (((type) A * B) >> prec) != 0
   (or == 0) where type is at least twice as wide as type of A and B,
   canonicalize to the following ((utype) A * B) > max (or <= max) pattern.  */
(for cmp (eq ne)
     out (le gt)
 (simplify
  (cmp (rshift (mult:s (convert@3 @0) (convert @1)) INTEGER_CST@2)
       integer_zerop)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@3))
       && TYPE_UNSIGNED (TREE_TYPE (@0))
       && (TYPE_PRECISION (TREE_TYPE (@3))
	   >= 2 * TYPE_PRECISION (TREE_TYPE (@0)))
       && tree_fits_uhwi_p (@2)
       && tree_to_uhwi (@2) == TYPE_PRECISION (TREE_TYPE (@0))
       && types_match (@0, @1)
       && TYPE_MAX_VALUE (TREE_TYPE (@0)))
   (with { tree utype = unsigned_type_for (TREE_TYPE (@3)); }
    (out (mult:utype (convert:utype @0) (convert:utype @1))
	 { fold_convert (utype, TYPE_MAX_VALUE (TREE_TYPE (@0))); })))))

/* For unsigned operands, ((type) A * B) > max where type is at least twice as wide
   as the type of unsigned A and B.
   Simplify it to __builtin_mul_overflow (A, B, <unused>). */
(for cmp (gt le)
     out (ne eq)
 (simplify
  (cmp (mult:s (convert@3 @0) (convert @1)) INTEGER_CST@2)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@3))
       && TYPE_UNSIGNED (TREE_TYPE (@0))
       && TYPE_UNSIGNED (TREE_TYPE (@3))
       && (TYPE_PRECISION (TREE_TYPE (@3))
	   >= 2 * TYPE_PRECISION (TREE_TYPE (@0)))
       && TYPE_MAX_VALUE (TREE_TYPE (@0))
       && (wi::to_widest (@2)
	   == wi::to_widest (TYPE_MAX_VALUE (TREE_TYPE (@0))))
       && types_match (@0, @1)
       && type_has_mode_precision_p (TREE_TYPE (@0))
       && (optab_handler (umulv4_optab, TYPE_MODE (TREE_TYPE (@0)))
	   != CODE_FOR_nothing))
   (with { tree t = TREE_TYPE (@0), cpx = build_complex_type (t); }
    (out (imagpart (IFN_MUL_OVERFLOW:cpx @0 @1)) { build_zero_cst (t); })))))

/* Demote operands of IFN_{ADD,SUB,MUL}_OVERFLOW.  */
(for ovf (IFN_ADD_OVERFLOW IFN_SUB_OVERFLOW IFN_MUL_OVERFLOW)
 (simplify
  (ovf (convert@2 @0) @1)
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@2))
       && TYPE_PRECISION (TREE_TYPE (@2)) > TYPE_PRECISION (TREE_TYPE (@0))
       && (!TYPE_UNSIGNED (TREE_TYPE (@2)) || TYPE_UNSIGNED (TREE_TYPE (@0))))
   (ovf @0 @1)))
 (simplify
  (ovf @1 (convert@2 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@2))
       && TYPE_PRECISION (TREE_TYPE (@2)) > TYPE_PRECISION (TREE_TYPE (@0))
       && (!TYPE_UNSIGNED (TREE_TYPE (@2)) || TYPE_UNSIGNED (TREE_TYPE (@0))))
   (ovf @1 @0))))

/* Optimize __builtin_mul_overflow_p (x, cst, (utype) 0) if all 3 types
   are unsigned to x > (umax / cst).  Similarly for signed type, but
   in that case it needs to be outside of a range.  */
(simplify
 (imagpart (IFN_MUL_OVERFLOW:cs@2 @0 integer_nonzerop@1))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_MAX_VALUE (TREE_TYPE (@0))
       && types_match (TREE_TYPE (@0), TREE_TYPE (TREE_TYPE (@2)))
       && int_fits_type_p (@1, TREE_TYPE (@0)))
   (if (TYPE_UNSIGNED (TREE_TYPE (@0)))
    (convert (gt @0 (trunc_div! { TYPE_MAX_VALUE (TREE_TYPE (@0)); } @1)))
    (if (TYPE_MIN_VALUE (TREE_TYPE (@0)))
     (if (!TYPE_UNSIGNED (TREE_TYPE (@1)) && integer_minus_onep (@1))
      (convert (eq @0 { TYPE_MIN_VALUE (TREE_TYPE (@0)); }))
      (with
       {
	 tree div = fold_convert (TREE_TYPE (@0), @1);
	 tree lo = int_const_binop (TRUNC_DIV_EXPR,
				    TYPE_MIN_VALUE (TREE_TYPE (@0)), div);
	 tree hi = int_const_binop (TRUNC_DIV_EXPR,
				    TYPE_MAX_VALUE (TREE_TYPE (@0)), div);
	 tree etype = range_check_type (TREE_TYPE (@0));
	 if (etype)
	   {
	     if (wi::neg_p (wi::to_wide (div)))
	       std::swap (lo, hi);
	     lo = fold_convert (etype, lo);
	     hi = fold_convert (etype, hi);
	     hi = int_const_binop (MINUS_EXPR, hi, lo);
	   }
       }
       (if (etype)
        (convert (gt (minus (convert:etype @0) { lo; }) { hi; })))))))))

/* Simplification of math builtins.  These rules must all be optimizations
   as well as IL simplifications.  If there is a possibility that the new
   form could be a pessimization, the rule should go in the canonicalization
   section that follows this one.

   Rules can generally go in this section if they satisfy one of
   the following:

   - the rule describes an identity

   - the rule replaces calls with something as simple as addition or
     multiplication

   - the rule contains unary calls only and simplifies the surrounding
     arithmetic.  (The idea here is to exclude non-unary calls in which
     one operand is constant and in which the call is known to be cheap
     when the operand has that value.)  */

(if (flag_unsafe_math_optimizations)
 /* Simplify x / sqrt(x) -> sqrt(x).  */
 (simplify
  (rdiv @0 (SQRT @0)) (SQRT @0))

 /* Simplify sqrt(x) * sqrt(x) -> x.  */
 (simplify
  (mult (SQRT_ALL@1 @0) @1)
  (if (!tree_expr_maybe_signaling_nan_p (@0))
   @0))

 (for op (plus minus)
  /* Simplify (A / C) +- (B / C) -> (A +- B) / C.  */
  (simplify
   (op (rdiv @0 @1)
       (rdiv @2 @1))
   (rdiv (op @0 @2) @1)))

 (for cmp (lt le gt ge)
      neg_cmp (gt ge lt le)
  /* Simplify (x * C1) cmp C2 -> x cmp (C2 / C1), where C1 != 0.  */
  (simplify
   (cmp (mult @0 REAL_CST@1) REAL_CST@2)
   (with
    { tree tem = const_binop (RDIV_EXPR, type, @2, @1); }
    (if (tem
	 && !(REAL_VALUE_ISINF (TREE_REAL_CST (tem))
	      || (real_zerop (tem) && !real_zerop (@1))))
     (switch
      (if (real_less (&dconst0, TREE_REAL_CST_PTR (@1)))
       (cmp @0 { tem; }))
      (if (real_less (TREE_REAL_CST_PTR (@1), &dconst0))
       (neg_cmp @0 { tem; })))))))

 /* Simplify sqrt(x) * sqrt(y) -> sqrt(x*y).  */
 (for root (SQRT CBRT)
  (simplify
   (mult (root:s @0) (root:s @1))
    (root (mult @0 @1))))

 /* Simplify expN(x) * expN(y) -> expN(x+y). */
 (for exps (EXP EXP2 EXP10 POW10)
  (simplify
   (mult (exps:s @0) (exps:s @1))
    (exps (plus @0 @1))))

 /* Simplify a/root(b/c) into a*root(c/b).  */
 (for root (SQRT CBRT)
  (simplify
   (rdiv @0 (root:s (rdiv:s @1 @2)))
    (mult @0 (root (rdiv @2 @1)))))

 /* Simplify x/expN(y) into x*expN(-y).  */
 (for exps (EXP EXP2 EXP10 POW10)
  (simplify
   (rdiv @0 (exps:s @1))
    (mult @0 (exps (negate @1)))))

 (for pow (POW_ALL)
  (if (! HONOR_INFINITIES (type)
       && ! flag_trapping_math
       && ! flag_errno_math)
   /* Simplify pow(1.0/x, y) into pow(x, -y).  */
   (simplify
    (pow (rdiv:s real_onep@0 @1) @2)
     (pow @1 (negate @2)))

   /* Simplify pow(0.0, x) into 0.0.  */
   (if (! HONOR_NANS (type) && ! HONOR_SIGNED_ZEROS (type))
    (simplify
     (pow real_zerop@0 @1)
      @0))))

 (if (! HONOR_SIGN_DEPENDENT_ROUNDING (type)
      && ! HONOR_NANS (type) && ! HONOR_INFINITIES (type)
      && ! flag_trapping_math
      && ! flag_errno_math)
  (for logs (LOG LOG2 LOG10)
   /* Simplify logN(1.0/a) into -logN(a).  */
   (simplify
    (logs (rdiv:s real_onep@0 @1))
     (negate (logs @1)))

   /* Simplify logN(C/a) into logN(C)-logN(a).  */
   (simplify
    (logs (rdiv:s REAL_CST@0 @1))
     (minus (logs! @0) (logs @1)))

   /* Simplify logN(a)+logN(b) into logN(a*b).  */
   (simplify
    (plus (logs:s @0) (logs:s @1))
     (logs (mult @0 @1)))

   /* Simplify logN(a)-logN(b) into logN(a/b).  */
   (simplify
    (minus (logs:s @0) (logs:s @1))
     (logs (rdiv @0 @1))))

   (for cmp (le ge)
    (for logs (LOG LOG2 LOG10)
	 exps (EXP EXP2 EXP10)
    /* Simplify logN (x) CMP CST into x CMP expN (CST) */
    (simplify
    (cmp:c (logs:s @0) REAL_CST@1)
     (cmp @0 (exps! @1)))

    /* Simplify expN (x) CMP CST into x CMP logN (CST) */
    (simplify
    (cmp:c (exps:s @0) REAL_CST@1)
     (cmp @0 (logs! @1))))))

 (for logs (LOG LOG2 LOG10 LOG10)
      exps (EXP EXP2 EXP10 POW10)
  /* logN(expN(x)) -> x.  */
  (simplify
   (logs (exps @0))
   @0)
  /* expN(logN(x)) -> x.  */
  (simplify
   (exps (logs @0))
   @0))

 /* Optimize logN(func()) for various exponential functions.  We
    want to determine the value "x" and the power "exponent" in
    order to transform logN(x**exponent) into exponent*logN(x).  */
 (for logs (LOG  LOG   LOG   LOG2 LOG2  LOG2  LOG10 LOG10)
      exps (EXP2 EXP10 POW10 EXP  EXP10 POW10 EXP   EXP2)
  (simplify
   (logs (exps @0))
   (if (SCALAR_FLOAT_TYPE_P (type))
    (with {
      tree x;
      switch (exps)
	{
	CASE_CFN_EXP:
	  /* Prepare to do logN(exp(exponent)) -> exponent*logN(e).  */
	  x = build_real_truncate (type, dconst_e ());
	  break;
	CASE_CFN_EXP2:
	  /* Prepare to do logN(exp2(exponent)) -> exponent*logN(2).  */
	  x = build_real (type, dconst2);
	  break;
	CASE_CFN_EXP10:
	CASE_CFN_POW10:
	  /* Prepare to do logN(exp10(exponent)) -> exponent*logN(10).  */
	  {
	    REAL_VALUE_TYPE dconst10;
	    real_from_integer (&dconst10, VOIDmode, 10, SIGNED);
	    x = build_real (type, dconst10);
	  }
	  break;
	default:
	  gcc_unreachable ();
	}
      }
     (mult (logs { x; }) @0)))))

 (for logs (LOG LOG
            LOG2 LOG2
	    LOG10 LOG10)
      exps (SQRT CBRT)
  (simplify
   (logs (exps @0))
   (if (SCALAR_FLOAT_TYPE_P (type))
    (with {
      tree x;
      switch (exps)
	{
	CASE_CFN_SQRT:
	  /* Prepare to do logN(sqrt(x)) -> 0.5*logN(x).  */
	  x = build_real (type, dconsthalf);
	  break;
	CASE_CFN_CBRT:
	  /* Prepare to do logN(cbrt(x)) -> (1/3)*logN(x).  */
	  x = build_real_truncate (type, dconst_third ());
	  break;
	default:
	  gcc_unreachable ();
	}
      }
     (mult { x; } (logs @0))))))

 /* logN(pow(x,exponent)) -> exponent*logN(x).  */
 (for logs (LOG LOG2 LOG10)
      pows (POW)
  (simplify
   (logs (pows @0 @1))
   (mult @1 (logs @0))))

 /* pow(C,x) -> exp(log(C)*x) if C > 0,
    or if C is a positive power of 2,
    pow(C,x) -> exp2(log2(C)*x).  */
#if GIMPLE
 (for pows (POW)
      exps (EXP)
      logs (LOG)
      exp2s (EXP2)
      log2s (LOG2)
  (simplify
   (pows REAL_CST@0 @1)
   (if (real_compare (GT_EXPR, TREE_REAL_CST_PTR (@0), &dconst0)
	&& real_isfinite (TREE_REAL_CST_PTR (@0))
	/* As libmvec doesn't have a vectorized exp2, defer optimizing
	   the use_exp2 case until after vectorization.  It seems actually
	   beneficial for all constants to postpone this until later,
	   because exp(log(C)*x), while faster, will have worse precision
	   and if x folds into a constant too, that is unnecessary
	   pessimization.  */
	&& canonicalize_math_after_vectorization_p ())
    (with {
       const REAL_VALUE_TYPE *const value = TREE_REAL_CST_PTR (@0);
       bool use_exp2 = false;
       if (targetm.libc_has_function (function_c99_misc, TREE_TYPE (@0))
	   && value->cl == rvc_normal)
	 {
	   REAL_VALUE_TYPE frac_rvt = *value;
	   SET_REAL_EXP (&frac_rvt, 1);
	   if (real_equal (&frac_rvt, &dconst1))
	     use_exp2 = true;
	 }
     }
     (if (!use_exp2)
      (if (optimize_pow_to_exp (@0, @1))
       (exps (mult (logs @0) @1)))
      (exp2s (mult (log2s @0) @1)))))))
#endif

 /* pow(C,x)*expN(y) -> expN(logN(C)*x+y) if C > 0.  */
 (for pows (POW)
      exps (EXP EXP2 EXP10 POW10)
      logs (LOG LOG2 LOG10 LOG10)
  (simplify
   (mult:c (pows:s REAL_CST@0 @1) (exps:s @2))
   (if (real_compare (GT_EXPR, TREE_REAL_CST_PTR (@0), &dconst0)
	&& real_isfinite (TREE_REAL_CST_PTR (@0)))
    (exps (plus (mult (logs @0) @1) @2)))))

 /* Simplify powi (powof2, i) to ldexp (1, i * log2 (powof2)). */
 (simplify
  (POWI REAL_CST@0 @1)
  (with { HOST_WIDE_INT tmp = 0; }
   (if (real_isinteger (&TREE_REAL_CST (@0), &tmp)
	&& tmp > 0 && pow2p_hwi (tmp))
    (LDEXP { build_one_cst (type); }
       (mult @1 {build_int_cst (integer_type_node,
	     exact_log2 (tmp)); })))))

 /* Simplify powof2 * ldexp (x, i) to ldexp (x, i + log2 (powof2)) */
 (simplify
  (mult:c REAL_CST@0 (LDEXP @1 @2))
  (with { HOST_WIDE_INT tmp = 0; }
   (if (real_isinteger (&TREE_REAL_CST (@0), &tmp)
	&& tmp > 0 && pow2p_hwi (tmp))
    (LDEXP @1 (plus {build_int_cst (integer_type_node,
	     exact_log2 (tmp)); } @2)))))

 /* Simplify a * ldexp (1., i) to ldexp (a, i). */
 (simplify
  (mult:c @0 (LDEXP REAL_CST@1 @2))
  (if (real_equal (TREE_REAL_CST_PTR (@1), &dconst1))
   (LDEXP @0 @2)))

 (for sqrts (SQRT)
      cbrts (CBRT)
      pows (POW)
      exps (EXP EXP2 EXP10 POW10)
  /* sqrt(expN(x)) -> expN(x*0.5).  */
  (simplify
   (sqrts (exps @0))
   (exps (mult @0 { build_real (type, dconsthalf); })))
  /* cbrt(expN(x)) -> expN(x/3).  */
  (simplify
   (cbrts (exps @0))
   (exps (mult @0 { build_real_truncate (type, dconst_third ()); })))
  /* pow(expN(x), y) -> expN(x*y).  */
  (simplify
   (pows (exps @0) @1)
   (exps (mult @0 @1))))

 /* tan(atan(x)) -> x.  */
 (for tans (TAN)
      atans (ATAN)
  (simplify
   (tans (atans @0))
   @0)))

 /* Simplify sin(atan(x)) -> x / sqrt(x*x + 1). */
 (for sins (SIN)
      atans (ATAN)
      sqrts (SQRT)
      copysigns (COPYSIGN)
  (simplify
   (sins (atans:s @0))
   (with
     {
      REAL_VALUE_TYPE r_cst;
      build_sinatan_real (&r_cst, type);
      tree t_cst = build_real (type, r_cst);
      tree t_one = build_one_cst (type);
     }
    (if (SCALAR_FLOAT_TYPE_P (type))
     (cond (lt (abs @0) { t_cst; })
      (rdiv @0 (sqrts (plus (mult @0 @0) { t_one; })))
      (copysigns { t_one; } @0))))))

/* Simplify cos(atan(x)) -> 1 / sqrt(x*x + 1). */
 (for coss (COS)
      atans (ATAN)
      sqrts (SQRT)
      copysigns (COPYSIGN)
  (simplify
   (coss (atans:s @0))
   (with
     {
      REAL_VALUE_TYPE r_cst;
      build_sinatan_real (&r_cst, type);
      tree t_cst = build_real (type, r_cst);
      tree t_one = build_one_cst (type);
      tree t_zero = build_zero_cst (type);
     }
    (if (SCALAR_FLOAT_TYPE_P (type))
     (cond (lt (abs @0) { t_cst; })
      (rdiv { t_one; } (sqrts (plus (mult @0 @0) { t_one; })))
      (copysigns { t_zero; } @0))))))

 (if (!flag_errno_math)
  /* Simplify sinh(atanh(x)) -> x / sqrt((1 - x)*(1 + x)). */
  (for sinhs (SINH)
       atanhs (ATANH)
       sqrts (SQRT)
   (simplify
    (sinhs (atanhs:s @0))
    (with { tree t_one = build_one_cst (type); }
    (rdiv @0 (sqrts (mult (minus { t_one; } @0) (plus { t_one; } @0)))))))

  /* Simplify cosh(atanh(x)) -> 1 / sqrt((1 - x)*(1 + x)) */
  (for coshs (COSH)
       atanhs (ATANH)
       sqrts (SQRT)
   (simplify
    (coshs (atanhs:s @0))
    (with { tree t_one = build_one_cst (type); }
    (rdiv { t_one; } (sqrts (mult (minus { t_one; } @0) (plus { t_one; } @0))))))))

/* cabs(x+0i) or cabs(0+xi) -> abs(x).  */
(simplify
 (CABS (complex:C @0 real_zerop@1))
 (abs @0))

/* trunc(trunc(x)) -> trunc(x), etc.  */
(for fns (TRUNC_ALL FLOOR_ALL CEIL_ALL ROUND_ALL NEARBYINT_ALL RINT_ALL)
 (simplify
  (fns (fns @0))
  (fns @0)))
/* f(x) -> x if x is integer valued and f does nothing for such values.  */
(for fns (TRUNC_ALL FLOOR_ALL CEIL_ALL ROUND_ALL NEARBYINT_ALL RINT_ALL)
 (simplify
  (fns integer_valued_real_p@0)
  @0))

/* hypot(x,0) and hypot(0,x) -> abs(x).  */
(simplify
 (HYPOT:c @0 real_zerop@1)
 (abs @0))

/* pow(1,x) -> 1.  */
(simplify
 (POW real_onep@0 @1)
 @0)

(simplify
 /* copysign(x,x) -> x.  */
 (COPYSIGN_ALL @0 @0)
 @0)

(simplify
 /* copysign(x,-x) -> -x.  */
 (COPYSIGN_ALL @0 (negate@1 @0))
 @1)

(simplify
 /* copysign(x,y) -> fabs(x) if y is nonnegative.  */
 (COPYSIGN_ALL @0 tree_expr_nonnegative_p@1)
 (abs @0))

(simplify
 /* fabs (copysign(x, y)) -> fabs (x).  */
 (abs (COPYSIGN_ALL @0 @1))
 (abs @0))

(for scale (LDEXP SCALBN SCALBLN)
 /* ldexp(0, x) -> 0.  */
 (simplify
  (scale real_zerop@0 @1)
  @0)
 /* ldexp(x, 0) -> x.  */
 (simplify
  (scale @0 integer_zerop@1)
  @0)
 /* ldexp(x, y) -> x if x is +-Inf or NaN.  */
 (simplify
  (scale REAL_CST@0 @1)
  (if (!real_isfinite (TREE_REAL_CST_PTR (@0)))
   @0)))

/* Canonicalization of sequences of math builtins.  These rules represent
   IL simplifications but are not necessarily optimizations.

   The sincos pass is responsible for picking "optimal" implementations
   of math builtins, which may be more complicated and can sometimes go
   the other way, e.g. converting pow into a sequence of sqrts.
   We only want to do these canonicalizations before the pass has run.  */

(if (flag_unsafe_math_optimizations && canonicalize_math_p ())
 /* Simplify tan(x) * cos(x) -> sin(x). */
 (simplify
  (mult:c (TAN:s @0) (COS:s @0))
   (SIN @0))

 /* Simplify x * pow(x,c) -> pow(x,c+1). */
 (simplify
  (mult:c @0 (POW:s @0 REAL_CST@1))
  (if (!TREE_OVERFLOW (@1))
   (POW @0 (plus @1 { build_one_cst (type); }))))

 /* Simplify sin(x) / cos(x) -> tan(x). */
 (simplify
  (rdiv (SIN:s @0) (COS:s @0))
   (TAN @0))

 /* Simplify sinh(x) / cosh(x) -> tanh(x). */
 (simplify
  (rdiv (SINH:s @0) (COSH:s @0))
   (TANH @0))

 /* Simplify tanh (x) / sinh (x) -> 1.0 / cosh (x). */
 (simplify
   (rdiv (TANH:s @0) (SINH:s @0))
   (rdiv {build_one_cst (type);} (COSH @0)))

 /* Simplify cos(x) / sin(x) -> 1 / tan(x). */
 (simplify
  (rdiv (COS:s @0) (SIN:s @0))
   (rdiv { build_one_cst (type); } (TAN @0)))

 /* Simplify sin(x) / tan(x) -> cos(x). */
 (simplify
  (rdiv (SIN:s @0) (TAN:s @0))
  (if (! HONOR_NANS (@0)
       && ! HONOR_INFINITIES (@0))
   (COS @0)))

 /* Simplify tan(x) / sin(x) -> 1.0 / cos(x). */
 (simplify
  (rdiv (TAN:s @0) (SIN:s @0))
  (if (! HONOR_NANS (@0)
       && ! HONOR_INFINITIES (@0))
   (rdiv { build_one_cst (type); } (COS @0))))

 /* Simplify pow(x,y) * pow(x,z) -> pow(x,y+z). */
 (simplify
  (mult (POW:s @0 @1) (POW:s @0 @2))
   (POW @0 (plus @1 @2)))

 /* Simplify pow(x,y) * pow(z,y) -> pow(x*z,y). */
 (simplify
  (mult (POW:s @0 @1) (POW:s @2 @1))
   (POW (mult @0 @2) @1))

 (if (! HONOR_INFINITIES (type) && ! flag_trapping_math)
  /* Simplify powi(1.0/x, y) into powi(x, -y).  */
  (simplify
   (POWI (rdiv@3 real_onep@0 @1) @2)
   (if (single_use (@3))
    (POWI @1 (negate @2))))

  /* Simplify powi(0.0, x) into 0.0.  */
  (if (! HONOR_NANS (type) && ! HONOR_SIGNED_ZEROS (type))
   (simplify
    (POWI real_zerop@0 @1)
     @0)))

 /* Simplify powi(x,y) * powi(z,y) -> powi(x*z,y). */
 (simplify
  (mult (POWI:s @0 @1) (POWI:s @2 @1))
   (POWI (mult @0 @2) @1))

 /* Simplify pow(x,c) / x -> pow(x,c-1). */
 (simplify
  (rdiv (POW:s @0 REAL_CST@1) @0)
  (if (!TREE_OVERFLOW (@1))
   (POW @0 (minus @1 { build_one_cst (type); }))))

 /* Simplify x / pow (y,z) -> x * pow(y,-z). */
 (simplify
  (rdiv @0 (POW:s @1 @2))
   (mult @0 (POW @1 (negate @2))))

 (for sqrts (SQRT)
      cbrts (CBRT)
      pows (POW)
  /* sqrt(sqrt(x)) -> pow(x,1/4).  */
  (simplify
   (sqrts (sqrts @0))
   (pows @0 { build_real (type, dconst_quarter ()); }))
  /* sqrt(cbrt(x)) -> pow(x,1/6).  */
  (simplify
   (sqrts (cbrts @0))
   (pows @0 { build_real_truncate (type, dconst_sixth ()); }))
  /* cbrt(sqrt(x)) -> pow(x,1/6).  */
  (simplify
   (cbrts (sqrts @0))
   (pows @0 { build_real_truncate (type, dconst_sixth ()); }))
  /* cbrt(cbrt(x)) -> pow(x,1/9), iff x is nonnegative.  */
  (simplify
   (cbrts (cbrts tree_expr_nonnegative_p@0))
   (pows @0 { build_real_truncate (type, dconst_ninth ()); }))
  /* sqrt(pow(x,y)) -> pow(|x|,y*0.5).  */
  (simplify
   (sqrts (pows @0 @1))
   (pows (abs @0) (mult @1 { build_real (type, dconsthalf); })))
  /* cbrt(pow(x,y)) -> pow(x,y/3), iff x is nonnegative.  */
  (simplify
   (cbrts (pows tree_expr_nonnegative_p@0 @1))
   (pows @0 (mult @1 { build_real_truncate (type, dconst_third ()); })))
  /* pow(sqrt(x),y) -> pow(x,y*0.5).  */
  (simplify
   (pows (sqrts @0) @1)
   (pows @0 (mult @1 { build_real (type, dconsthalf); })))
  /* pow(cbrt(x),y) -> pow(x,y/3) iff x is nonnegative.  */
  (simplify
   (pows (cbrts tree_expr_nonnegative_p@0) @1)
   (pows @0 (mult @1 { build_real_truncate (type, dconst_third ()); })))
  /* pow(pow(x,y),z) -> pow(x,y*z) iff x is nonnegative.  */
  (simplify
   (pows (pows tree_expr_nonnegative_p@0 @1) @2)
   (pows @0 (mult @1 @2))))

 /* cabs(x+xi) -> fabs(x)*sqrt(2).  */
 (simplify
  (CABS (complex @0 @0))
  (mult (abs @0) { build_real_truncate (type, dconst_sqrt2 ()); }))

 /* hypot(x,x) -> fabs(x)*sqrt(2).  */
 (simplify
  (HYPOT @0 @0)
  (mult (abs @0) { build_real_truncate (type, dconst_sqrt2 ()); }))

 /* cexp(x+yi) -> exp(x)*cexpi(y).  */
 (for cexps (CEXP)
      exps (EXP)
      cexpis (CEXPI)
  (simplify
   (cexps compositional_complex@0)
   (if (targetm.libc_has_function (function_c99_math_complex, TREE_TYPE (@0)))
    (complex
     (mult (exps@1 (realpart @0)) (realpart (cexpis:type@2 (imagpart @0))))
     (mult @1 (imagpart @2)))))))

(if (canonicalize_math_p ())
 /* floor(x) -> trunc(x) if x is nonnegative.  */
 (for floors (FLOOR_ALL)
      truncs (TRUNC_ALL)
  (simplify
   (floors tree_expr_nonnegative_p@0)
   (truncs @0))))

(match double_value_p
 @0
 (if (TYPE_MAIN_VARIANT (TREE_TYPE (@0)) == double_type_node)))
(for froms (BUILT_IN_TRUNCL
	    BUILT_IN_FLOORL
	    BUILT_IN_CEILL
	    BUILT_IN_ROUNDL
	    BUILT_IN_NEARBYINTL
	    BUILT_IN_RINTL)
     tos (BUILT_IN_TRUNC
	  BUILT_IN_FLOOR
	  BUILT_IN_CEIL
	  BUILT_IN_ROUND
	  BUILT_IN_NEARBYINT
	  BUILT_IN_RINT)
 /* truncl(extend(x)) -> extend(trunc(x)), etc., if x is a double.  */
 (if (optimize && canonicalize_math_p ())
  (simplify
   (froms (convert double_value_p@0))
   (convert (tos @0)))))

(match float_value_p
 @0
 (if (TYPE_MAIN_VARIANT (TREE_TYPE (@0)) == float_type_node)))
(for froms (BUILT_IN_TRUNCL BUILT_IN_TRUNC
	    BUILT_IN_FLOORL BUILT_IN_FLOOR
	    BUILT_IN_CEILL BUILT_IN_CEIL
	    BUILT_IN_ROUNDL BUILT_IN_ROUND
	    BUILT_IN_NEARBYINTL BUILT_IN_NEARBYINT
	    BUILT_IN_RINTL BUILT_IN_RINT)
     tos (BUILT_IN_TRUNCF BUILT_IN_TRUNCF
	  BUILT_IN_FLOORF BUILT_IN_FLOORF
	  BUILT_IN_CEILF BUILT_IN_CEILF
	  BUILT_IN_ROUNDF BUILT_IN_ROUNDF
	  BUILT_IN_NEARBYINTF BUILT_IN_NEARBYINTF
	  BUILT_IN_RINTF BUILT_IN_RINTF)
 /* truncl(extend(x)) and trunc(extend(x)) -> extend(truncf(x)), etc.,
    if x is a float.  */
 (if (optimize && canonicalize_math_p ()
      && targetm.libc_has_function (function_c99_misc, NULL_TREE))
  (simplify
   (froms (convert float_value_p@0))
   (convert (tos @0)))))

#if GIMPLE
(match float16_value_p
 @0
 (if (TYPE_MAIN_VARIANT (TREE_TYPE (@0)) == float16_type_node)))
(for froms (BUILT_IN_TRUNCL BUILT_IN_TRUNC BUILT_IN_TRUNCF
	    BUILT_IN_FLOORL BUILT_IN_FLOOR BUILT_IN_FLOORF
	    BUILT_IN_CEILL BUILT_IN_CEIL BUILT_IN_CEILF
	    BUILT_IN_ROUNDEVENL BUILT_IN_ROUNDEVEN BUILT_IN_ROUNDEVENF
	    BUILT_IN_ROUNDL BUILT_IN_ROUND BUILT_IN_ROUNDF
	    BUILT_IN_NEARBYINTL BUILT_IN_NEARBYINT BUILT_IN_NEARBYINTF
	    BUILT_IN_RINTL BUILT_IN_RINT BUILT_IN_RINTF
	    BUILT_IN_SQRTL BUILT_IN_SQRT BUILT_IN_SQRTF)
     tos (IFN_TRUNC IFN_TRUNC IFN_TRUNC
	  IFN_FLOOR IFN_FLOOR IFN_FLOOR
	  IFN_CEIL IFN_CEIL IFN_CEIL
	  IFN_ROUNDEVEN IFN_ROUNDEVEN IFN_ROUNDEVEN
	  IFN_ROUND IFN_ROUND IFN_ROUND
	  IFN_NEARBYINT IFN_NEARBYINT IFN_NEARBYINT
	  IFN_RINT IFN_RINT IFN_RINT
	  IFN_SQRT IFN_SQRT IFN_SQRT)
 /* (_Float16) round ((doube) x) -> __built_in_roundf16 (x), etc.,
    if x is a _Float16.  */
 (simplify
   (convert (froms (convert float16_value_p@0)))
     (if (optimize
	  && types_match (type, TREE_TYPE (@0))
	  && direct_internal_fn_supported_p (as_internal_fn (tos),
					     type, OPTIMIZE_FOR_BOTH))
       (tos @0))))

/* Simplify (trunc)copysign ((extend)x, (extend)y) to copysignf (x, y),
   x,y is float value, similar for _Float16/double.  */
(for copysigns (COPYSIGN_ALL)
 (simplify
  (convert (copysigns (convert@2 @0) (convert @1)))
   (if (optimize
       && !HONOR_SNANS (@2)
       && types_match (type, TREE_TYPE (@0))
       && types_match (type, TREE_TYPE (@1))
       && element_precision (type) < element_precision (TREE_TYPE (@2))
       && direct_internal_fn_supported_p (IFN_COPYSIGN,
					  type, OPTIMIZE_FOR_BOTH))
    (IFN_COPYSIGN @0 @1)))
 /* Simplify (trunc)copysign ((extend)x, CST) to copysign (x, -1.0/1.0).  */
 (simplify
  (convert (copysigns (convert@2 @0) REAL_CST@1))
   (if (optimize
       && !HONOR_SNANS (@2)
       && types_match (type, TREE_TYPE (@0))
       && element_precision (type) < element_precision (TREE_TYPE (@2))
       && direct_internal_fn_supported_p (IFN_COPYSIGN,
					  type, OPTIMIZE_FOR_BOTH))
    (if (REAL_VALUE_NEGATIVE (TREE_REAL_CST (@1)))
     (IFN_COPYSIGN @0 { build_minus_one_cst (type); })
     (IFN_COPYSIGN @0 { build_one_cst (type); })))))

(for froms (BUILT_IN_FMAF BUILT_IN_FMA BUILT_IN_FMAL)
     tos (IFN_FMA IFN_FMA IFN_FMA)
 (simplify
  (convert (froms (convert@3 @0) (convert @1) (convert @2)))
   (if (flag_unsafe_math_optimizations
       && optimize
       && FLOAT_TYPE_P (type)
       && FLOAT_TYPE_P (TREE_TYPE (@3))
       && types_match (type, TREE_TYPE (@0))
       && types_match (type, TREE_TYPE (@1))
       && types_match (type, TREE_TYPE (@2))
       && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@3))
       && direct_internal_fn_supported_p (as_internal_fn (tos),
					  type, OPTIMIZE_FOR_BOTH))
    (tos @0 @1 @2))))

(for maxmin (max min)
 (simplify
  (convert (maxmin (convert@2 @0) (convert @1)))
   (if (optimize
       && FLOAT_TYPE_P (type)
       && FLOAT_TYPE_P (TREE_TYPE (@2))
       && types_match (type, TREE_TYPE (@0))
       && types_match (type, TREE_TYPE (@1))
       && element_precision (type) < element_precision (TREE_TYPE (@2)))
    (maxmin @0 @1))))
#endif

(for froms (XFLOORL XCEILL XROUNDL XRINTL)
     tos (XFLOOR XCEIL XROUND XRINT)
 /* llfloorl(extend(x)) -> llfloor(x), etc., if x is a double.  */
 (if (optimize && canonicalize_math_p ())
  (simplify
   (froms (convert double_value_p@0))
   (tos @0))))

(for froms (XFLOORL XCEILL XROUNDL XRINTL
	    XFLOOR XCEIL XROUND XRINT)
     tos (XFLOORF XCEILF XROUNDF XRINTF)
 /* llfloorl(extend(x)) and llfloor(extend(x)) -> llfloorf(x), etc.,
    if x is a float.  */
 (if (optimize && canonicalize_math_p ())
  (simplify
   (froms (convert float_value_p@0))
   (tos @0))))

(if (canonicalize_math_p ())
 /* xfloor(x) -> fix_trunc(x) if x is nonnegative.  */
 (for floors (IFLOOR LFLOOR LLFLOOR)
  (simplify
   (floors tree_expr_nonnegative_p@0)
   (fix_trunc @0))))

(if (canonicalize_math_p ())
 /* xfloor(x) -> fix_trunc(x), etc., if x is integer valued.  */
 (for fns (IFLOOR LFLOOR LLFLOOR
	   ICEIL LCEIL LLCEIL
	   IROUND LROUND LLROUND)
  (simplify
   (fns integer_valued_real_p@0)
   (fix_trunc @0)))
 (if (!flag_errno_math)
  /* xrint(x) -> fix_trunc(x), etc., if x is integer valued.  */
  (for rints (IRINT LRINT LLRINT)
   (simplify
    (rints integer_valued_real_p@0)
    (fix_trunc @0)))))

(if (canonicalize_math_p ())
 (for ifn (IFLOOR ICEIL IROUND IRINT)
      lfn (LFLOOR LCEIL LROUND LRINT)
      llfn (LLFLOOR LLCEIL LLROUND LLRINT)
  /* Canonicalize iround (x) to lround (x) on ILP32 targets where
     sizeof (int) == sizeof (long).  */
  (if (TYPE_PRECISION (integer_type_node)
       == TYPE_PRECISION (long_integer_type_node))
   (simplify
    (ifn @0)
    (lfn:long_integer_type_node @0)))
  /* Canonicalize llround (x) to lround (x) on LP64 targets where
     sizeof (long long) == sizeof (long).  */
  (if (TYPE_PRECISION (long_long_integer_type_node)
       == TYPE_PRECISION (long_integer_type_node))
   (simplify
    (llfn @0)
    (lfn:long_integer_type_node @0)))))

/* cproj(x) -> x if we're ignoring infinities.  */
(simplify
 (CPROJ @0)
 (if (!HONOR_INFINITIES (type))
   @0))

/* If the real part is inf and the imag part is known to be
   nonnegative, return (inf + 0i).  */
(simplify
 (CPROJ (complex REAL_CST@0 tree_expr_nonnegative_p@1))
 (if (real_isinf (TREE_REAL_CST_PTR (@0)))
  { build_complex_inf (type, false); }))

/* If the imag part is inf, return (inf+I*copysign(0,imag)).  */
(simplify
 (CPROJ (complex @0 REAL_CST@1))
 (if (real_isinf (TREE_REAL_CST_PTR (@1)))
  { build_complex_inf (type, TREE_REAL_CST_PTR (@1)->sign); }))

/* cproj(cproj(x)) -> cproj(x).  */
(simplify
 (CPROJ (CPROJ @0))
 (CPROJ @0))

/* cabs(cproj(x)) -> cabs(x).  */
(simplify
 (CABS (CPROJ @0))
 (CABS @0))

(for pows (POW)
     sqrts (SQRT)
     cbrts (CBRT)
 (simplify
  (pows @0 REAL_CST@1)
  (with {
    const REAL_VALUE_TYPE *value = TREE_REAL_CST_PTR (@1);
    REAL_VALUE_TYPE tmp;
   }
   (switch
    /* pow(x,0) -> 1.  */
    (if (real_equal (value, &dconst0))
     { build_real (type, dconst1); })
    /* pow(x,1) -> x.  */
    (if (real_equal (value, &dconst1))
     @0)
    /* pow(x,-1) -> 1/x.  */
    (if (real_equal (value, &dconstm1)
	 && (!flag_errno_math || !real_maybe_zerop (@0)))
     (rdiv { build_real (type, dconst1); } @0))
    /* pow(x,0.5) -> sqrt(x).  */
    (if (flag_unsafe_math_optimizations
	 && canonicalize_math_p ()
	 && real_equal (value, &dconsthalf))
     (sqrts @0))
    /* pow(x,1/3) -> cbrt(x).  */
    (if (flag_unsafe_math_optimizations
	 && canonicalize_math_p ()
	 && (tmp = real_value_truncate (TYPE_MODE (type), dconst_third ()),
	     real_equal (value, &tmp)))
     (cbrts @0))))))

/* powi(1,x) -> 1.  */
(simplify
 (POWI real_onep@0 @1)
 @0)

(simplify
 (POWI @0 INTEGER_CST@1)
 (switch
  /* powi(x,0) -> 1.  */
  (if (wi::to_wide (@1) == 0)
   { build_real (type, dconst1); })
  /* powi(x,1) -> x.  */
  (if (wi::to_wide (@1) == 1)
   @0)
  /* powi(x,-1) -> 1/x.  */
  (if (wi::to_wide (@1) == -1)
   (rdiv { build_real (type, dconst1); } @0))))

/* Narrowing of arithmetic and logical operations.

   These are conceptually similar to the transformations performed for
   the C/C++ front-ends by shorten_binary_op and shorten_compare.  Long
   term we want to move all that code out of the front-ends into here.  */

/* Convert (outertype)((innertype0)a+(innertype1)b)
   into ((newtype)a+(newtype)b) where newtype
   is the widest mode from all of these.  */
(for op (plus minus mult rdiv)
 (simplify
   (convert (op:s@0 (convert1?@3 @1) (convert2?@4 @2)))
   /* If we have a narrowing conversion of an arithmetic operation where
      both operands are widening conversions from the same type as the outer
      narrowing conversion.  Then convert the innermost operands to a
      suitable unsigned type (to avoid introducing undefined behavior),
      perform the operation and convert the result to the desired type.  */
   (if (INTEGRAL_TYPE_P (type)
	&& op != MULT_EXPR
	&& op != RDIV_EXPR
	/* We check for type compatibility between @0 and @1 below,
	   so there's no need to check that @2/@4 are integral types.  */
	&& INTEGRAL_TYPE_P (TREE_TYPE (@1))
	&& INTEGRAL_TYPE_P (TREE_TYPE (@3))
	/* The precision of the type of each operand must match the
	   precision of the mode of each operand, similarly for the
	   result.  */
	&& type_has_mode_precision_p (TREE_TYPE (@1))
	&& type_has_mode_precision_p (TREE_TYPE (@2))
	&& type_has_mode_precision_p (type)
	/* The inner conversion must be a widening conversion.  */
	&& TYPE_PRECISION (TREE_TYPE (@3)) > TYPE_PRECISION (TREE_TYPE (@1))
	&& types_match (@1, type)
	&& (types_match (@1, @2)
	    /* Or the second operand is const integer or converted const
	       integer from valueize.  */
	    || poly_int_tree_p (@4)))
     (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
       (op @1 (convert @2))
       (with { tree utype = unsigned_type_for (TREE_TYPE (@1)); }
	(convert (op (convert:utype @1)
		     (convert:utype @2)))))
     (if (FLOAT_TYPE_P (type)
	  && DECIMAL_FLOAT_TYPE_P (TREE_TYPE (@0))
	       == DECIMAL_FLOAT_TYPE_P (type))
      (with { tree arg0 = strip_float_extensions (@1);
	      tree arg1 = strip_float_extensions (@2);
	      tree itype = TREE_TYPE (@0);
	      tree ty1 = TREE_TYPE (arg0);
	      tree ty2 = TREE_TYPE (arg1);
	      enum tree_code code = TREE_CODE (itype); }
	(if (FLOAT_TYPE_P (ty1)
	     && FLOAT_TYPE_P (ty2))
	 (with { tree newtype = type;
		 if (TYPE_MODE (ty1) == SDmode
		     || TYPE_MODE (ty2) == SDmode
		     || TYPE_MODE (type) == SDmode)
		   newtype = dfloat32_type_node;
		 if (TYPE_MODE (ty1) == DDmode
		     || TYPE_MODE (ty2) == DDmode
		     || TYPE_MODE (type) == DDmode)
		   newtype = dfloat64_type_node;
		 if (TYPE_MODE (ty1) == TDmode
		     || TYPE_MODE (ty2) == TDmode
		     || TYPE_MODE (type) == TDmode)
		   newtype = dfloat128_type_node; }
	  (if ((newtype == dfloat32_type_node
		|| newtype == dfloat64_type_node
		|| newtype == dfloat128_type_node)
	      && newtype == type
	      && types_match (newtype, type))
	    (op (convert:newtype @1) (convert:newtype @2))
	    (with { if (element_precision (ty1) > element_precision (newtype))
		      newtype = ty1;
		    if (element_precision (ty2) > element_precision (newtype))
		      newtype = ty2; }
	       /* Sometimes this transformation is safe (cannot
		  change results through affecting double rounding
		  cases) and sometimes it is not.  If NEWTYPE is
		  wider than TYPE, e.g. (float)((long double)double
		  + (long double)double) converted to
		  (float)(double + double), the transformation is
		  unsafe regardless of the details of the types
		  involved; double rounding can arise if the result
		  of NEWTYPE arithmetic is a NEWTYPE value half way
		  between two representable TYPE values but the
		  exact value is sufficiently different (in the
		  right direction) for this difference to be
		  visible in ITYPE arithmetic.  If NEWTYPE is the
		  same as TYPE, however, the transformation may be
		  safe depending on the types involved: it is safe
		  if the ITYPE has strictly more than twice as many
		  mantissa bits as TYPE, can represent infinities
		  and NaNs if the TYPE can, and has sufficient
		  exponent range for the product or ratio of two
		  values representable in the TYPE to be within the
		  range of normal values of ITYPE.  */
	      (if (element_precision (newtype) < element_precision (itype)
		   && (!VECTOR_MODE_P (TYPE_MODE (newtype))
		       || target_supports_op_p (newtype, op, optab_default))
		   && (flag_unsafe_math_optimizations
		       || (element_precision (newtype) == element_precision (type)
			   && real_can_shorten_arithmetic (element_mode (itype),
							   element_mode (type))
			   && !excess_precision_type (newtype)))
		   && !types_match (itype, newtype))
		 (convert:type (op (convert:newtype @1)
				   (convert:newtype @2)))
	 )))) )
   ))
)))

/* This is another case of narrowing, specifically when there's an outer
   BIT_AND_EXPR which masks off bits outside the type of the innermost
   operands.   Like the previous case we have to convert the operands
   to unsigned types to avoid introducing undefined behavior for the
   arithmetic operation.  */
(for op (minus plus)
 (simplify
  (bit_and (op:s (convert@2 @0) (convert@3 @1)) INTEGER_CST@4)
  (if (INTEGRAL_TYPE_P (type)
       /* We check for type compatibility between @0 and @1 below,
	  so there's no need to check that @1/@3 are integral types.  */
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (TREE_TYPE (@2))
       /* The precision of the type of each operand must match the
	  precision of the mode of each operand, similarly for the
	  result.  */
       && type_has_mode_precision_p (TREE_TYPE (@0))
       && type_has_mode_precision_p (TREE_TYPE (@1))
       && type_has_mode_precision_p (type)
       /* The inner conversion must be a widening conversion.  */
       && TYPE_PRECISION (TREE_TYPE (@2)) > TYPE_PRECISION (TREE_TYPE (@0))
       && types_match (@0, @1)
       && (tree_int_cst_min_precision (@4, TYPE_SIGN (TREE_TYPE (@0)))
	   <= TYPE_PRECISION (TREE_TYPE (@0)))
       && (wi::to_wide (@4)
	   & wi::mask (TYPE_PRECISION (TREE_TYPE (@0)),
		       true, TYPE_PRECISION (type))) == 0)
   (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
    (with { tree ntype = TREE_TYPE (@0); }
     (convert (bit_and (op @0 @1) (convert:ntype @4))))
    (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
     (convert (bit_and (op (convert:utype @0) (convert:utype @1))
	       (convert:utype @4))))))))

/* Transform (@0 < @1 and @0 < @2) to use min,
   (@0 > @1 and @0 > @2) to use max */
(for logic (bit_and bit_and bit_and bit_and bit_ior bit_ior bit_ior bit_ior)
     op    (lt      le      gt      ge      lt      le      gt      ge     )
     ext   (min     min     max     max     max     max     min     min    )
 (simplify
  (logic (op:cs @0 @1) (op:cs @0 @2))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TREE_CODE (@0) != INTEGER_CST)
   (op @0 (ext @1 @2)))))

/* Max<bool0, bool1> -> bool0 | bool1
   Min<bool0, bool1> -> bool0 & bool1 */
(for op    (max     min)
     logic (bit_ior bit_and)
 (simplify
  (op zero_one_valued_p@0 zero_one_valued_p@1)
  (logic @0 @1)))

/* signbit(x) != 0 ? -x : x -> abs(x)
   signbit(x) == 0 ? -x : x -> -abs(x) */
(for sign (SIGNBIT)
 (for neeq (ne eq)
  (simplify
   (cond (neeq (sign @0) integer_zerop) (negate @0) @0)
    (if (neeq == NE_EXPR)
     (abs @0)
     (negate (abs @0))))))

/* (signbit (x) cmp1 0) cmp (signbit (y) cmp2 0) ? y : -y
   -> copysign (y, +-x).  */
#if GIMPLE
(for cmp (eq ne)
 (for icmp1 (eq ne)
  (for icmp2 (eq ne)
   (for sign (SIGNBIT)
    (simplify
     (cond (cmp:c (icmp1 (sign @0) integer_zerop)
		  (icmp2 (sign @1) integer_zerop))
	   @1 (negate @1))
     (if (SCALAR_FLOAT_TYPE_P (type)
	  && types_match (type, TREE_TYPE (@0))
	  && direct_internal_fn_supported_p (IFN_COPYSIGN, type,
					     OPTIMIZE_FOR_BOTH))
      (if ((cmp == EQ_EXPR) == (icmp1 == icmp2))
       (IFN_COPYSIGN @1 @0)
       (IFN_COPYSIGN @1 (negate @0)))))))))

/* signbit (x) cmp signbit (y) ? y : -y -> copysign (y, +-x).  The nonzero
   value of signbit is unspecified, so the comparison is too when both x
   and y are negative; the fold takes the two values to be equal.  */
(for cmp (eq ne)
 (for sign (SIGNBIT)
  (simplify
   (cond (cmp:c (sign @0) (sign @1)) @1 (negate @1))
   (if (SCALAR_FLOAT_TYPE_P (type)
	&& types_match (type, TREE_TYPE (@0))
	&& direct_internal_fn_supported_p (IFN_COPYSIGN, type,
					   OPTIMIZE_FOR_BOTH))
    (if (cmp == EQ_EXPR)
     (IFN_COPYSIGN @1 @0)
     (IFN_COPYSIGN @1 (negate @0)))))))
#endif

(simplify
 /* signbit(x) -> 0 if x is nonnegative.  */
 (SIGNBIT tree_expr_nonnegative_p@0)
 { integer_zero_node; })

(simplify
 /* signbit(x) -> x<0 if x doesn't have signed zeros.  */
 (SIGNBIT @0)
 (if (!HONOR_SIGNED_ZEROS (@0))
  (convert (lt @0 { build_real (TREE_TYPE (@0), dconst0); }))))

/* Transform comparisons of the form X +- C1 CMP C2 to X CMP C2 -+ C1.  */
(for cmp (eq ne)
 (for op (plus minus)
      rop (minus plus)
  (simplify
   (cmp (op@3 @0 INTEGER_CST@1) INTEGER_CST@2)
   (if (!TREE_OVERFLOW (@1) && !TREE_OVERFLOW (@2)
	&& !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0))
	&& !TYPE_OVERFLOW_TRAPS (TREE_TYPE (@0))
	&& !TYPE_SATURATING (TREE_TYPE (@0)))
    (with { tree res = int_const_binop (rop, @2, @1); }
     (if (TREE_OVERFLOW (res)
	  && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
      { constant_boolean_node (cmp == NE_EXPR, type); }
      (if (single_use (@3) || wi::to_wide (res) == 0)
       (cmp @0 { TREE_OVERFLOW (res)
		 ? drop_tree_overflow (res) : res; }))))))))
(for cmp (lt le gt ge)
     rcmp (gt ge lt le)
 (for op (plus minus)
      rop (minus plus)
  (simplify
   (cmp (op@3 @0 INTEGER_CST@1) INTEGER_CST@2)
   (if (!TREE_OVERFLOW (@1) && !TREE_OVERFLOW (@2)
	&& TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0)))
    (with { tree res = int_const_binop (rop, @2, @1); }
     (if (TREE_OVERFLOW (res))
      {
        bool less = cmp == LE_EXPR || cmp == LT_EXPR;
	/* wi::ges_p (@2, 0) should be sufficient for a signed type.  */
	bool ovf_high = wi::lt_p (wi::to_wide (@1), 0,
				  TYPE_SIGN (TREE_TYPE (@1)))
			!= (op == MINUS_EXPR);
	constant_boolean_node (less == ovf_high, type);
      }
      (if (single_use (@3))
	(cmp @0 { res; }))))
/* For wrapping types, simplify the following cases of X +- C1 CMP C2:

   (a) If CMP is <= and C2 -+ C1 == +INF (1), simplify to X >= -INF -+ C1
       by observing the following:

	X +- C1 <= C2
  ==>  -INF <= X +- C1 <= C2 (add left hand side which holds for any X, C1)
  ==>  -INF -+ C1 <= X <= C2 -+ C1 (add -+C1 to all 3 expressions)
  ==>  -INF -+ C1 <= X <= +INF (due to (1))
  ==>  -INF -+ C1 <= X (eliminate the right hand side since it holds for any X)

    (b) Similarly, if CMP is >= and C2 -+ C1 == -INF (1):

	X +- C1 >= C2
  ==>  +INF >= X +- C1 >= C2 (add left hand side which holds for any X, C1)
  ==>  +INF -+ C1 >= X >= C2 -+ C1 (add -+C1 to all 3 expressions)
  ==>  +INF -+ C1 >= X >= -INF (due to (1))
  ==>  +INF -+ C1 >= X (eliminate the right hand side since it holds for any X)

    (c) The > and < cases are negations of (a) and (b), respectively.  */
   (if (TYPE_OVERFLOW_WRAPS (TREE_TYPE (@0)))
     (with
       {
	wide_int max = wi::max_value (TREE_TYPE (@0));
	wide_int min = wi::min_value (TREE_TYPE (@0));

	wide_int c2 = rop == PLUS_EXPR
			      ? wi::add (wi::to_wide (@2), wi::to_wide (@1))
			      : wi::sub (wi::to_wide (@2), wi::to_wide (@1));
	}
	(if (((cmp == LE_EXPR || cmp == GT_EXPR) && wi::eq_p (c2, max))
	    || ((cmp == LT_EXPR || cmp == GE_EXPR) && wi::eq_p (c2, min)))
	  (with
	   {
	     wide_int c1 = rop == PLUS_EXPR
			   ? wi::add (wi::bit_not (c2), wi::to_wide (@1))
			   : wi::sub (wi::bit_not (c2), wi::to_wide (@1));
	     tree c1_cst = wide_int_to_tree (TREE_TYPE (@0), c1);
	   }
	   (rcmp @0 { c1_cst; })))))))))

/* Invert sign of X in comparisons of the form C1 - X CMP C2.  */

(for cmp (lt le gt ge eq ne)
     rcmp (gt ge lt le eq ne)
  (simplify
   (cmp (minus INTEGER_CST@0 @1) INTEGER_CST@2)
/* For UB-on-overflow types, simply switch sides for X and C2
   to arrive at X RCMP C1 - C2, handling the case when the latter
   expression overflows.  */
   (if (!TREE_OVERFLOW (@0) && !TREE_OVERFLOW (@2)
       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@1)))
     (with { tree res = int_const_binop (MINUS_EXPR, @0, @2); }
      (if (TREE_OVERFLOW (res))
	(switch
	 (if (cmp == NE_EXPR)
	  { constant_boolean_node (true, type); })
	 (if (cmp == EQ_EXPR)
	  { constant_boolean_node (false, type); })
	 {
	   bool less = cmp == LE_EXPR || cmp == LT_EXPR;
	   bool ovf_high = wi::lt_p (wi::to_wide (@0), 0,
				     TYPE_SIGN (TREE_TYPE (@0)));
	   constant_boolean_node (less == ovf_high, type);
	 })
      (rcmp @1 { res; })))
/* For unsigned types, transform like so (using < as example):
	 C1 - X < C2
      ==>  C1 - X = { 0, 1, ..., C2 - 1 }
      ==>  X = { C1 - (C2 - 1), ..., C1 + 1, C1 }
      ==>  X - (C1 - (C2 - 1)) = { 0, 1, ..., C1 - (C1 - (C2 - 1)) }
      ==>  X - (C1 - C2 + 1) = { 0, 1, ..., C2 - 1 }
      ==>  X - (C1 - C2 + 1) < C2.

      Similarly,
	 C1 - X <= C2 ==> X - (C1 - C2) <= C2;
	 C1 - X >= C2 ==> X - (C1 - C2 + 1) >= C2;
	 C1 - X > C2 ==> X - (C1 - C2) > C2.  */
   (if (TYPE_UNSIGNED (TREE_TYPE (@1)))
     (switch
       (if (cmp == EQ_EXPR || cmp == NE_EXPR)
	 (cmp @1 (minus @0 @2)))
       (if (cmp == LE_EXPR || cmp == GT_EXPR)
	 (cmp (plus @1 (minus @2 @0)) @2))
       (if (cmp == LT_EXPR || cmp == GE_EXPR)
	 (cmp (plus @1 (minus @2
		   (plus @0 { build_one_cst (TREE_TYPE (@1)); }))) @2)))
/* For wrapping signed types (-fwrapv), transform like so (using < as example):
	 C1 - X < C2
      ==>  C1 - X = { -INF, -INF + 1, ..., C2 - 1 }
      ==>  X = { C1 - (-INF), C1 - (-INF + 1), ..., C1 - C2 + 1 }
      ==>  X - (C1 + 1) = { - (-INF) - 1, - (-INF) - 2, ..., -C2 }
      ==>  X - (C1 + 1) = { +INF, +INF - 1, ..., -C2 }
      ==>  X - (C1 + 1) > -C2 - 1
      ==>  X - (C1 + 1) > bit_not (C2)

      Similarly,
	 C1 - X <= C2 ==> X - (C1 + 1) >= bit_not (C2);
	 C1 - X >= C2 ==> X - (C1 + 1) <= bit_not (C2);
	 C1 - X > C2 ==> X - (C1 + 1) < bit_not (C2).  */
   (if (!TYPE_UNSIGNED (TREE_TYPE (@1))
	&& TYPE_OVERFLOW_WRAPS (TREE_TYPE (@1)))
     (if (cmp == EQ_EXPR || cmp == NE_EXPR)
	(cmp @1 (minus @0 @2))
     (rcmp (minus @1 (plus @0 { build_one_cst (TREE_TYPE (@1)); }))
	     (bit_not @2))))))))

/* Canonicalizations of BIT_FIELD_REFs.  */

(simplify
 (BIT_FIELD_REF (BIT_FIELD_REF @0 @1 @2) @3 @4)
 (BIT_FIELD_REF @0 @3 { const_binop (PLUS_EXPR, bitsizetype, @2, @4); }))

(simplify
 (BIT_FIELD_REF (view_convert @0) @1 @2)
 (if (! INTEGRAL_TYPE_P (TREE_TYPE (@0))
      || type_has_mode_precision_p (TREE_TYPE (@0)))
  (BIT_FIELD_REF @0 @1 @2)))

(simplify
 (BIT_FIELD_REF @0 @1 integer_zerop)
 (if (tree_int_cst_equal (@1, TYPE_SIZE (TREE_TYPE (@0))))
  (view_convert @0)))

(simplify
 (BIT_FIELD_REF @0 @1 @2)
 (switch
  (if (TREE_CODE (TREE_TYPE (@0)) == COMPLEX_TYPE
       && tree_int_cst_equal (@1, TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0)))))
   (switch
    (if (integer_zerop (@2))
     (view_convert (realpart @0)))
    (if (tree_int_cst_equal (@2, TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0)))))
     (view_convert (imagpart @0)))))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && INTEGRAL_TYPE_P (type)
       /* On GIMPLE this should only apply to register arguments.  */
       && (! GIMPLE || is_gimple_reg (@0))
       /* A bit-field-ref that referenced the full argument can be stripped.  */
       && ((compare_tree_int (@1, TYPE_PRECISION (TREE_TYPE (@0))) == 0
	    && integer_zerop (@2))
	   /* Low-parts can be reduced to integral conversions.
	      ???  The following doesn't work for PDP endian.  */
	   || (BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
	       /* But only do this after vectorization.  */
	       && canonicalize_math_after_vectorization_p ()
	       /* Don't even think about BITS_BIG_ENDIAN.  */
	       && TYPE_PRECISION (TREE_TYPE (@0)) % BITS_PER_UNIT == 0
	       && TYPE_PRECISION (type) % BITS_PER_UNIT == 0
	       && compare_tree_int (@2, (BYTES_BIG_ENDIAN
					 ? (TYPE_PRECISION (TREE_TYPE (@0))
					    - TYPE_PRECISION (type))
					 : 0)) == 0)))
   (convert @0))))

/* Simplify vector extracts.  */

(simplify
 (BIT_FIELD_REF CONSTRUCTOR@0 @1 @2)
 (if (VECTOR_TYPE_P (TREE_TYPE (@0))
      && tree_fits_uhwi_p (TYPE_SIZE (type))
      && ((tree_to_uhwi (TYPE_SIZE (type))
	   == tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0)))))
          || (VECTOR_TYPE_P (type)
	      && (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (type)))
		  == tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0))))))))
  (with
   {
     tree ctor = (TREE_CODE (@0) == SSA_NAME
		  ? gimple_assign_rhs1 (SSA_NAME_DEF_STMT (@0)) : @0);
     tree eltype = TREE_TYPE (TREE_TYPE (ctor));
     unsigned HOST_WIDE_INT width = tree_to_uhwi (TYPE_SIZE (eltype));
     unsigned HOST_WIDE_INT n = tree_to_uhwi (@1);
     unsigned HOST_WIDE_INT idx = tree_to_uhwi (@2);
   }
   (if (n != 0
	&& (idx % width) == 0
	&& (n % width) == 0
	&& known_le ((idx + n) / width,
		     TYPE_VECTOR_SUBPARTS (TREE_TYPE (ctor))))
    (with
     {
       idx = idx / width;
       n = n / width;
       /* Constructor elements can be subvectors.  */
       poly_uint64 k = 1;
       if (CONSTRUCTOR_NELTS (ctor) != 0)
         {
           tree cons_elem = TREE_TYPE (CONSTRUCTOR_ELT (ctor, 0)->value);
	   if (TREE_CODE (cons_elem) == VECTOR_TYPE)
	     k = TYPE_VECTOR_SUBPARTS (cons_elem);
	 }
       unsigned HOST_WIDE_INT elt, count, const_k;
     }
     (switch
      /* We keep an exact subset of the constructor elements.  */
      (if (multiple_p (idx, k, &elt) && multiple_p (n, k, &count))
       (if (CONSTRUCTOR_NELTS (ctor) == 0)
        { build_zero_cst (type); }
	(if (count == 1)
	 (if (elt < CONSTRUCTOR_NELTS (ctor))
	  (view_convert { CONSTRUCTOR_ELT (ctor, elt)->value; })
	  { build_zero_cst (type); })
	 /* We don't want to emit new CTORs unless the old one goes away.
	    ???  Eventually allow this if the CTOR ends up constant or
	    uniform.  */
	 (if (single_use (@0))
	  (with
	    {
	      vec<constructor_elt, va_gc> *vals;
	      vec_alloc (vals, count);
	      bool constant_p = true;
	      tree res;
	      for (unsigned i = 0;
		   i < count && elt + i < CONSTRUCTOR_NELTS (ctor); ++i)
		{
		  tree e = CONSTRUCTOR_ELT (ctor, elt + i)->value;
		  CONSTRUCTOR_APPEND_ELT (vals, NULL_TREE, e);
		  if (!CONSTANT_CLASS_P (e))
		    constant_p = false;
		}
	      tree evtype = (types_match (TREE_TYPE (type),
					  TREE_TYPE (TREE_TYPE (ctor)))
			     ? type
			     : build_vector_type (TREE_TYPE (TREE_TYPE (ctor)),
						  count * k));
	      /* We used to build a CTOR in the non-constant case here
		 but that's not a GIMPLE value.  We'd have to expose this
		 operation somehow so the code generation can properly
		 split it out to a separate stmt.  */
	      res = (constant_p ? build_vector_from_ctor (evtype, vals)
		     : (GIMPLE ? NULL_TREE : build_constructor (evtype, vals)));
	    }
	    (if (res)
	     (view_convert { res; })))))))
      /* The bitfield references a single constructor element.  */
      (if (k.is_constant (&const_k)
	   && idx + n <= (idx / const_k + 1) * const_k)
       (switch
	(if (CONSTRUCTOR_NELTS (ctor) <= idx / const_k)
	 { build_zero_cst (type); })
	(if (n == const_k)
	 (view_convert { CONSTRUCTOR_ELT (ctor, idx / const_k)->value; }))
	(BIT_FIELD_REF { CONSTRUCTOR_ELT (ctor, idx / const_k)->value; }
		       @1 { bitsize_int ((idx % const_k) * width); })))))))))

(simplify
 (BIT_FIELD_REF (vec_perm@0 @1 @2 VECTOR_CST@3) @rsize @rpos)
 (with
  {
    tree elem_type = TREE_TYPE (TREE_TYPE (@0));
    poly_uint64 elem_size = tree_to_poly_uint64 (TYPE_SIZE (elem_type));
    poly_uint64 size = tree_to_poly_uint64 (TYPE_SIZE (type));
    unsigned HOST_WIDE_INT nelts, idx;
    unsigned HOST_WIDE_INT nelts_op = 0;
  }
  /* NELTS is the number of elements in each input, which may differ
     from that of the result; the selector indexes their concatenation.  */
  (if (constant_multiple_p (tree_to_poly_uint64 (@rpos), elem_size, &idx)
       && VECTOR_CST_NELTS (@3).is_constant ()
       && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@1)).is_constant (&nelts)
       && (known_eq (size, elem_size)
	   || (constant_multiple_p (size, elem_size, &nelts_op)
	       && pow2p_hwi (nelts_op))))
   (with
    {
      bool ok = true;
      /* One element.  */
      if (known_eq (size, elem_size))
        idx = TREE_INT_CST_LOW (VECTOR_CST_ELT (@3, idx)) % (2 * nelts);
      else
        {
	  /* Clamp vec_perm_expr index.  */
	  unsigned start
	    = TREE_INT_CST_LOW (vector_cst_elt (@3, idx)) % (2 * nelts);
	  unsigned end
	    = (TREE_INT_CST_LOW (vector_cst_elt (@3, idx + nelts_op - 1))
	       % (2 * nelts));
	  /* Be in the same vector.  */
	  if ((start < nelts) != (end < nelts))
	    ok = false;
	  else
	    for (unsigned HOST_WIDE_INT i = 1; i != nelts_op; i++)
	      {
	        /* Continuous area.  */
		if ((TREE_INT_CST_LOW (vector_cst_elt (@3, idx + i))
		     % (2 * nelts) - 1)
		    != (TREE_INT_CST_LOW (vector_cst_elt (@3, idx + i - 1))
			% (2 * nelts)))
		  {
		    ok = false;
		    break;
		  }
	      }
	  /* Alignment not worse than before.  */
	  if (start % nelts_op)
	    ok = false;
	  idx = start;
	}
    }
    (if (ok)
     (if (idx < nelts)
      (BIT_FIELD_REF @1 @rsize { bitsize_int (idx * elem_size); })
      (BIT_FIELD_REF @2 @rsize { bitsize_int ((idx - nelts) * elem_size); })))))))

/* Simplify a bit extraction from a bit insertion for the cases with
   the inserted element fully covering the extraction or the insertion
   not touching the extraction.  */
(simplify
 (BIT_FIELD_REF (bit_insert @0 @1 @ipos) @rsize @rpos)
 (with
  {
    unsigned HOST_WIDE_INT isize;
    if (INTEGRAL_TYPE_P (TREE_TYPE (@1)))
      isize = TYPE_PRECISION (TREE_TYPE (@1));
    else
      isize = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (@1)));
  }
  (switch
   (if ((!INTEGRAL_TYPE_P (TREE_TYPE (@1))
	 || type_has_mode_precision_p (TREE_TYPE (@1)))
	&& wi::leu_p (wi::to_wide (@ipos), wi::to_wide (@rpos))
	&& wi::leu_p (wi::to_wide (@rpos) + wi::to_wide (@rsize),
		      wi::to_wide (@ipos) + isize))
    (BIT_FIELD_REF @1 @rsize { wide_int_to_tree (bitsizetype,
						 wi::to_wide (@rpos)
						 - wi::to_wide (@ipos)); }))
   (if (wi::eq_p (wi::to_wide (@ipos), wi::to_wide (@rpos))
        && compare_tree_int (@rsize, isize) == 0)
    (convert @1))
   (if (wi::geu_p (wi::to_wide (@ipos),
		   wi::to_wide (@rpos) + wi::to_wide (@rsize))
	|| wi::geu_p (wi::to_wide (@rpos),
		      wi::to_wide (@ipos) + isize))
    (BIT_FIELD_REF @0 @rsize @rpos)))))

/* Simplify vector inserts of other vector extracts to a permute.  */
(simplify
 (bit_insert @0 (BIT_FIELD_REF@2 @1 @rsize @rpos) @ipos)
 (if (VECTOR_TYPE_P (type)
      && (VECTOR_MODE_P (TYPE_MODE (type))
	  || optimize_vectors_before_lowering_p ())
      && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			  TYPE_SIZE (TREE_TYPE (@1)), 0)
      && types_match (TREE_TYPE (TREE_TYPE (@0)), TREE_TYPE (@2))
      && TYPE_VECTOR_SUBPARTS (type).is_constant ()
      && multiple_p (wi::to_poly_offset (@rpos),
		     wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (type)))))
  (with
   {
     unsigned HOST_WIDE_INT elsz
       = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0))));
     poly_uint64 relt = exact_div (tree_to_poly_uint64 (@rpos), elsz);
     poly_uint64 ielt = exact_div (tree_to_poly_uint64 (@ipos), elsz);
     unsigned nunits = TYPE_VECTOR_SUBPARTS (type).to_constant ();
     vec_perm_builder builder;
     builder.new_vector (nunits, nunits, 1);
     for (unsigned i = 0; i < nunits; ++i)
       builder.quick_push (known_eq (ielt, i) ? nunits + relt : i);
     vec_perm_indices sel (builder, 2, nunits);
   }
   (if (!VECTOR_MODE_P (TYPE_MODE (type))
	|| can_vec_perm_const_p (TYPE_MODE (type),
				 TYPE_MODE (type), sel, false))
    (vec_perm @0 (view_convert @1)
     { vec_perm_indices_to_tree (build_vector_type (ssizetype, nunits),
				 sel); })))))

(if (canonicalize_math_after_vectorization_p ())
 (for fmas (FMA)
  (simplify
   (fmas:c (negate @0) @1 @2)
   (IFN_FNMA @0 @1 @2))
  (simplify
   (fmas @0 @1 (negate @2))
   (IFN_FMS @0 @1 @2))
  (simplify
   (fmas:c (negate @0) @1 (negate @2))
   (IFN_FNMS @0 @1 @2))
  (simplify
   (negate (fmas@3 @0 @1 @2))
   (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
	&& !HONOR_SIGNED_ZEROS (type)
	&& single_use (@3))
    (IFN_FNMS @0 @1 @2)))
  (simplify
   (fmas:c (nop_convert (negate @0)) @1 @2)
   (with { tree t = TREE_TYPE (@0); }
    (if ((ANY_INTEGRAL_TYPE_P (type)
	  && (TYPE_UNSIGNED (type)
	      || !TYPE_OVERFLOW_SANITIZED (type))
	  && ANY_INTEGRAL_TYPE_P (t)
	  && (TYPE_UNSIGNED (t)
	      || !TYPE_OVERFLOW_SANITIZED (t))))
   /* Move the negation into FNMA only when signed overflow is
      unobservable for both the outer operation and the inner negate.  */
     (with { tree utype = unsigned_type_for (type); }
      (convert (IFN_FNMA (convert:utype @0) (convert:utype @1)
			 (convert:utype @2))))))))

 (simplify
  (IFN_FMS:c (negate @0) @1 @2)
  (IFN_FNMS @0 @1 @2))
 (simplify
  (IFN_FMS @0 @1 (negate @2))
  (IFN_FMA @0 @1 @2))
 (simplify
  (IFN_FMS:c (negate @0) @1 (negate @2))
  (IFN_FNMA @0 @1 @2))
 (simplify
  (negate (IFN_FMS@3 @0 @1 @2))
   (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
	&& !HONOR_SIGNED_ZEROS (type)
	&& single_use (@3))
    (IFN_FNMA @0 @1 @2)))

 (simplify
  (IFN_FNMA:c (negate @0) @1 @2)
  (IFN_FMA @0 @1 @2))
 (simplify
  (IFN_FNMA @0 @1 (negate @2))
  (IFN_FNMS @0 @1 @2))
 (simplify
  (IFN_FNMA:c (negate @0) @1 (negate @2))
  (IFN_FMS @0 @1 @2))
 (simplify
  (negate (IFN_FNMA@3 @0 @1 @2))
  (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
       && !HONOR_SIGNED_ZEROS (type)
       && single_use (@3))
   (IFN_FMS @0 @1 @2)))

 (simplify
  (IFN_FNMS:c (negate @0) @1 @2)
  (IFN_FMS @0 @1 @2))
 (simplify
  (IFN_FNMS @0 @1 (negate @2))
  (IFN_FNMA @0 @1 @2))
 (simplify
  (IFN_FNMS:c (negate @0) @1 (negate @2))
  (IFN_FMA @0 @1 @2))
 (simplify
  (negate (IFN_FNMS@3 @0 @1 @2))
  (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
       && !HONOR_SIGNED_ZEROS (type)
       && single_use (@3))
   (IFN_FMA @0 @1 @2))))

(if (canonicalize_math_after_vectorization_p ())
 (for fmas (IFN_COND_FMA)
  (simplify
   (fmas:c @3 (negate @0) @1 @2 @4)
   (IFN_COND_FNMA @3 @0 @1 @2 @4))
  (simplify
   (fmas @3 @0 @1 (negate @2) @4)
   (IFN_COND_FMS @3 @0 @1 @2 @4))
  (simplify
   (fmas:c @3 (negate @0) @1 (negate @2) @4)
   (IFN_COND_FNMS @3 @0 @1 @2 @4))
  (simplify
   (negate (fmas@3 @4 @0 @1 @2 @5))
   (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
	&& !HONOR_SIGNED_ZEROS (type)
	&& single_use (@3))
    (IFN_COND_FNMS @4 @0 @1 @2 (negate @5)))))

 (simplify
  (IFN_COND_FMS:c @3 (negate @0) @1 @2 @4)
  (IFN_COND_FNMS @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FMS @3 @0 @1 (negate @2) @4)
  (IFN_COND_FMA @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FMS:c @3 (negate @0) @1 (negate @2) @4)
  (IFN_COND_FNMA @3 @0 @1 @2 @4))
 (simplify
  (negate (IFN_COND_FMS@3 @4 @0 @1 @2 @5))
   (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
	&& !HONOR_SIGNED_ZEROS (type)
	&& single_use (@3))
    (IFN_COND_FNMA @4 @0 @1 @2 (negate @5))))

 (simplify
  (IFN_COND_FNMA:c @3 (negate @0) @1 @2 @4)
  (IFN_COND_FMA @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FNMA @3 @0 @1 (negate @2) @4)
  (IFN_COND_FNMS @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FNMA:c @3 (negate @0) @1 (negate @2) @4)
  (IFN_COND_FMS @3 @0 @1 @2 @4))
 (simplify
  (negate (IFN_COND_FNMA@3 @4 @0 @1 @2 @5))
  (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
       && !HONOR_SIGNED_ZEROS (type)
       && single_use (@3))
   (IFN_COND_FMS @4 @0 @1 @2 (negate @5))))

 (simplify
  (IFN_COND_FNMS:c @3 (negate @0) @1 @2 @4)
  (IFN_COND_FMS @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FNMS @3 @0 @1 (negate @2) @4)
  (IFN_COND_FNMA @3 @0 @1 @2 @4))
 (simplify
  (IFN_COND_FNMS:c @3 (negate @0) @1 (negate @2) @4)
  (IFN_COND_FMA @3 @0 @1 @2 @4))
 (simplify
  (negate (IFN_COND_FNMS@3 @4 @0 @1 @2 @5))
  (if (!HONOR_SIGN_DEPENDENT_ROUNDING (type)
       && !HONOR_SIGNED_ZEROS (type)
       && single_use (@3))
   (IFN_COND_FMA @4 @0 @1 @2 (negate @5)))))

/* min (clz (x), clz (y)) -> clz (x | y) and
   min (ctz (x), ctz (y)) -> ctz (x | y).  */
(for func (CLZ CTZ)
 (simplify
  (min (func:s @0) (func:s @1))
  (if (types_match (@0, @1)
       && (!sanitize_flags_p (SANITIZE_BUILTIN)
	   || (cfun && (cfun->curr_properties & PROP_ssa) != 0)))
   (func (bit_ior @0 @1)))))
(for func (IFN_CLZ IFN_CTZ)
 (simplify
  (min (func:s @0 INTEGER_CST@2) (func:s @1 @2))
  (if (types_match (@0, @1)
       && wi::to_widest (@2) >= element_precision (@0))
   (func (bit_ior @0 @1) @2))))

/* CLZ simplifications.  */
(for clz (CLZ)
 (for op (eq ne)
      cmp (lt ge)
  (simplify
   (op (clz:s@2 @0) INTEGER_CST@1)
   (if (!sanitize_flags_p (SANITIZE_BUILTIN)
	/* For -fsanitize=builtin give ubsan pass a chance
	   to instrument it first.  */
	|| (cfun && (cfun->curr_properties & PROP_ssa) != 0))
    (if (integer_zerop (@1) && single_use (@2))
     /* clz(X) == 0 is (int)X < 0 and clz(X) != 0 is (int)X >= 0.  */
     (with { tree stype = signed_type_for (TREE_TYPE (@0)); }
      (cmp (convert:stype @0) { build_zero_cst (stype); }))
     /* clz(X) == (prec-1) is X == 1 and clz(X) != (prec-1) is X != 1.  */
     (if (wi::to_wide (@1) == TYPE_PRECISION (TREE_TYPE (@0)) - 1)
      (op @0 { build_one_cst (TREE_TYPE (@0)); })))))))
(for op (eq ne)
     cmp (lt ge)
 (simplify
  (op (IFN_CLZ:s@2 @0 @3) INTEGER_CST@1)
  (if (integer_zerop (@1) && single_use (@2))
   /* clz(X) == 0 is (int)X < 0 and clz(X) != 0 is (int)X >= 0.  */
   (with { tree type0 = TREE_TYPE (@0);
	   tree stype = signed_type_for (TREE_TYPE (@0));
	   /* Punt if clz(0) == 0.  */
	   if (integer_zerop (@3))
	     stype = NULL_TREE;
	 }
    (if (stype)
     (cmp (convert:stype @0) { build_zero_cst (stype); })))
   /* clz(X) == (prec-1) is X == 1 and clz(X) != (prec-1) is X != 1.  */
   (with { bool ok = true;
	   tree type0 = TREE_TYPE (@0);
	   /* Punt if clz(0) == prec - 1.  */
	   if (wi::to_widest (@3) == TYPE_PRECISION (type0) - 1)
	     ok = false;
	 }
    (if (ok && wi::to_wide (@1) == (TYPE_PRECISION (type0) - 1))
     (op @0 { build_one_cst (type0); }))))))

/* CTZ simplifications.  */
(for ctz (CTZ)
 /* ctz (-X) => ctz (X).  ctz (abs (X)) => ctz (X).  */
 (for op (negate abs)
  (simplify
   (ctz (nop_convert?@0 (op @1)))
    (with { tree t = TREE_TYPE (@0); }
     (ctz (convert:t @1)))))
 (for op (ge gt le lt)
      cmp (eq eq ne ne)
  (simplify
   /* __builtin_ctz (x) >= C -> (x & ((1 << C) - 1)) == 0.  */
   (op (ctz:s @0) INTEGER_CST@1)
    (with { bool ok = true;
	    HOST_WIDE_INT val = 0;
	    if (sanitize_flags_p (SANITIZE_BUILTIN)
		/* For -fsanitize=builtin give ubsan pass a chance
		   to instrument it first.  */
		&& (!cfun
		    || (cfun->curr_properties & PROP_ssa) == 0))
	      ok = false;
	    else if (!tree_fits_shwi_p (@1))
	      ok = false;
	    else
	      {
		val = tree_to_shwi (@1);
		/* Canonicalize to >= or <.  */
		if (op == GT_EXPR || op == LE_EXPR)
		  {
		    if (val == HOST_WIDE_INT_MAX)
		      ok = false;
		    else
		      val++;
		  }
	      }
	    tree type0 = TREE_TYPE (@0);
	    int prec = TYPE_PRECISION (type0);
	  }
     (if (ok && prec <= MAX_FIXED_MODE_SIZE)
      (if (val <= 0)
       { constant_boolean_node (cmp == EQ_EXPR, type); }
       (if (val >= prec)
	{ constant_boolean_node (cmp != EQ_EXPR, type); }
	(cmp (bit_and @0 { wide_int_to_tree (type0,
					     wi::mask (val, false, prec)); })
	     { build_zero_cst (type0); })))))))
 (for op (eq ne)
  (simplify
   /* __builtin_ctz (x) == C -> (x & ((1 << (C + 1)) - 1)) == (1 << C).  */
   (op (ctz:s @0) INTEGER_CST@1)
    (with { tree type0 = TREE_TYPE (@0);
	    int prec = TYPE_PRECISION (type0);
	    bool ok = true;
	    if (sanitize_flags_p (SANITIZE_BUILTIN)
		/* For -fsanitize=builtin give ubsan pass a chance
		   to instrument it first.  */
		&& (!cfun
		    || (cfun->curr_properties & PROP_ssa) == 0))
	      ok = false;
	  }
     (if (ok && prec <= MAX_FIXED_MODE_SIZE)
      (if (tree_int_cst_sgn (@1) < 0 || wi::to_widest (@1) >= prec)
       { constant_boolean_node (op != EQ_EXPR, type); }
       (op (bit_and @0 { wide_int_to_tree (type0,
					   wi::mask (tree_to_uhwi (@1) + 1,
						     false, prec)); })
	   { wide_int_to_tree (type0,
			       wi::shifted_mask (tree_to_uhwi (@1), 1,
						 false, prec)); })))))))
(for op (ge gt le lt)
     cmp (eq eq ne ne)
 (simplify
  /* __builtin_ctz (x) >= C -> (x & ((1 << C) - 1)) == 0.  */
  (op (IFN_CTZ:s @0 @2) INTEGER_CST@1)
   (with { bool ok = true;
	   HOST_WIDE_INT val = 0;
	   if (!tree_fits_shwi_p (@1))
	     ok = false;
	   else
	     {
	       val = tree_to_shwi (@1);
	       /* Canonicalize to >= or <.  */
	       if (op == GT_EXPR || op == LE_EXPR)
		 {
		   if (val == HOST_WIDE_INT_MAX)
		     ok = false;
		   else
		     val++;
		 }
	     }
	   HOST_WIDE_INT zero_val = tree_to_shwi (@2);
	   tree type0 = TREE_TYPE (@0);
	   int prec = TYPE_PRECISION (type0);
	   if (prec > MAX_FIXED_MODE_SIZE)
	     ok = false;
	  }
     (if (val <= 0)
      (if (ok && zero_val >= val)
       { constant_boolean_node (cmp == EQ_EXPR, type); })
      (if (val >= prec)
       (if (ok && zero_val < val)
	{ constant_boolean_node (cmp != EQ_EXPR, type); })
       (if (ok && zero_val >= val)
	(cmp (bit_and @0 { wide_int_to_tree (type0,
					     wi::mask (val, false, prec)); })
	     { build_zero_cst (type0); })))))))
(for op (eq ne)
 (simplify
  /* __builtin_ctz (x) == C -> (x & ((1 << (C + 1)) - 1)) == (1 << C).  */
  (op (IFN_CTZ:s @0 @2) INTEGER_CST@1)
   (with { HOST_WIDE_INT zero_val = tree_to_shwi (@2);
	   tree type0 = TREE_TYPE (@0);
	   int prec = TYPE_PRECISION (type0);
	 }
    (if (prec <= MAX_FIXED_MODE_SIZE)
     (if (tree_int_cst_sgn (@1) < 0 || wi::to_widest (@1) >= prec)
      (if (zero_val != wi::to_widest (@1))
       { constant_boolean_node (op != EQ_EXPR, type); })
      (if (zero_val < 0 || zero_val >= prec)
       (op (bit_and @0 { wide_int_to_tree (type0,
					   wi::mask (tree_to_uhwi (@1) + 1,
						     false, prec)); })
	   { wide_int_to_tree (type0,
			       wi::shifted_mask (tree_to_uhwi (@1), 1,
						 false, prec)); })))))))

#if GIMPLE
/* ctz(ext(X)) == ctz(X).  Valid just for the UB at zero cases though.  */
(simplify
  (CTZ (convert@1 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_PRECISION (TREE_TYPE (@1)) > TYPE_PRECISION (TREE_TYPE (@0)))
   (with { combined_fn cfn = CFN_LAST;
	   tree type0 = TREE_TYPE (@0);
	   if (BITINT_TYPE_P (type0))
	     {
	       if (TYPE_PRECISION (type0) > MAX_FIXED_MODE_SIZE)
		 cfn = CFN_CTZ;
	       else
		 type0
		   = build_nonstandard_integer_type (TYPE_PRECISION (type0),
						     1);
	     }
	   type0 = unsigned_type_for (type0);
	   if (cfn == CFN_LAST
	       && direct_internal_fn_supported_p (IFN_CTZ, type0,
						  OPTIMIZE_FOR_BOTH))
	     cfn = CFN_CTZ;
	   if (cfn == CFN_LAST
	       && TYPE_PRECISION (TREE_TYPE (@1)) > BITS_PER_WORD
	       && !direct_internal_fn_supported_p (IFN_CTZ,
						   TREE_TYPE (@1),
						   OPTIMIZE_FOR_BOTH))
	     {
	       if (TYPE_PRECISION (type0)
		   == TYPE_PRECISION (unsigned_type_node))
		 cfn = CFN_BUILT_IN_CTZ;
	       else if (TYPE_PRECISION (type0)
			== TYPE_PRECISION (long_long_unsigned_type_node))
		 cfn = CFN_BUILT_IN_CTZLL;
	     }
	   if (sanitize_flags_p (SANITIZE_BUILTIN)
	       /* For -fsanitize=builtin give ubsan pass a chance
		  to instrument it first.  */
	       && (!cfun
		   || (cfun->curr_properties & PROP_ssa) == 0))
	    cfn = CFN_LAST; }
    (if (cfn == CFN_CTZ)
     (IFN_CTZ (convert:type0 @0))
     (if (cfn == CFN_BUILT_IN_CTZ)
      (BUILT_IN_CTZ (convert:type0 @0))
      (if (cfn == CFN_BUILT_IN_CTZLL)
       (BUILT_IN_CTZLL (convert:type0 @0))))))))
#endif

/* POPCOUNT simplifications.  */
/* popcount(X) + popcount(Y) is popcount(X|Y) when X&Y must be zero.  */
(simplify
  (plus (POPCOUNT:s @0) (POPCOUNT:s @1))
  (if (INTEGRAL_TYPE_P (type)
       && (wi::bit_and (widest_int::from (tree_nonzero_bits (@0), UNSIGNED),
			widest_int::from (tree_nonzero_bits (@1), UNSIGNED))
	   == 0))
   (with { tree utype = TREE_TYPE (@0);
	   if (TYPE_PRECISION (utype) < TYPE_PRECISION (TREE_TYPE (@1)))
	     utype = TREE_TYPE (@1); }
    (POPCOUNT (bit_ior (convert:utype @0) (convert:utype @1))))))

/* popcount(X) == 0 is X == 0, and related (in)equalities.  */
(for popcount (POPCOUNT)
  (for cmp (le eq ne gt)
       rep (eq eq ne ne)
    (simplify
      (cmp (popcount @0) integer_zerop)
      (rep @0 { build_zero_cst (TREE_TYPE (@0)); }))))

/* (a != 0) | (popcount (a) == n)  ->  (a != 0) | (n == 0)
   (a == 0) & (popcount (a) != n)  ->  (a == 0) & (n != 0)  */
(for popcount (POPCOUNT)
 (for bop (bit_ior bit_and)
      popcmp (eq ne)
      zerocmp (ne      eq)
  (simplify
   (bop:c (zerocmp@2 @0 integer_zerop)
	  (popcmp:cs (popcount (convert?@3 @0)) @1))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& INTEGRAL_TYPE_P (TREE_TYPE (@3)))
    (bop @2 (popcmp @1 { build_zero_cst (TREE_TYPE (@1)); }))))))

/* popcount(bswap(x)) is popcount(x).  */
(for popcount (POPCOUNT)
  (for bswap (BSWAP BITREVERSE)
    (simplify
      (popcount (convert?@0 (bswap:s@1 @2)))
      (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && INTEGRAL_TYPE_P (TREE_TYPE (@1)))
	(with { tree type0 = TREE_TYPE (@0);
		tree type1 = TREE_TYPE (@1);
		unsigned int prec0 = TYPE_PRECISION (type0);
		unsigned int prec1 = TYPE_PRECISION (type1); }
	  (if (prec0 == prec1 || (prec0 > prec1 && TYPE_UNSIGNED (type1)))
	    (popcount (convert:type0 (convert:type1 @2)))))))))

/* popcount(rotate(X Y)) is popcount(X).  */
(for popcount (POPCOUNT)
  (for rot (lrotate rrotate)
    (simplify
      (popcount (convert?@0 (rot:s@1 @2 @3)))
      (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && INTEGRAL_TYPE_P (TREE_TYPE (@1))
	   && (GIMPLE || !TREE_SIDE_EFFECTS (@3)))
	(with { tree type0 = TREE_TYPE (@0);
		tree type1 = TREE_TYPE (@1);
		unsigned int prec0 = TYPE_PRECISION (type0);
		unsigned int prec1 = TYPE_PRECISION (type1); }
	  (if (prec0 == prec1 || (prec0 > prec1 && TYPE_UNSIGNED (type1)))
	    (popcount (convert:type0 @2))))))))

/* Canonicalize POPCOUNT(x)&1 as PARITY(X).  */
(simplify
  (bit_and (POPCOUNT @0) integer_onep)
  (PARITY @0))

/* popcount(X&Y) + popcount(X|Y) is popcount(x) + popcount(Y).  */
(simplify
  (plus:c (POPCOUNT:s (bit_and:s @0 @1)) (POPCOUNT:s (bit_ior:cs @0 @1)))
  (plus (POPCOUNT:type @0) (POPCOUNT:type @1)))

/* popcount(X) + popcount(Y) - popcount(X&Y) is popcount(X|Y).  */
/* popcount(X) + popcount(Y) - popcount(X|Y) is popcount(X&Y).  */
(for popcount (POPCOUNT)
  (for log1 (bit_and bit_ior)
       log2 (bit_ior bit_and)
    (simplify
      (minus (plus:s (popcount:s @0) (popcount:s @1))
	     (popcount:s (log1:cs @0 @1)))
      (popcount (log2 @0 @1)))
    (simplify
      (plus:c (minus:s (popcount:s @0) (popcount:s (log1:cs @0 @1)))
	      (popcount:s @1))
      (popcount (log2 @0 @1)))))

/* popcount (X & -X) is -X != 0.  */
(simplify
  (POPCOUNT (convert?@2 (bit_and:c @0 (negate@1 @0))))
    (with { tree type0 = TREE_TYPE (@0);
	    tree type2 = TREE_TYPE (@2); }
      (if (INTEGRAL_TYPE_P (type0)
	   && TYPE_PRECISION (type2) <= TYPE_PRECISION (type0))
	/* Use an explicit bool conversion to avoid errors when the
	   POPCOUNT result is assigned to a type (e.g. int) that
	   will generate a "bogus comparison result type" error.  */
	(convert:type
	  (ne:boolean_type_node
	    (convert:type2 @1) { build_zero_cst (type2); })))))

#if GIMPLE
/* popcount(zext(X)) == popcount(X).  */
(simplify
  (POPCOUNT (convert@1 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_UNSIGNED (TREE_TYPE (@0))
       && TYPE_PRECISION (TREE_TYPE (@1)) > TYPE_PRECISION (TREE_TYPE (@0)))
   (with { combined_fn cfn = CFN_LAST;
	   tree type0 = TREE_TYPE (@0);
	   if (BITINT_TYPE_P (type0))
	     {
	       if (TYPE_PRECISION (type0) > MAX_FIXED_MODE_SIZE)
		 cfn = CFN_POPCOUNT;
	       else
		 type0
		   = build_nonstandard_integer_type (TYPE_PRECISION (type0),
						     1);
	     }
	   if (cfn == CFN_LAST
	       && direct_internal_fn_supported_p (IFN_POPCOUNT, type0,
						  OPTIMIZE_FOR_BOTH))
	     cfn = CFN_POPCOUNT;
	   if (cfn == CFN_LAST
	       && TYPE_PRECISION (TREE_TYPE (@1)) > BITS_PER_WORD
	       && !direct_internal_fn_supported_p (IFN_POPCOUNT,
						   TREE_TYPE (@1),
						   OPTIMIZE_FOR_BOTH))
	     {
	       if (TYPE_PRECISION (type0)
		   == TYPE_PRECISION (unsigned_type_node))
		 cfn = CFN_BUILT_IN_POPCOUNT;
	       else if (TYPE_PRECISION (type0)
			== TYPE_PRECISION (long_long_unsigned_type_node))
		 cfn = CFN_BUILT_IN_POPCOUNTLL;
	     } }
    (if (cfn == CFN_POPCOUNT)
     (IFN_POPCOUNT (convert:type0 @0))
     (if (cfn == CFN_BUILT_IN_POPCOUNT)
      (BUILT_IN_POPCOUNT (convert:type0 @0))
      (if (cfn == CFN_BUILT_IN_POPCOUNTLL)
       (BUILT_IN_POPCOUNTLL (convert:type0 @0))))))))

/* popcount (X ^ (X - 1)) is the same as ffs(x) when x is nonzero, and
   using ctz+1 will generate better code.  */
(simplify
 (POPCOUNT (bit_xor:c tree_expr_nonzero_p@0 (plus @0 integer_minus_onep)))
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && direct_internal_fn_supported_p (IFN_CTZ, TREE_TYPE (@0),
					 OPTIMIZE_FOR_SPEED))
  (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
   (plus (CTZ:type (convert:utype @0)) { build_one_cst (type); }))))
#endif

/* PARITY simplifications.  */
/* parity(~X) is parity(X) for even precision and parity(X) ^ 1 otherwise.  */
(simplify
  (PARITY (bit_not @0))
  (if ((element_precision (TREE_TYPE (@0)) & 1) == 0)
   (PARITY @0)
   (bit_xor (PARITY:type @0) { build_one_cst (type); })))

/* parity(bswap(x)) is parity(x).  */
(for parity (PARITY)
  (for bswap (BSWAP BITREVERSE)
    (simplify
      (parity (convert?@0 (bswap:s@1 @2)))
      (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && INTEGRAL_TYPE_P (TREE_TYPE (@1))
	   && TYPE_PRECISION (TREE_TYPE (@0))
	      >= TYPE_PRECISION (TREE_TYPE (@1)))
	(with { tree type0 = TREE_TYPE (@0);
		tree type1 = TREE_TYPE (@1); }
	  (parity (convert:type0 (convert:type1 @2))))))))

/* parity(rotate(X Y)) is parity(X).  */
(for parity (PARITY)
  (for rot (lrotate rrotate)
    (simplify
      (parity (convert?@0 (rot:s@1 @2 @3)))
      (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	   && INTEGRAL_TYPE_P (TREE_TYPE (@1))
	   && (GIMPLE || !TREE_SIDE_EFFECTS (@3))
	   && TYPE_PRECISION (TREE_TYPE (@0))
	      >= TYPE_PRECISION (TREE_TYPE (@1)))
	(with { tree type0 = TREE_TYPE (@0); }
	  (parity (convert:type0 @2)))))))

/* parity(X)^parity(Y) is parity(X^Y).  */
(simplify
  (bit_xor (PARITY:s @0) (PARITY:s @1))
  (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1)))
   (PARITY (bit_xor @0 @1))
   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
	&& INTEGRAL_TYPE_P (TREE_TYPE (@1)))
    (with { tree utype = TREE_TYPE (@0);
	    if (TYPE_PRECISION (utype) < TYPE_PRECISION (TREE_TYPE (@1)))
	      utype = TREE_TYPE (@1); }
     (PARITY (bit_xor (convert:utype @0) (convert:utype @1)))))))

#if GIMPLE
/* parity(zext(X)) == parity(X).  */
/* parity(sext(X)) == parity(X) if the difference in precision is even.  */
(simplify
  (PARITY (convert@1 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_PRECISION (TREE_TYPE (@1)) > TYPE_PRECISION (TREE_TYPE (@0))
       && (TYPE_UNSIGNED (TREE_TYPE (@0))
	   || ((TYPE_PRECISION (TREE_TYPE (@1))
		- TYPE_PRECISION (TREE_TYPE (@0))) & 1) == 0))
   (with { combined_fn cfn = CFN_LAST;
	   tree type0 = TREE_TYPE (@0);
	   if (BITINT_TYPE_P (type0))
	     {
	       if (TYPE_PRECISION (type0) > MAX_FIXED_MODE_SIZE)
		 cfn = CFN_PARITY;
	       else
		 type0
		   = build_nonstandard_integer_type (TYPE_PRECISION (type0),
						     1);
	     }
	   type0 = unsigned_type_for (type0);
	   if (cfn == CFN_LAST
	       && direct_internal_fn_supported_p (IFN_PARITY, type0,
						  OPTIMIZE_FOR_BOTH))
	     cfn = CFN_PARITY;
	   if (cfn == CFN_LAST
	       && TYPE_PRECISION (TREE_TYPE (@1)) > BITS_PER_WORD
	       && !direct_internal_fn_supported_p (IFN_PARITY,
						   TREE_TYPE (@1),
						   OPTIMIZE_FOR_BOTH))
	     {
	       if (TYPE_PRECISION (type0)
		   == TYPE_PRECISION (unsigned_type_node))
		 cfn = CFN_BUILT_IN_PARITY;
	       else if (TYPE_PRECISION (type0)
			== TYPE_PRECISION (long_long_unsigned_type_node))
		 cfn = CFN_BUILT_IN_PARITYLL;
	     } }
    (if (cfn == CFN_PARITY)
     (IFN_PARITY (convert:type0 @0))
     (if (cfn == CFN_BUILT_IN_PARITY)
      (BUILT_IN_PARITY (convert:type0 @0))
      (if (cfn == CFN_BUILT_IN_PARITYLL)
       (BUILT_IN_PARITYLL (convert:type0 @0))))))))
#endif

/* a != 0 ? FUN(a) : 0 -> Fun(a) for some builtin functions. */
(for func (POPCOUNT BSWAP FFS PARITY)
 (simplify
  (cond (ne @0 integer_zerop@1) (func@3 (convert? @0)) integer_zerop@2)
  @3))

/* a != 0 ? FUN(a) : CST -> Fun(a) for some CLRSB builtins
   where CST is precision-1. */
(for func (CLRSB)
 (simplify
  (cond (ne @0 integer_zerop@1) (func@4 (convert?@3 @0)) INTEGER_CST@2)
  (if (wi::to_widest (@2) == TYPE_PRECISION (TREE_TYPE (@3)) - 1)
   @4)))

#if GIMPLE
/* a != 0 ? CLZ(a) : CST -> .CLZ(a) where CST is the result of the internal function for 0. */
(for func (CLZ)
 (simplify
  (cond (ne @0 integer_zerop@1) (func (convert?@3 @0)) INTEGER_CST@2)
  (with { int val;
	  internal_fn ifn = IFN_LAST;
	  if (BITINT_TYPE_P (TREE_TYPE (@3)))
	    {
	      if (tree_fits_shwi_p (@2))
		{
		  HOST_WIDE_INT valw = tree_to_shwi (@2);
		  if ((int) valw == valw)
		    {
		      val = valw;
		      ifn = IFN_CLZ;
		    }
		}
	    }
	  else if (direct_internal_fn_supported_p (IFN_CLZ, TREE_TYPE (@3),
						   OPTIMIZE_FOR_BOTH)
		   && CLZ_DEFINED_VALUE_AT_ZERO
			(SCALAR_INT_TYPE_MODE (TREE_TYPE (@3)), val) == 2)
	    ifn = IFN_CLZ;
	}
   (if (ifn == IFN_CLZ && wi::to_widest (@2) == val)
    (IFN_CLZ @3 @2)))))
(for cnd (cond vec_cond)
 (simplify
  (cnd (ne @0 integer_zerop@1) (IFN_CLZ (convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4)
  (with { int val;
	  internal_fn ifn = IFN_LAST;
	  if (BITINT_TYPE_P (TREE_TYPE (@3)))
	    ifn = IFN_CLZ;
	  else if (direct_internal_fn_supported_p (IFN_CLZ, TREE_TYPE (@3),
						   OPTIMIZE_FOR_BOTH))
	    ifn = IFN_CLZ;
	}
   (if (ifn == IFN_CLZ && tree_int_cst_equal (@2, uniform_integer_cst_p (@4)))
    (IFN_CLZ @3 @2)))))

/* a != 0 ? CTZ(a) : CST -> .CTZ(a) where CST is the result of the internal function for 0. */
(for func (CTZ)
 (simplify
  (cond (ne @0 integer_zerop@1) (func (convert?@3 @0)) INTEGER_CST@2)
  (with { int val;
	  internal_fn ifn = IFN_LAST;
	  if (BITINT_TYPE_P (TREE_TYPE (@3)))
	    {
	      if (tree_fits_shwi_p (@2))
		{
		  HOST_WIDE_INT valw = tree_to_shwi (@2);
		  if ((int) valw == valw)
		    {
		      val = valw;
		      ifn = IFN_CTZ;
		    }
		}
	    }
	  else if (direct_internal_fn_supported_p (IFN_CTZ, TREE_TYPE (@3),
						   OPTIMIZE_FOR_BOTH)
		   && CTZ_DEFINED_VALUE_AT_ZERO
			(SCALAR_INT_TYPE_MODE (TREE_TYPE (@3)), val) == 2)
	    ifn = IFN_CTZ;
	}
   (if (ifn == IFN_CTZ && wi::to_widest (@2) == val)
    (IFN_CTZ @3 @2)))))
(for cnd (cond vec_cond)
 (simplify
  (cnd (ne @0 integer_zerop@1) (IFN_CTZ (convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4)
  (with { int val;
	  internal_fn ifn = IFN_LAST;
	  if (BITINT_TYPE_P (TREE_TYPE (@3)))
	    ifn = IFN_CTZ;
	  else if (direct_internal_fn_supported_p (IFN_CTZ, TREE_TYPE (@3),
						   OPTIMIZE_FOR_BOTH))
	    ifn = IFN_CTZ;
	}
   (if (ifn == IFN_CTZ && tree_int_cst_equal (@2, uniform_integer_cst_p (@4)))
    (IFN_CTZ @3 @2)))))
#endif

/* Common POPCOUNT/PARITY simplifications.  */
/* popcount(X&C1) is (X>>C2)&1 when C1 == 1<<C2.  Same for parity(X&C1).  */
(for pfun (POPCOUNT PARITY)
  (simplify
    (pfun @0)
    (if (INTEGRAL_TYPE_P (type))
     (with { wide_int nz = tree_nonzero_bits (@0); }
       (switch
	 (if (nz == 1)
	   (convert @0))
	 (if (wi::popcount (nz) == 1)
	   (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
	     (convert (rshift:utype (convert:utype @0)
				    { build_int_cst (integer_type_node,
						     wi::ctz (nz)); })))))))))

#if GIMPLE
/* 64- and 32-bits branchless implementations of popcount are detected:

   int popcount64c (uint64_t x)
   {
     x -= (x >> 1) & 0x5555555555555555ULL;
     x = (x & 0x3333333333333333ULL) + ((x >> 2) & 0x3333333333333333ULL);
     x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
     return (x * 0x0101010101010101ULL) >> 56;
   }

   int popcount32c (uint32_t x)
   {
     x -= (x >> 1) & 0x55555555;
     x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
     x = (x + (x >> 4)) & 0x0f0f0f0f;
     return (x * 0x01010101) >> 24;
   }  */
(simplify
 (rshift
  (mult
   (bit_and
    (plus:c
     (rshift @8 INTEGER_CST@5)
      (plus:c@8
       (bit_and @6 INTEGER_CST@7)
	(bit_and
	 (rshift
	  (minus@6 @0
	   (bit_and (rshift @0 INTEGER_CST@4) INTEGER_CST@11))
	  INTEGER_CST@10)
	 INTEGER_CST@9)))
    INTEGER_CST@3)
   INTEGER_CST@2)
  INTEGER_CST@1)
  /* Check constants and optab.  */
  (with { unsigned prec = TYPE_PRECISION (type);
	  int shift = (64 - prec) & 63;
	  unsigned HOST_WIDE_INT c1
	    = HOST_WIDE_INT_UC (0x0101010101010101) >> shift;
	  unsigned HOST_WIDE_INT c2
	    = HOST_WIDE_INT_UC (0x0F0F0F0F0F0F0F0F) >> shift;
	  unsigned HOST_WIDE_INT c3
	    = HOST_WIDE_INT_UC (0x3333333333333333) >> shift;
	  unsigned HOST_WIDE_INT c4
	    = HOST_WIDE_INT_UC (0x5555555555555555) >> shift;
   }
   (if (prec >= 16
	&& prec <= 64
	&& pow2p_hwi (prec)
	&& TYPE_UNSIGNED (type)
	&& integer_onep (@4)
	&& wi::to_widest (@10) == 2
	&& wi::to_widest (@5) == 4
	&& wi::to_widest (@1) == prec - 8
	&& tree_to_uhwi (@2) == c1
	&& tree_to_uhwi (@3) == c2
	&& tree_to_uhwi (@9) == c3
	&& tree_to_uhwi (@7) == c3
	&& tree_to_uhwi (@11) == c4)
    (if (direct_internal_fn_supported_p (IFN_POPCOUNT, type,
					 OPTIMIZE_FOR_BOTH))
     (convert (IFN_POPCOUNT:type @0))
     /* Try to do popcount in two halves.  PREC must be at least
	five bits for this to work without extension before adding.  */
     (with {
       tree half_type = NULL_TREE;
       opt_machine_mode m = mode_for_size ((prec + 1) / 2, MODE_INT, 1);
       int half_prec = 8;
       if (m.exists ()
	   && m.require () != TYPE_MODE (type))
	 {
	   half_prec = GET_MODE_PRECISION (as_a <scalar_int_mode> (m));
	   half_type = build_nonstandard_integer_type (half_prec, 1);
	 }
       gcc_assert (half_prec > 2);
      }
      (if (half_type != NULL_TREE
	   && direct_internal_fn_supported_p (IFN_POPCOUNT, half_type,
					      OPTIMIZE_FOR_BOTH))
       (convert (plus
	 (IFN_POPCOUNT:half_type (convert @0))
	 (IFN_POPCOUNT:half_type (convert (rshift @0
	    { build_int_cst (integer_type_node, half_prec); } )))))))))))

/* __builtin_ffs needs to deal on many targets with the possible zero
   argument.  If we know the argument is always non-zero, __builtin_ctz + 1
   should lead to better code.  */
(simplify
 (FFS tree_expr_nonzero_p@0)
 (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
      && direct_internal_fn_supported_p (IFN_CTZ, TREE_TYPE (@0),
					 OPTIMIZE_FOR_SPEED))
  (with { tree utype = unsigned_type_for (TREE_TYPE (@0)); }
   (plus (CTZ:type (convert:utype @0)) { build_one_cst (type); }))))

/* To recognize the popcnt pattern for 32-bit from Hacker's Delight
   int Gia_WordCountOnes32c ( uint32_t uword)
   {
   uword = (uword & 0x55555555) + ((uword>>1) & 0x55555555);
   uword = (uword & 0x33333333) + ((uword>>2) & 0x33333333);
   uword = (uword & 0x0f0f0f0f) + ((uword>>4) & 0x0f0f0f0f);
   uword = (uword & 0x00ff00ff) + ((uword>>8) & 0x00ff00ff);
   return (uword & 0x0000ffff) + (uword>>16);
   or
   return (uword & 0x0000ffff) + ((uword>>16) & 0x0000ffff);
   }
*/

  (simplify
     (plus:c
       (bit_and @step4 INTEGER_CST@9)
       (rshift
	 (plus:c@step4
	   (bit_and @step3 INTEGER_CST@7)
	(bit_and
	   (rshift
	      (plus:c@step3
		  (bit_and @step2 INTEGER_CST@5)
		  (bit_and
		     (rshift
			(plus:c@step2
			   (bit_and @step1 INTEGER_CST@3)
			   (bit_and
			      (rshift
				  (plus:c@step1
				      (bit_and @0 INTEGER_CST@1)
				      (bit_and (rshift @0 INTEGER_CST@2) @1))
				  INTEGER_CST@4)
			      INTEGER_CST@3))
			INTEGER_CST@6)
		     INTEGER_CST@5))
	      INTEGER_CST@8)
	   INTEGER_CST@7))
	 INTEGER_CST@10))
   (with {
    unsigned prec = TYPE_PRECISION (type);
    }
    (if (prec == 32
	 && TYPE_UNSIGNED (type)
	 && compare_tree_int (@2, 1) == 0
	 && compare_tree_int (@4, 2) == 0
	 && compare_tree_int (@6, 4) == 0
	 && compare_tree_int (@8, 8) == 0
	 && compare_tree_int (@10, 16) == 0
	 && compare_tree_int (@1, 0x55555555) == 0
	 && compare_tree_int (@3, 0x33333333) == 0
	 && compare_tree_int (@5, 0x0F0F0F0F) == 0
	 && compare_tree_int (@7, 0x00FF00FF) == 0
	 && compare_tree_int (@9, 0x0000FFFF) == 0)
       (if (direct_internal_fn_supported_p (IFN_POPCOUNT, type,
					    OPTIMIZE_FOR_BOTH))
	(convert (IFN_POPCOUNT:type @0))))))

/* To recognize the popcnt pattern for 32-bit from Hacker's Delight
   int pop (unsigned x)
   {
    x = x - ((x >> 1) & 0x55555555);
    x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
    x = (x + (x >> 4)) & 0x0F0F0F0F;
    x = x + (x >> 8);
    x = x + (x >> 16);
    return x & 0x0000003F
   }
*/
(simplify
 (bit_and
   (plus
    (rshift @step4 INTEGER_CST@10)
     (plus:c@step4
	(rshift @step3 INTEGER_CST@8)
	   (bit_and@step3
	     (plus
		(rshift @step2 INTEGER_CST@6)
		   (plus:c@step2
		      (bit_and @step1 INTEGER_CST@3)
		      (bit_and
			  (rshift
			    (minus@step1
			     @0
			     (bit_and (rshift @0 INTEGER_CST@2) INTEGER_CST@1))
			    INTEGER_CST@4)
			 INTEGER_CST@3)))
	     INTEGER_CST@5)))
   INTEGER_CST@7)
 (with {
   unsigned prec = TYPE_PRECISION (type);
   }
   (if (prec == 32
	&& TYPE_UNSIGNED (type)
	&& compare_tree_int (@2, 1) == 0
	&& compare_tree_int (@4, 2) == 0
	&& compare_tree_int (@6, 4) == 0
	&& compare_tree_int (@8, 8) == 0
	&& compare_tree_int (@10, 16) == 0
	&& compare_tree_int (@1, 0x55555555) == 0
	&& compare_tree_int (@3, 0x33333333) == 0
	&& compare_tree_int (@5, 0x0F0F0F0F) == 0
	&& compare_tree_int (@7, 0x0000003F) == 0)
     (if (direct_internal_fn_supported_p (IFN_POPCOUNT, type,
					  OPTIMIZE_FOR_BOTH))
	(convert (IFN_POPCOUNT:type @0))))))

/* To recognize the popcnt pattern for 32-bit from Hacker's Delight
   int pop (unsigned x)
   {
     x = x - ((x >> 1) & 0x55555555);
     x = x - 3*((x >> 2) & 0x33333333)
     x = (x + (x >> 4)) & 0x0F0F0F0F;
     x = x + (x >> 8);
     x = x + (x >> 16);
     return x & 0x0000003F;
   }
*/
(simplify
 (bit_and
   (plus
    (rshift @step4 INTEGER_CST@10)
     (plus:c@step4
	(rshift @step3 INTEGER_CST@8)
	   (bit_and@step3
	      (plus
		(rshift @step2 INTEGER_CST@6)
		(minus@step2
		   @step1
		   (mult:c
		      (bit_and
			(rshift
			  (minus@step1
			     @0
			     (bit_and (rshift @0 INTEGER_CST@2) INTEGER_CST@1))
			  INTEGER_CST@4)
			INTEGER_CST@3)
		      INTEGER_CST@11)))
	      INTEGER_CST@5)))
   INTEGER_CST@7)
 (with {
   unsigned prec = TYPE_PRECISION (type);
   }
   (if (prec == 32
	&& TYPE_UNSIGNED (type)
	&& compare_tree_int (@2, 1) == 0
	&& compare_tree_int (@4, 2) == 0
	&& compare_tree_int (@6, 4) == 0
	&& compare_tree_int (@8, 8) == 0
	&& compare_tree_int (@10, 16) == 0
	&& compare_tree_int (@11, 3) == 0
	&& compare_tree_int (@1, 0x55555555) == 0
	&& compare_tree_int (@3, 0x33333333) == 0
	&& compare_tree_int (@5, 0x0F0F0F0F) == 0
	&& compare_tree_int (@7, 0x0000003F) == 0)
     (if (direct_internal_fn_supported_p (IFN_POPCOUNT, type,
					  OPTIMIZE_FOR_BOTH))
	(convert (IFN_POPCOUNT:type @0))))))
#endif

(for ffs (FFS)
 /* __builtin_ffs (X) == 0 -> X == 0.
    __builtin_ffs (X) == 6 -> (X & 63) == 32.  */
 (for cmp (eq ne)
  (simplify
   (cmp (ffs@2 @0) INTEGER_CST@1)
    (with { int prec = TYPE_PRECISION (TREE_TYPE (@0)); }
     (switch
      (if (integer_zerop (@1))
       (cmp @0 { build_zero_cst (TREE_TYPE (@0)); }))
      (if (tree_int_cst_sgn (@1) < 0 || wi::to_widest (@1) > prec)
       { constant_boolean_node (cmp == NE_EXPR, type); })
      (if (single_use (@2))
       (cmp (bit_and @0 { wide_int_to_tree (TREE_TYPE (@0),
					    wi::mask (tree_to_uhwi (@1),
						      false, prec)); })
	    { wide_int_to_tree (TREE_TYPE (@0),
				wi::shifted_mask (tree_to_uhwi (@1) - 1, 1,
						  false, prec)); }))))))

 /* __builtin_ffs (X) > 6 -> X != 0 && (X & 63) == 0.  */
 (for cmp (gt le)
      cmp2 (ne eq)
      cmp3 (eq ne)
      bit_op (bit_and bit_ior)
  (simplify
   (cmp (ffs@2 @0) INTEGER_CST@1)
    (with { int prec = TYPE_PRECISION (TREE_TYPE (@0)); }
     (switch
      (if (integer_zerop (@1))
       (cmp2 @0 { build_zero_cst (TREE_TYPE (@0)); }))
      (if (tree_int_cst_sgn (@1) < 0)
       { constant_boolean_node (cmp == GT_EXPR, type); })
      (if (wi::to_widest (@1) >= prec)
       { constant_boolean_node (cmp != GT_EXPR, type); })
      (if (wi::to_widest (@1) == prec - 1)
       (cmp3 @0 { wide_int_to_tree (TREE_TYPE (@0),
				    wi::shifted_mask (prec - 1, 1,
						      false, prec)); }))
      (if (single_use (@2))
       (bit_op (cmp2 @0 { build_zero_cst (TREE_TYPE (@0)); })
	       (cmp3 (bit_and @0
			      { wide_int_to_tree (TREE_TYPE (@0),
						  wi::mask (tree_to_uhwi (@1),
						  false, prec)); })
		     { build_zero_cst (TREE_TYPE (@0)); }))))))))

#if GIMPLE
/* ffs(ext(X)) == ffs(X).  */
(simplify
  (FFS (convert@1 @0))
  (if (INTEGRAL_TYPE_P (TREE_TYPE (@1))
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && TYPE_PRECISION (TREE_TYPE (@1)) > TYPE_PRECISION (TREE_TYPE (@0)))
   (with { combined_fn cfn = CFN_LAST;
	   tree type0 = TREE_TYPE (@0);
	   if (BITINT_TYPE_P (type0))
	     {
	       if (TYPE_PRECISION (type0) > MAX_FIXED_MODE_SIZE)
		 cfn = CFN_FFS;
	       else
		 type0
		   = build_nonstandard_integer_type (TYPE_PRECISION (type0),
						     0);
	     }
	   type0 = signed_type_for (type0);
	   if (cfn == CFN_LAST
	       && direct_internal_fn_supported_p (IFN_FFS, type0,
						  OPTIMIZE_FOR_BOTH))
	     cfn = CFN_FFS;
	   if (cfn == CFN_LAST
	       && TYPE_PRECISION (TREE_TYPE (@1)) > BITS_PER_WORD
	       && !direct_internal_fn_supported_p (IFN_FFS,
						   TREE_TYPE (@1),
						   OPTIMIZE_FOR_BOTH))
	     {
	       if (TYPE_PRECISION (type0)
		   == TYPE_PRECISION (integer_type_node))
		 cfn = CFN_BUILT_IN_FFS;
	       else if (TYPE_PRECISION (type0)
			== TYPE_PRECISION (long_long_integer_type_node))
		 cfn = CFN_BUILT_IN_FFSLL;
	     } }
    (if (cfn == CFN_FFS)
     (IFN_FFS (convert:type0 @0))
     (if (cfn == CFN_BUILT_IN_FFS)
      (BUILT_IN_FFS (convert:type0 @0))
      (if (cfn == CFN_BUILT_IN_FFSLL)
       (BUILT_IN_FFSLL (convert:type0 @0))))))))
#endif

#if GIMPLE

/* Simplify:
     a = op a1
     r = cond ? a : b
     --> r = .COND_FN (cond, a, b)
and,
    a = op a1
    r = cond ? b : a
    --> r = .COND_FN (~cond, b, a).  */

(for uncond_op (UNCOND_UNARY)
     cond_op (COND_UNARY)
 (simplify
  (vec_cond @0 (view_convert? (uncond_op@3 @1)) @2)
   (with { tree op_type = TREE_TYPE (@3); }
    (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
        && is_truth_type_for (op_type, TREE_TYPE (@0)))
     (cond_op @0 (view_convert @1) @2))))
 (simplify
  (vec_cond @0 @1 (view_convert? (uncond_op@3 @2)))
   (with { tree op_type = TREE_TYPE (@3); }
    (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
        && is_truth_type_for (op_type, TREE_TYPE (@0)))
     (cond_op (bit_not @0) (view_convert @2) @1)))))

(for uncond_op (UNCOND_UNARY)
     cond_op (COND_LEN_UNARY)
 (simplify
  (IFN_VCOND_MASK_LEN @0 (view_convert? (uncond_op@3 @1)) @2 @4 @5)
   (with { tree op_type = TREE_TYPE (@3); }
    (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
        && is_truth_type_for (op_type, TREE_TYPE (@0)))
     (cond_op @0 (view_convert @1) @2 @4 @5))))
 (simplify
  (IFN_VCOND_MASK_LEN @0 @1 (view_convert? (uncond_op@3 @2)) @4 @5)
   (with { tree op_type = TREE_TYPE (@3); }
    (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
        && is_truth_type_for (op_type, TREE_TYPE (@0)))
     (cond_op (bit_not @0) (view_convert @2) @1 @4 @5)))))

/* `(a ? -1 : 0) ^ b` can be converted into a conditional not.  */
(simplify
 (bit_xor:c (vec_cond @0 uniform_integer_cst_p@1 uniform_integer_cst_p@2) @3)
 (if (canonicalize_math_after_vectorization_p ()
      && vectorized_internal_fn_supported_p (IFN_COND_NOT, type)
      && is_truth_type_for (type, TREE_TYPE (@0)))
 (if (integer_all_onesp (@1) && integer_zerop (@2))
  (IFN_COND_NOT @0 @3 @3))
  (if (integer_all_onesp (@2) && integer_zerop (@1))
   (IFN_COND_NOT (bit_not @0) @3 @3))))

/* Simplify:

     a = a1 op a2
     r = c ? a : b;

   to:

     r = c ? a1 op a2 : b;

   if the target can do it in one go.  This makes the operation conditional
   on c, so could drop potentially-trapping arithmetic, but that's a valid
   simplification if the result of the operation isn't needed.

   Avoid speculatively generating a stand-alone vector comparison
   on targets that might not support them.  Any target implementing
   conditional internal functions must support the same comparisons
   inside and outside a VEC_COND_EXPR.  */

(for uncond_op (UNCOND_BINARY)
     cond_op (COND_BINARY)
 (simplify
  (vec_cond @0 (view_convert? (uncond_op@4 @1 @2)) @3)
  (with { tree op_type = TREE_TYPE (@4); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@4))
    (view_convert (cond_op @0 @1 @2 (view_convert:op_type @3))))))
 (simplify
  (vec_cond @0 @1 (view_convert? (uncond_op@4 @2 @3)))
  (with { tree op_type = TREE_TYPE (@4); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@4))
    (view_convert (cond_op (bit_not @0) @2 @3 (view_convert:op_type @1)))))))

(for uncond_op (UNCOND_BINARY)
     cond_op (COND_LEN_BINARY)
 (simplify
  (IFN_VCOND_MASK_LEN @0 (view_convert? (uncond_op@4 @1 @2)) @3 @5 @6)
  (with { tree op_type = TREE_TYPE (@4); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@4))
    (view_convert (cond_op @0 @1 @2 (view_convert:op_type @3) @5 @6)))))
 (simplify
  (IFN_VCOND_MASK_LEN @0 @1 (view_convert? (uncond_op@4 @2 @3)) @5 @6)
  (with { tree op_type = TREE_TYPE (@4); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@4))
    (view_convert (cond_op (bit_not @0) @2 @3 (view_convert:op_type @1) @5 @6))))))

/* Same for ternary operations.  */
(for uncond_op (UNCOND_TERNARY)
     cond_op (COND_TERNARY)
 (simplify
  (vec_cond @0 (view_convert? (uncond_op@5 @1 @2 @3)) @4)
  (with { tree op_type = TREE_TYPE (@5); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@5))
    (view_convert (cond_op @0 @1 @2 @3 (view_convert:op_type @4))))))
 (simplify
  (vec_cond @0 @1 (view_convert? (uncond_op@5 @2 @3 @4)))
  (with { tree op_type = TREE_TYPE (@5); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@5))
    (view_convert (cond_op (bit_not @0) @2 @3 @4
		  (view_convert:op_type @1)))))))

(for uncond_op (UNCOND_TERNARY)
     cond_op (COND_LEN_TERNARY)
 (simplify
  (IFN_VCOND_MASK_LEN @0 (view_convert? (uncond_op@5 @1 @2 @3)) @4 @6 @7)
  (with { tree op_type = TREE_TYPE (@5); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@5))
    (view_convert (cond_op @0 @1 @2 @3 (view_convert:op_type @4) @6 @7)))))
 (simplify
  (IFN_VCOND_MASK_LEN @0 @1 (view_convert? (uncond_op@5 @2 @3 @4 @6 @7)))
  (with { tree op_type = TREE_TYPE (@5); }
   (if (vectorized_internal_fn_supported_p (as_internal_fn (cond_op), op_type)
	&& is_truth_type_for (op_type, TREE_TYPE (@0))
	&& single_use (@5))
    (view_convert (cond_op (bit_not @0) @2 @3 @4 (view_convert:op_type @1) @6 @7))))))
#endif

/* Detect cases in which a VEC_COND_EXPR effectively replaces the
   "else" value of an IFN_COND_*.  */
(for cond_op (COND_BINARY)
 (simplify
  (vec_cond @0 (view_convert? (cond_op @0 @1 @2 @3)) @4)
  (with { tree op_type = TREE_TYPE (@3); }
   (if (element_precision (type) == element_precision (op_type))
    (view_convert (cond_op @0 @1 @2 (view_convert:op_type @4))))))
 (simplify
  (vec_cond @0 @1 (view_convert? (cond_op @2 @3 @4 @5)))
  (with { tree op_type = TREE_TYPE (@5); }
   (if (inverse_conditions_p (@0, @2)
        && element_precision (type) == element_precision (op_type))
    (view_convert (cond_op @2 @3 @4 (view_convert:op_type @1)))))))

/* Same for ternary operations.  */
(for cond_op (COND_TERNARY)
 (simplify
  (vec_cond @0 (view_convert? (cond_op @0 @1 @2 @3 @4)) @5)
  (with { tree op_type = TREE_TYPE (@4); }
   (if (element_precision (type) == element_precision (op_type))
    (view_convert (cond_op @0 @1 @2 @3 (view_convert:op_type @5))))))
 (simplify
  (vec_cond @0 @1 (view_convert? (cond_op @2 @3 @4 @5 @6)))
  (with { tree op_type = TREE_TYPE (@6); }
   (if (inverse_conditions_p (@0, @2)
        && element_precision (type) == element_precision (op_type))
    (view_convert (cond_op @2 @3 @4 @5 (view_convert:op_type @1)))))))

/* Detect cases in which a VEC_COND_EXPR effectively replaces the
   "else" value of an IFN_COND_LEN_*.  */
(for cond_len_op (COND_LEN_BINARY)
 (simplify
  (vec_cond @0 (view_convert? (cond_len_op @0 @1 @2 @3 @4 @5)) @6)
  (with { tree op_type = TREE_TYPE (@3); }
   (if (element_precision (type) == element_precision (op_type))
    (view_convert (cond_len_op @0 @1 @2 (view_convert:op_type @6) @4 @5)))))
 (simplify
  (vec_cond @0 @1 (view_convert? (cond_len_op @2 @3 @4 @5 @6 @7)))
  (with { tree op_type = TREE_TYPE (@5); }
   (if (inverse_conditions_p (@0, @2)
        && element_precision (type) == element_precision (op_type))
    (view_convert (cond_len_op @2 @3 @4 (view_convert:op_type @1) @6 @7))))))

/* Same for ternary operations.  */
(for cond_len_op (COND_LEN_TERNARY)
 (simplify
  (vec_cond @0 (view_convert? (cond_len_op @0 @1 @2 @3 @4 @5 @6)) @7)
  (with { tree op_type = TREE_TYPE (@4); }
   (if (element_precision (type) == element_precision (op_type))
    (view_convert (cond_len_op @0 @1 @2 @3 (view_convert:op_type @7) @5 @6)))))
 (simplify
  (vec_cond @0 @1 (view_convert? (cond_len_op @2 @3 @4 @5 @6 @7 @8)))
  (with { tree op_type = TREE_TYPE (@6); }
   (if (inverse_conditions_p (@0, @2)
        && element_precision (type) == element_precision (op_type))
    (view_convert (cond_len_op @2 @3 @4 @5 (view_convert:op_type @1) @7 @8))))))

/* Detect simplication for a conditional reduction where

   a = mask1 ? b : 0
   c = mask2 ? d + a : d

   is turned into

   c = mask1 && mask2 ? d + b : d.  */
(simplify
  (IFN_COND_ADD @0 @1 (vec_cond @2 @3 zerop@4) @1)
   (if (ANY_INTEGRAL_TYPE_P (type)
	|| (FLOAT_TYPE_P (type)
	    && fold_real_zero_addition_p (type, NULL_TREE, @4, 0)))
   (IFN_COND_ADD (bit_and @0 @2) @1 @3 @1)))

/* Detect simplication for a conditional length reduction where

   a = mask ? b : 0
   c = i < len + bias ? d + a : d

   is turned into

   c = mask && i < len + bias ? d + b : d.  */
(simplify
  (IFN_COND_LEN_ADD integer_truep @0 (vec_cond @1 @2 zerop@5) @0 @3 @4)
   (if (ANY_INTEGRAL_TYPE_P (type)
	|| (FLOAT_TYPE_P (type)
	    && fold_real_zero_addition_p (type, NULL_TREE, @5, 0)))
    (IFN_COND_LEN_ADD @1 @0 @2 @0 @3 @4)))

/* Detect simplification for vector condition folding where

  c = mask1 ? (masked_op mask2 a b els) : els

  into

  c = masked_op (mask1 & mask2) a b els

  where the operation can be partially applied to one operand. */

(for cond_op (COND_BINARY)
 (simplify
  (vec_cond @0
   (cond_op:s @1 @2 @3 @4) @4)
  (cond_op (bit_and @1 @0) @2 @3 @4)))

/* And same for ternary expressions.  */

(for cond_op (COND_TERNARY)
 (simplify
  (vec_cond @0
   (cond_op:s @1 @2 @3 @4 @5) @5)
  (cond_op (bit_and @1 @0) @2 @3 @4 @5)))

/* For pointers @0 and @2 and nonnegative constant offset @1, look for
   expressions like:

   A: (@0 + @1 < @2) | (@2 + @1 < @0)
   B: (@0 + @1 <= @2) | (@2 + @1 <= @0)

   If pointers are known not to wrap, B checks whether @1 bytes starting
   at @0 and @2 do not overlap, while A tests the same thing for @1 + 1
   bytes.  A is more efficiently tested as:

   A: (sizetype) (@0 + @1 - @2) > @1 * 2

   The equivalent expression for B is given by replacing @1 with @1 - 1:

   B: (sizetype) (@0 + (@1 - 1) - @2) > (@1 - 1) * 2

   @0 and @2 can be swapped in both expressions without changing the result.

   The folds rely on sizetype's being unsigned (which is always true)
   and on its being the same width as the pointer (which we have to check).

   The fold replaces two pointer_plus expressions, two comparisons and
   an IOR with a pointer_plus, a pointer_diff, and a comparison, so in
   the best case it's a saving of two operations.  The A fold retains one
   of the original pointer_pluses, so is a win even if both pointer_pluses
   are used elsewhere.  The B fold is a wash if both pointer_pluses are
   used elsewhere, since all we end up doing is replacing a comparison with
   a pointer_plus.  We do still apply the fold under those circumstances
   though, in case applying it to other conditions eventually makes one of the
   pointer_pluses dead.  */
(for ior (truth_orif truth_or bit_ior)
 (for cmp (le lt)
  (simplify
   (ior (cmp:cs (pointer_plus@3 @0 INTEGER_CST@1) @2)
	(cmp:cs (pointer_plus@4 @2 @1) @0))
   (if (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
	&& TYPE_OVERFLOW_WRAPS (sizetype)
	&& TYPE_PRECISION (TREE_TYPE (@0)) == TYPE_PRECISION (sizetype))
    /* Calculate the rhs constant.  */
    (with { offset_int off = wi::to_offset (@1) - (cmp == LE_EXPR ? 1 : 0);
	    offset_int rhs = off * 2; }
     /* Always fails for negative values.  */
     (if (wi::min_precision (rhs, UNSIGNED) <= TYPE_PRECISION (sizetype))
      /* Since the order of @0 and @2 doesn't matter, let tree_swap_operands_p
	 pick a canonical order.  This increases the chances of using the
	 same pointer_plus in multiple checks.  */
      (with { bool swap_p = tree_swap_operands_p (@0, @2);
	      tree rhs_tree = wide_int_to_tree (sizetype, rhs); }
       (if (cmp == LT_EXPR)
	(gt (convert:sizetype
	     (pointer_diff:ssizetype { swap_p ? @4 : @3; }
				     { swap_p ? @0 : @2; }))
	    { rhs_tree; })
	(gt (convert:sizetype
	     (pointer_diff:ssizetype
	      (pointer_plus { swap_p ? @2 : @0; }
			    { wide_int_to_tree (sizetype, off); })
	      { swap_p ? @0 : @2; }))
	    { rhs_tree; })))))))))

/* Fold REDUC (@0 & @1) -> @0[I] & @1[I] if element I is the only nonzero
   element of @1.  */
(for reduc (IFN_REDUC_PLUS IFN_REDUC_IOR IFN_REDUC_XOR)
 (simplify (reduc (view_convert? (bit_and @0 VECTOR_CST@1)))
  (with { int i = single_nonzero_element (@1); }
   (if (i >= 0)
    (with { tree elt = vector_cst_elt (@1, i);
	    tree elt_type = TREE_TYPE (elt);
	    unsigned int elt_bits = tree_to_uhwi (TYPE_SIZE (elt_type));
	    tree size = bitsize_int (elt_bits);
	    tree pos = bitsize_int (elt_bits * i); }
     (if (compare_tree_int (TYPE_SIZE (type), elt_bits) == 0)
      (view_convert
       (bit_and:elt_type
        (BIT_FIELD_REF:elt_type @0 { size; } { pos; })
        { elt; }))))))))

/* Fold reduction of a single nonzero element constructor.  */
(for reduc (IFN_REDUC_PLUS IFN_REDUC_IOR IFN_REDUC_XOR)
  (simplify (reduc (CONSTRUCTOR@0))
    (with { tree ctor = (TREE_CODE (@0) == SSA_NAME
			 ? gimple_assign_rhs1 (SSA_NAME_DEF_STMT (@0)) : @0);
	    tree elt = ctor_single_nonzero_element (ctor); }
      (if (elt
	   && types_match (type, TREE_TYPE (elt))
	   && !HONOR_SNANS (type)
	   && !HONOR_SIGNED_ZEROS (type))
	{ elt; }))))

/* Fold REDUC (@0 op VECTOR_CST) as REDUC (@0) op REDUC (VECTOR_CST)
   when the constant reduction simplifies to a scalar operand.  */
(for reduc (IFN_REDUC_PLUS IFN_REDUC_MAX IFN_REDUC_MIN IFN_REDUC_FMAX
	    IFN_REDUC_FMIN IFN_REDUC_AND IFN_REDUC_IOR IFN_REDUC_XOR)
     op (plus max min IFN_FMAX IFN_FMIN bit_and bit_ior bit_xor)
  (simplify (reduc (op @0 VECTOR_CST@1))
    (op (reduc:type @0) (reduc!:type @1))))

/* Fold REDUC_PLUS (@0) +- REDUC_PLUS (@1) as REDUC_PLUS (@0 +- @1).  This
   trades one lane-crossing reduction for one elementwise operation.  The
   elementwise operation is done in the unsigned type because the reduction
   itself wraps and the signed vector type has undefined overflow.  */
(for op (plus minus)
 (simplify
  (op (IFN_REDUC_PLUS:s @0) (IFN_REDUC_PLUS:s @1))
  (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1))
       && ((ANY_INTEGRAL_TYPE_P (type)
	    && !TYPE_OVERFLOW_TRAPS (type)
	    && !TYPE_OVERFLOW_SANITIZED (type))
	   || (FLOAT_TYPE_P (type)
	       && flag_associative_math
	       && !HONOR_SIGNED_ZEROS (type)
	       && !flag_trapping_math))
       && target_supports_op_p (TREE_TYPE (@0), op, optab_vector))
   (with { tree vtype = (ANY_INTEGRAL_TYPE_P (type)
			 ? unsigned_type_for (TREE_TYPE (@0))
			 : TREE_TYPE (@0));
	   tree stype = TREE_TYPE (vtype); }
    (convert (IFN_REDUC_PLUS:stype
	      (op (view_convert:vtype @0) (view_convert:vtype @1))))))))

/* Fold REDUC (@0) op REDUC (@1) as REDUC (@0 op @1) for the reductions whose
   scalar operation is associative and commutative with no flag needed.  This
   removes one lane-crossing reduction.  */
(for reduc (IFN_REDUC_MAX IFN_REDUC_MIN IFN_REDUC_AND IFN_REDUC_IOR
	    IFN_REDUC_XOR)
     op (max min bit_and bit_ior bit_xor)
 (simplify
  (op (reduc:s @0) (reduc:s @1))
  (if (types_match (TREE_TYPE (@0), TREE_TYPE (@1))
       && target_supports_op_p (TREE_TYPE (@0), op, optab_vector))
   (reduc (op @0 @1)))))

/* Simplify .REDUC_IOR (@0) ==/!= 0 to @0 ==/!= 0.  */
(for cmp (eq ne)
 (simplify
  (cmp (IFN_REDUC_IOR @0) integer_zerop)
  (if (VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (@0)))
       && can_compare_p (cmp == EQ_EXPR ? EQ : NE, TYPE_MODE (TREE_TYPE (@0)),
			 ccp_jump))
   (cmp @0 { build_zero_cst (TREE_TYPE (@0)); }))))

/* Simplify vector floating point operations of alternating sub/add pairs
   into using an fneg of a wider element type followed by a normal add.
   under IEEE 754 the fneg of the wider type will negate every even entry
   and when doing an add we get a sub of the even and add of every odd
   elements.  */
(for plusminus (plus minus)
     minusplus (minus plus)
 (simplify
  (vec_perm (plusminus @0 @1) (minusplus @2 @3) VECTOR_CST@4)
   (if (!VECTOR_INTEGER_TYPE_P (type)
	&& !FLOAT_WORDS_BIG_ENDIAN
        /* plus is commutative, while minus is not, so :c can't be used.
	   Do equality comparisons by hand and at the end pick the operands
	   from the minus.  */
	&& (operand_equal_p (@0, @2, 0)
	    ? operand_equal_p (@1, @3, 0)
	    : operand_equal_p (@0, @3, 0) && operand_equal_p (@1, @2, 0)))
   (with
    {
      /* Build a vector of integers from the tree mask.  */
      vec_perm_builder builder;
    }
    (if (tree_to_vec_perm_builder (&builder, @4))
     (with
      {
	/* Create a vec_perm_indices for the integer vector.  */
	poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (type);
	vec_perm_indices sel (builder, 2, nelts);
	machine_mode vec_mode = TYPE_MODE (type);
	machine_mode wide_mode;
	scalar_mode wide_elt_mode;
	poly_uint64 wide_nunits;
	scalar_mode inner_mode = GET_MODE_INNER (vec_mode);
      }
      (if (VECTOR_MODE_P (vec_mode)
	   && sel.series_p (0, 2, 0, 2)
	   && sel.series_p (1, 2, nelts + 1, 2)
	   && GET_MODE_2XWIDER_MODE (inner_mode).exists (&wide_elt_mode)
	   && multiple_p (GET_MODE_NUNITS (vec_mode), 2, &wide_nunits)
	   && related_vector_mode (vec_mode, wide_elt_mode,
				   wide_nunits).exists (&wide_mode))
       (with
	{
	  tree stype
	    = lang_hooks.types.type_for_mode (GET_MODE_INNER (wide_mode),
					      TYPE_UNSIGNED (type));
	  tree ntype = build_vector_type_for_mode (stype, wide_mode);

	  /* The format has to be a non-extended ieee format.  */
	  const struct real_format *fmt_old = FLOAT_MODE_FORMAT (vec_mode);
	  const struct real_format *fmt_new = FLOAT_MODE_FORMAT (wide_mode);
	}
	(if (TYPE_MODE (stype) != BLKmode
	     && VECTOR_TYPE_P (ntype)
	     && fmt_old != NULL
	     && fmt_new != NULL)
	 (with
	  {
	    /* If the target doesn't support v1xx vectors, try using
	       scalar mode xx instead.  */
	    if (known_eq (GET_MODE_NUNITS (wide_mode), 1)
		&& !target_supports_op_p (ntype, NEGATE_EXPR, optab_vector))
	      ntype = stype;
	  }
	  (if (fmt_new->signbit_rw
	       == fmt_old->signbit_rw + GET_MODE_UNIT_BITSIZE (vec_mode)
	       && fmt_new->signbit_rw == fmt_new->signbit_ro
	       && targetm.can_change_mode_class (TYPE_MODE (ntype),
						 TYPE_MODE (type), ALL_REGS)
	       && ((optimize_vectors_before_lowering_p ()
		    && VECTOR_TYPE_P (ntype))
		   || target_supports_op_p (ntype, NEGATE_EXPR, optab_vector)))
	   (if (plusminus == PLUS_EXPR)
	    (plus (view_convert:type (negate (view_convert:ntype @3))) @2)
	    (minus @0 (view_convert:type
			(negate (view_convert:ntype @1))))))))))))))))

(simplify
 (vec_perm @0 @1 VECTOR_CST@2)
 (with
  {
    tree op0 = @0, op1 = @1, op2 = @2;
    machine_mode result_mode = TYPE_MODE (type);
    machine_mode op_mode = TYPE_MODE (TREE_TYPE (op0));

    /* Build a vector of integers from the tree mask.  */
    vec_perm_builder builder;
  }
  (if (tree_to_vec_perm_builder (&builder, op2))
   (with
    {
      /* Create a vec_perm_indices for the integer vector.  */
      poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (type);
      poly_uint64 nelts_in = TYPE_VECTOR_SUBPARTS (TREE_TYPE (op0));
      bool single_arg = (op0 == op1);
      vec_perm_indices sel (builder, single_arg ? 1 : 2, nelts_in);
    }
    (if (known_eq (nelts, nelts_in) && sel.series_p (0, 1, 0, 1))
     (view_convert @0)
     (if (known_eq (nelts, nelts_in) && sel.series_p (0, 1, nelts_in, 1))
      (view_convert @1)
      (with
       {
         if (!single_arg)
	   {
	     if (sel.all_from_input_p (0))
	       op1 = op0;
	     else if (sel.all_from_input_p (1))
	       {
	         op0 = op1;
	         sel.rotate_inputs (1);
	       }
	     else if (known_ge (poly_uint64 (sel[0]), nelts_in))
	       {
	         std::swap (op0, op1);
	         sel.rotate_inputs (1);
	       }
	   }
	 gassign *def;
	 tree cop0 = op0, cop1 = op1;
	 if (TREE_CODE (op0) == SSA_NAME
	     && (def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op0)))
	     && gimple_assign_rhs_code (def) == CONSTRUCTOR)
	   cop0 = gimple_assign_rhs1 (def);
	 if (TREE_CODE (op1) == SSA_NAME
	     && (def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op1)))
	     && gimple_assign_rhs_code (def) == CONSTRUCTOR)
	   cop1 = gimple_assign_rhs1 (def);
	 tree t;
       }
       (if ((TREE_CODE (cop0) == VECTOR_CST
	     || TREE_CODE (cop0) == CONSTRUCTOR)
	    && (TREE_CODE (cop1) == VECTOR_CST
		|| TREE_CODE (cop1) == CONSTRUCTOR)
	    && (t = fold_vec_perm (type, cop0, cop1, sel)))
	{ t; }
	(with
	 {
	   bool changed = (op0 == op1 && !single_arg);
	   tree ins = NULL_TREE;
	   unsigned at = 0;

	   /* See if the permutation is performing a single element
	      insert from a CONSTRUCTOR or constant and use a BIT_INSERT_EXPR
	      in that case.  But only if the vector mode is supported,
	      otherwise this is invalid GIMPLE.  */
	   if (op_mode != BLKmode
	       && known_eq (nelts, nelts_in)
	       && (TREE_CODE (cop0) == VECTOR_CST
		   || TREE_CODE (cop0) == CONSTRUCTOR
		   || TREE_CODE (cop1) == VECTOR_CST
		   || TREE_CODE (cop1) == CONSTRUCTOR))
	     {
	       bool insert_first_p = sel.series_p (1, 1, nelts + 1, 1);
	       if (insert_first_p)
	         {
		   /* After canonicalizing the first elt to come from the
		      first vector we only can insert the first elt from
		      the first vector.  */
		   at = 0;
		   if ((ins = fold_read_from_vector (cop0, sel[0])))
		     op0 = op1;
		 }
	       /* The above can fail for two-element vectors which always
		  appear to insert the first element, so try inserting
		  into the second lane as well.  For more than two
		  elements that's wasted time.  */
	       if (!insert_first_p || (!ins && maybe_eq (nelts, 2u)))
		 {
		   unsigned int encoded_nelts = sel.encoding ().encoded_nelts ();
		   for (at = 0; at < encoded_nelts; ++at)
		     if (maybe_ne (sel[at], at))
		       break;
		   if (at < encoded_nelts
		       && (known_eq (at + 1, nelts)
			   || sel.series_p (at + 1, 1, at + 1, 1)))
		     {
		       if (known_lt (poly_uint64 (sel[at]), nelts))
			 ins = fold_read_from_vector (cop0, sel[at]);
		       else
			 ins = fold_read_from_vector (cop1, sel[at] - nelts);
		     }
		 }
	     }

	   /* Generate a canonical form of the selector.  */
	   if (!ins && sel.encoding () != builder)
	     {
	       /* Some targets are deficient and fail to expand a single
		  argument permutation while still allowing an equivalent
		  2-argument version.  */
	       tree oldop2 = op2;
	       if (sel.ninputs () == 2
		   || can_vec_perm_const_p (result_mode, op_mode, sel, false))
		 op2 = vec_perm_indices_to_tree (TREE_TYPE (op2), sel);
	       else
		 {
		   vec_perm_indices sel2 (builder, 2, nelts);
		   if (can_vec_perm_const_p (result_mode, op_mode, sel2, false))
		     op2 = vec_perm_indices_to_tree (TREE_TYPE (op2), sel2);
		   else
		     /* Not directly supported with either encoding,
			so use the preferred form.  */
		     op2 = vec_perm_indices_to_tree (TREE_TYPE (op2), sel);
		 }
	       if (!operand_equal_p (op2, oldop2, 0))
		 changed = true;
	     }
	 }
	 (if (ins)
	  (bit_insert { op0; } { ins; }
	   { bitsize_int (at * vector_element_bits (type)); })
	  (if (changed)
	   (vec_perm { op0; } { op1; } { op2; }))))))))))))

/* VEC_PERM_EXPR (v, v, mask) -> v where v contains same element.  */

(match vec_same_elem_p
 (vec_duplicate @0))

(match vec_same_elem_p
 CONSTRUCTOR@0
 (if (TREE_CODE (@0) == SSA_NAME
      && uniform_vector_p (gimple_assign_rhs1 (SSA_NAME_DEF_STMT (@0))))))

(match vec_same_elem_p
 @0
 (if (uniform_vector_p (@0))))


(simplify
 (vec_perm vec_same_elem_p@0 @0 @1)
 (if (types_match (type, TREE_TYPE (@0)))
  @0
  (with
   {
     tree elem = uniform_vector_p (@0);
   }
   (if (elem)
    { build_vector_from_val (type, elem); }))))

/* A reduction of a duplicate vector with an idempotent operation is the
   duplicated element.  */
(for reduc (IFN_REDUC_MAX IFN_REDUC_MIN IFN_REDUC_AND IFN_REDUC_IOR)
 (simplify
  (reduc (vec_duplicate @0))
  (if (!tree_expr_maybe_signaling_nan_p (@0))
   @0)))

/* Fold an XOR reduction of an even-length duplicate vector to zero.  */
(simplify
 (IFN_REDUC_XOR (vec_duplicate@1 @0))
 (if (multiple_p (TYPE_VECTOR_SUBPARTS (TREE_TYPE (@1)), 2))
  { build_zero_cst (type); }))

/* Push VEC_PERM earlier if that may help FMA perception (PR101895).  */
(simplify
 (plus:c (vec_perm:s (mult:c@0 @1 vec_same_elem_p@2) @0 @3) @4)
 (if (TREE_CODE (@0) == SSA_NAME && num_imm_uses (@0) == 2)
  (plus (mult (vec_perm @1 @1 @3) @2) @4)))
(simplify
 (minus (vec_perm:s (mult:c@0 @1 vec_same_elem_p@2) @0 @3) @4)
 (if (TREE_CODE (@0) == SSA_NAME && num_imm_uses (@0) == 2)
  (minus (mult (vec_perm @1 @1 @3) @2) @4)))


/* Merge
     inner = VEC_PERM_EXPR <op2, op3, sel4>;
     vc = VIEW_CONVERT_EXPR <inner>;  (optional)
     result = VEC_PERM_EXPR <vc, vc, sel5>;
   to
     result = VIEW_CONVERT_EXPR <VEC_PERM_EXPR <op2, op3, new_sel>>;
   The VIEW_CONVERT_EXPR may change the element type but not the element
   count.  op2 and op3 have the same number of elements, but inner may have
   a different number than op2 and op3, and result a different number
   than inner, so index each selector by the element count of its own
   inputs.  */

(simplify
 (vec_perm (view_convert?@0 (vec_perm@1 @2 @3 VECTOR_CST@4)) @0 VECTOR_CST@5)
 (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()
      && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant ()
      && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@2)).is_constant ())
  (with
   {
     machine_mode result_mode = TYPE_MODE (type);
     machine_mode vc_mode = TYPE_MODE (TREE_TYPE (@0));
     machine_mode inner_mode = TYPE_MODE (TREE_TYPE (@1));
     machine_mode op2_mode = TYPE_MODE (TREE_TYPE (@2));
     int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
     vec_perm_indices sel4, sel5;
   }
   (if (tree_to_vec_perm_indices (&sel4, @2, @3, @4)
	&& tree_to_vec_perm_indices (&sel5, @0, @0, @5)
	&& TYPE_SIZE (TREE_TYPE (TREE_TYPE (@0)))
	   == TYPE_SIZE (TREE_TYPE (TREE_TYPE (@1))))
    (with
     {
       vec_perm_builder new_builder (nelts, nelts, 1);
       for (int i = 0; i < nelts; i++)
	 new_builder.quick_push (sel4[sel5[i].to_constant ()]);

       vec_perm_indices new_sel (new_builder, 2, sel4.nelts_per_input ());

       /* The new permute has the element count of result and the element
	  type of op2, so it has the type of result unless the
	  VIEW_CONVERT_EXPR changes the element type.  */
       tree perm_type = type;
       if (!types_match (TREE_TYPE (type), TREE_TYPE (TREE_TYPE (@2))))
	 perm_type = build_vector_type (TREE_TYPE (TREE_TYPE (@2)), nelts);
       machine_mode perm_mode = TYPE_MODE (perm_type);

       tree new_vcst = NULL_TREE;
       /* If the new VEC_PERM_EXPR can't be handled but both
	  original VEC_PERM_EXPRs can, punt.
	  If one or both of the original VEC_PERM_EXPRs can't be
	  handled and the new one can't be either, don't increase
	  number of VEC_PERM_EXPRs that can't be handled.  */
       if (can_vec_perm_const_p (perm_mode, op2_mode, new_sel, false)
	   || (single_use (@0)
	       ? (!can_vec_perm_const_p (inner_mode, op2_mode, sel4, true)
		  || !can_vec_perm_const_p (result_mode, vc_mode, sel5, true))
	       : !can_vec_perm_const_p (result_mode, vc_mode, sel5, true)))
	 /* Give the new mask ssizetype elements, as the vectorizer does:
	    the element type of sel5's mask can be too narrow to index
	    both op2 and op3.  */
	 new_vcst
	   = vec_perm_indices_to_tree (build_vector_type (ssizetype, nelts),
				       new_sel);
     }
     (if (new_vcst)
      (view_convert (vec_perm:perm_type @2 @3 { new_vcst; }))))))))

/* Merge
     c = VEC_PERM_EXPR <a, b, VCST0>;
     d = VEC_PERM_EXPR <x, c, VCST1>;
   to
     d = VEC_PERM_EXPR <x, {a,b}, NEW_VCST>;
   when all elements from a or b are replaced by the later
   permutation.

   If x is a, the elements c takes from a can be read from x instead,
   so only b remains and d becomes VEC_PERM_EXPR <a, b, NEW_VCST>
   whatever mix of a and b it uses.  Likewise if x is b.  Do this only
   when c has no other use, as the new mask may be more costly.  */

(simplify
 (vec_perm @5 (vec_perm@0 @1 @2 VECTOR_CST@3) VECTOR_CST@4)
 /* The new permute replaces c by a or b, which therefore must have the
    type of x.  c may have more elements than d.  */
 (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()
      && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant ()
      && types_match (@1, @5))
  (with
   {
     machine_mode result_mode = TYPE_MODE (type);
     machine_mode op_mode = TYPE_MODE (TREE_TYPE (@1));
     machine_mode c_mode = TYPE_MODE (TREE_TYPE (@0));
     int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
     vec_perm_indices sel0, sel1;
   }
   (if (tree_to_vec_perm_indices (&sel0, @1, @2, @3)
        && tree_to_vec_perm_indices (&sel1, @5, @0, @4))
    (with
     {
       bool use_1 = false, use_2 = false;
       /* Whether to read the elements c takes from a or b from x.  */
       bool x_is_1 = single_use (@0) && operand_equal_p (@5, @1, 0);
       bool x_is_2 = single_use (@0) && operand_equal_p (@5, @2, 0);

       vec_perm_builder builder2 (nelts, nelts, 2);
       for (int i = 0; i < nelts; i++)
         {
	   if (known_lt ((poly_uint64)sel1[i], sel1.nelts_per_input ()))
	     builder2.quick_push (sel1[i]);
	   else
	     {
	       poly_uint64 j = sel0[(sel1[i] - sel1.nelts_per_input ())
				    .to_constant ()];
	       bool from_x;
	       if (known_lt (j, sel0.nelts_per_input ()))
		 {
		   from_x = x_is_1;
		   use_1 |= !from_x;
		 }
	       else
	         {
		   from_x = x_is_2;
		   use_2 |= !from_x;
		   j -= sel0.nelts_per_input ();
		 }
	       builder2.quick_push (from_x ? j : j + sel1.nelts_per_input ());
	     }
	 }
     }
     (if (x_is_1 || x_is_2 || (use_1 ^ use_2))
      (with
       {
	 vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ());
	 tree op0 = NULL_TREE;
	 /* If the new VEC_PERM_EXPR can't be handled but both
	    original VEC_PERM_EXPRs can, punt.
	    If one or both of the original VEC_PERM_EXPRs can't be
	    handled and the new one can't be either, don't increase
	    number of VEC_PERM_EXPRs that can't be handled.  */
	 if (can_vec_perm_const_p (result_mode, op_mode, sel2, false)
	     || (single_use (@0)
		 ? (!can_vec_perm_const_p (c_mode, op_mode, sel0, true)
		    || !can_vec_perm_const_p (result_mode, op_mode, sel1, true))
		 : !can_vec_perm_const_p (result_mode, op_mode, sel1, true)))
	   /* Give the new mask ssizetype elements, as the vectorizer does:
	      the element type of d's mask can be too narrow to index both
	      inputs.  */
	   op0
	     = vec_perm_indices_to_tree (build_vector_type (ssizetype, nelts),
					 sel2);
       }
       (if (op0)
	(switch
	 (if (use_1)
	  (vec_perm @5 @1 { op0; }))
	 (if (use_2)
	  (vec_perm @5 @2 { op0; }))
	 (vec_perm @5 @5 { op0; }))))))))))

/* And the case with swapped outer permute sources.  */

(simplify
 (vec_perm (vec_perm@0 @1 @2 VECTOR_CST@3) @5 VECTOR_CST@4)
 /* The new permute replaces c by a or b, which therefore must have the
    type of x.  c may have more elements than d.  */
 (if (TYPE_VECTOR_SUBPARTS (type).is_constant ()
      && TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)).is_constant ()
      && types_match (@1, @5))
  (with
   {
     machine_mode result_mode = TYPE_MODE (type);
     machine_mode op_mode = TYPE_MODE (TREE_TYPE (@1));
     machine_mode c_mode = TYPE_MODE (TREE_TYPE (@0));
     int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
     vec_perm_indices sel0, sel1;
   }
   (if (tree_to_vec_perm_indices (&sel0, @1, @2, @3)
        && tree_to_vec_perm_indices (&sel1, @0, @5, @4))
    (with
     {
       bool use_1 = false, use_2 = false;
       /* Whether to read the elements c takes from a or b from x.  */
       bool x_is_1 = single_use (@0) && operand_equal_p (@5, @1, 0);
       bool x_is_2 = single_use (@0) && operand_equal_p (@5, @2, 0);

       vec_perm_builder builder2 (nelts, nelts, 2);
       for (int i = 0; i < nelts; i++)
         {
	   if (known_ge ((poly_uint64)sel1[i], sel1.nelts_per_input ()))
	     builder2.quick_push (sel1[i]);
	   else
	     {
	       poly_uint64 j = sel0[sel1[i].to_constant ()];
	       bool from_x;
	       if (known_lt (j, sel0.nelts_per_input ()))
		 {
		   from_x = x_is_1;
		   use_1 |= !from_x;
		 }
	       else
	         {
		   from_x = x_is_2;
		   use_2 |= !from_x;
		   j -= sel0.nelts_per_input ();
		 }
	       builder2.quick_push (from_x ? j + sel1.nelts_per_input () : j);
	     }
	 }
     }
     (if (x_is_1 || x_is_2 || (use_1 ^ use_2))
      (with
       {
	 vec_perm_indices sel2 (builder2, 2, sel1.nelts_per_input ());
	 tree op0 = NULL_TREE;
	 /* If the new VEC_PERM_EXPR can't be handled but both
	    original VEC_PERM_EXPRs can, punt.
	    If one or both of the original VEC_PERM_EXPRs can't be
	    handled and the new one can't be either, don't increase
	    number of VEC_PERM_EXPRs that can't be handled.  */
	 if (can_vec_perm_const_p (result_mode, op_mode, sel2, false)
	     || (single_use (@0)
		 ? (!can_vec_perm_const_p (c_mode, op_mode, sel0, true)
		    || !can_vec_perm_const_p (result_mode, op_mode, sel1, true))
		 : !can_vec_perm_const_p (result_mode, op_mode, sel1, true)))
	   /* Give the new mask ssizetype elements, as the vectorizer does:
	      the element type of d's mask can be too narrow to index both
	      inputs.  */
	   op0
	     = vec_perm_indices_to_tree (build_vector_type (ssizetype, nelts),
					 sel2);
       }
       (if (op0)
	(switch
	 (if (use_1)
	  (vec_perm @1 @5 { op0; }))
	 (if (use_2)
	  (vec_perm @2 @5 { op0; }))
	 (vec_perm @5 @5 { op0; }))))))))))


#if GIMPLE
/* Match count trailing zeroes for simplify_count_zeroes in forwprop.
   The canonical form is array[((x & -x) * C) >> SHIFT] where C is a magic
   constant which when multiplied by a power of 2 contains a unique value
   in the top 5 or 6 bits.  This is then indexed into a table which maps it
   to the number of trailing zeroes.  */
(match (ctz_table_index @1 @2 @3)
  (rshift (mult (bit_and:c (negate @1) @1) INTEGER_CST@2) INTEGER_CST@3))

/* Helpers for clz_table_index and its variants: match the 5- or
   6-stage MSB OR-cascade that drives a DeBruijn CLZ lookup.  Given a
   value @1, the cascade sets every bit from 0 to the MSB of @1,
   producing 2^(k+1) - 1 where k is the MSB position.  */
(match (msb_or_cascade_32 @1)
  (bit_ior (rshift
    (bit_ior@d (rshift
      (bit_ior@c (rshift
	(bit_ior@b (rshift
	  (bit_ior@a (rshift @1 INTEGER_CST@c1) @1)
	  INTEGER_CST@c2) @a)
	INTEGER_CST@c4) @b)
      INTEGER_CST@c8) @c)
    INTEGER_CST@c16) @d)
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_UNSIGNED (type)
       && TYPE_PRECISION (type) == 32
       && compare_tree_int (@c1, 1) == 0
       && compare_tree_int (@c2, 2) == 0
       && compare_tree_int (@c4, 4) == 0
       && compare_tree_int (@c8, 8) == 0
       && compare_tree_int (@c16, 16) == 0)))

(match (msb_or_cascade_64 @1)
  (bit_ior (rshift
    (bit_ior@e (rshift
      (bit_ior@d (rshift
	(bit_ior@c (rshift
	  (bit_ior@b (rshift
	    (bit_ior@a (rshift @1 INTEGER_CST@c1) @1)
	    INTEGER_CST@c2) @a)
	  INTEGER_CST@c4) @b)
	INTEGER_CST@c8) @c)
      INTEGER_CST@c16) @d)
    INTEGER_CST@c32) @e)
  (if (INTEGRAL_TYPE_P (type)
       && TYPE_UNSIGNED (type)
       && TYPE_PRECISION (type) == 64
       && compare_tree_int (@c1, 1) == 0
       && compare_tree_int (@c2, 2) == 0
       && compare_tree_int (@c4, 4) == 0
       && compare_tree_int (@c8, 8) == 0
       && compare_tree_int (@c16, 16) == 0
       && compare_tree_int (@c32, 32) == 0)))

/* Match count leading zeros for simplify_count_zeroes in forwprop.
   One canonical form is 31 - array[idx] where IDX is computed from X
   by first setting all bits from the topmost set bits down via a
   series of shifts and ors to X' and then computing (X' * C) >> SHIFT.  */
(match (clz_table_index @1 @2 @3)
  (rshift (mult (msb_or_cascade_32 @1) INTEGER_CST@2) INTEGER_CST@3))
(match (clz_table_index @1 @2 @3)
  (rshift (mult (msb_or_cascade_64 @1) INTEGER_CST@2) INTEGER_CST@3))

/* Match count leading zeros for simplify_count_zeroes in forwprop,
   variant that isolates the MSB as a power of two via (s - (s >> 1))
   after the OR-cascade.  For s = 2^(k+1) - 1 (the cascade output),
   (s - (s >> 1)) equals 2^k, so the subsequent DeBruijn
   multiply-and-shift is a CTZ-style lookup on the isolated MSB.  The
   table has to satisfy the direct CTZ DeBruijn property (validated by
   the CTZ checkfn in simplify_count_zeroes).
   PR tree-optimization/122569.  */
(match (clz_msb_iso_table_index @1 @2 @3)
  (rshift (mult
   (minus (msb_or_cascade_64@f @1) (rshift @f INTEGER_CST@sub1))
   INTEGER_CST@2) INTEGER_CST@3)
  (if (compare_tree_int (@sub1, 1) == 0)))

/* Match low and high parts of longhand multiplication.
   Given a 2N-bit unsigned type, x = xh*2^N + xl and y = yh*2^N + yl,
   where xh, xl, yh, yl are N-bit halves extracted via shifts and masks.  */

/* High half: op >> N.  */
(match (mul_hi @op @0)
  (rshift @op INTEGER_CST@0)
  (with { tree op_type = TREE_TYPE (@op); }
  (if (INTEGRAL_TYPE_P (op_type)
       && TYPE_UNSIGNED (op_type)
       && TYPE_PRECISION (op_type) % 2 == 0
       && wi::to_wide (@0) == TYPE_PRECISION (op_type) / 2))))
/* Low half: op & mask.  */
(match (mul_lo @op @0)
  (bit_and @op INTEGER_CST@0)
  (with { tree op_type = TREE_TYPE (@op); }
  (if (INTEGRAL_TYPE_P (op_type)
       && TYPE_UNSIGNED (op_type)
       && TYPE_PRECISION (op_type) % 2 == 0
       && (wi::to_wide (@0)
	   == wi::mask (TYPE_PRECISION (op_type) / 2, false,
			TYPE_PRECISION (op_type)))))))
/* Cross product: high(op0) * low(op1).  */
(match (mul_hilo @op0 @op1 @0 @1)
  (mult:c
    (mul_hi @op0 @0)
    (mul_lo @op1 @1)))
/* Low-low product: low(op0) * low(op1).  */
(match (mul_lolo @op0 @op1 @0)
  (mult:c
    (mul_lo @op0 @0)
    (mul_lo @op1 @0)))
/* High-high product: high(op0) * high(op1).  */
(match (mul_hihi @op0 @op1 @0)
  (mult:c
    (mul_hi @op0 @0)
    (mul_hi @op1 @0)))
/* Cross sum: xh*yl + xl*yh.
   Note: matches any PLUS; operands are validated as actual cross
   products by the forwprop consumer (long_mul_check_consistency).  */
(match (mul_cross_sum @mul_hilo0 @mul_hilo1)
  (plus @mul_hilo0 @mul_hilo1))
/* Low sum: cross_sum + (xl*yl >> N).  */
(match (mul_low_sum @op0 @op1 @mul_hilo0 @mul_hilo1 @0 @1)
  (plus:c
    (mul_cross_sum @mul_hilo0 @mul_hilo1)
    (mul_hi
      (mul_lolo @op0 @op1 @1)
      @0)))
/* Carry from low-sum overflow: (cast?) (hilo > low_sum) << N.
   No explicit type/width guard needed: mul_low_sum delegates to
   mul_cross_sum + mul_hi + mul_lolo, which provide deep structural
   constraints, and @shift0 ties the shift amount to the inner constants.  */
(match (mul_carry_low_sum @op0 @op1 @mul_hilo0 @mul_hilo1 @mul_hilo2 @shift0)
  (lshift
    (convert (gt
      @mul_hilo0
      (mul_low_sum @op0 @op1 @mul_hilo1 @mul_hilo2 @shift0
       @1)))
    @shift0))
/* Carry from cross-sum overflow: (cast?) (hilo > cross_sum) << N.
   Explicit guard required because mul_cross_sum is just (plus:c @0 @1)
   with no inherent type or halfwidth constraint.  */
(match (mul_carry_cross_sum @mul_hilo0 @mul_hilo1 @mul_hilo2 @0)
  (lshift
    (convert (gt
      @mul_hilo0
      (mul_cross_sum @mul_hilo1 @mul_hilo2)))
    INTEGER_CST@0)
  (with { tree op_type = TREE_TYPE (@mul_hilo0); }
  (if (INTEGRAL_TYPE_P (op_type)
       && TYPE_UNSIGNED (op_type)
       && TYPE_PRECISION (op_type) % 2 == 0
       && wi::to_wide (@0) == TYPE_PRECISION (op_type) / 2))))
/* Carry from addition overflow: (cast?) (a > a + b).
   :c on gt also matches the LT form: (cast?) (a + b < a).  */
(match (mul_carry_low @0 @1)
  (convert?
    (gt:c @0 (plus:c @1 @0)))
  (with { tree op_type = TREE_TYPE (@0); }
  (if (INTEGRAL_TYPE_P (op_type) && TYPE_UNSIGNED (op_type)))))
/* Low accumulate: (xl*yl >> N) + (cross_sum & mask).  */
(match (mul_low_accum @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (mul_hi
      (mul_lolo @op0 @op1 @0)
      @1)
    (mul_lo (mul_cross_sum @mul_hilo0 @mul_hilo1) @0)))
/* Ladder sum form 1: (hilo0 & mask) + hilo1 + (xl*yl >> N).
   First variant: @mul_hilo1 is inside the inner plus:c alongside
   (mul_lo ...).  */
(match (mul_ladder_sum1 @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (plus:c
      (mul_lo
	@mul_hilo0
	@0)
      @mul_hilo1)
    (mul_hi (mul_lolo @op0 @op1 @0) @1)))
/* Second variant: @mul_hilo1 is the outermost addend and could match
   anything, so guard that its definition is a MULT_EXPR.  */
(match (mul_ladder_sum1 @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (plus:c
      (mul_lo @mul_hilo0 @0)
      (mul_hi (mul_lolo @op0 @op1 @0) @1))
    (mult@mul_hilo1 @2 @3)))
/* Partial ladder sum: (xl*yl >> N) + hilo.  */
(match (mul_ladder_part_sum @op0 @op1 @mul_hilo0 @0 @1)
  (plus:c
    (mul_hi (mul_lolo @op0 @op1 @0) @1)
    @mul_hilo0))
/* Ladder sum form 2: (ladder_part_sum & mask) + hilo.  */
(match (mul_ladder_sum2 @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (mul_lo
      (mul_ladder_part_sum @op0 @op1 @mul_hilo0 @0 @1)
      @0)
    @mul_hilo1))
/* Ladder sum form 3: (hilo0 & mask) + (hilo1 & mask) + (xl*yl >> N).  */
(match (mul_ladder_sum3 @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (plus:c
      (mul_lo @mul_hilo0 @0)
      (mul_lo @mul_hilo1 @0))
    (mul_hi (mul_lolo @op0 @op1 @0) @1)))
(match (mul_ladder_sum3 @op0 @op1 @mul_hilo0 @mul_hilo1)
  (plus:c
    (plus:c
      (mul_lo @mul_hilo0 @0)
      (mul_hi (mul_lolo @op0 @op1 @0) @1))
    (mul_lo @mul_hilo1 @0)))

/* Long-multiply high-part emit chain produced by forwprop's recognizer:

     (N) ((2N) op1 * (2N) op2) >> N

   `(convert? @X)` accepts a bare operand (PRE may hoist the (T_2N) cast
   out into a PHI on a shared slot).  The lowering splits an operand
   wider than T_N into T_N halves rather than truncating it.  Wide type
   must be unsigned mode-precision integer of even width; BITINT_TYPE is
   refused.  */
(match (long_mul_high_chain @0 @1)
  (convert (rshift (mult:c@3 (convert? @0) (convert? @1)) INTEGER_CST@2))
  (with { tree wide_type = TREE_TYPE (@3); }
  (if (INTEGRAL_TYPE_P (wide_type)
       && TYPE_UNSIGNED (wide_type)
       && type_has_mode_precision_p (wide_type)
       && TREE_CODE (wide_type) != BITINT_TYPE
       && TYPE_PRECISION (wide_type) >= 8
       && TYPE_PRECISION (wide_type) % 4 == 0
       && wi::to_wide (@2) == TYPE_PRECISION (wide_type) / 2))))

/* Floatint point/integer comparison and integer->integer
   or floating point -> float point conversion.  */
(match (cond_expr_convert_p @0 @2 @3 @6)
 (cond (simple_comparison@6 @0 @1) (convert@4 @2) (convert@5 @3))
  (if ((INTEGRAL_TYPE_P (type)
       || (!flag_trapping_math && SCALAR_FLOAT_TYPE_P (type)))
       && ((INTEGRAL_TYPE_P (TREE_TYPE (@2))
	    && tree_nop_conversion_p (TREE_TYPE (@2), TREE_TYPE (@3)))
	   || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (@2))
	       && types_match (TREE_TYPE (@2), TREE_TYPE (@3))))
       && !operand_equal_p (TYPE_SIZE (type), TYPE_SIZE (TREE_TYPE (@0)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@2)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@3)))
       /* For vect_recog_cond_expr_convert_pattern, @2 and @3 can differ in
	  signess when convert is truncation, but not ok for extension since
	  it's sign_extend vs zero_extend.  */
       && (known_gt (tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (@0))),
		     tree_to_poly_uint64 (TYPE_SIZE (type)))
	   || (TYPE_UNSIGNED (TREE_TYPE (@2))
	       == TYPE_UNSIGNED (TREE_TYPE (@3))))
       && single_use (@4)
       && single_use (@5))))

/* Floating point or integer comparison and integer to floating point
   conversion.  */
(match (cond_expr_convert_p @0 @2 @3 @6)
 (cond (simple_comparison@6 @0 @1) (float@4 @2) (float@5 @3))
  (if (SCALAR_FLOAT_TYPE_P (type) && !flag_trapping_math
       && INTEGRAL_TYPE_P (TREE_TYPE (@2))
       && types_match (TREE_TYPE (@2), TREE_TYPE (@3))
       && !operand_equal_p (TYPE_SIZE (type), TYPE_SIZE (TREE_TYPE (@0)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@2)))
       && single_use (@4)
       && single_use (@5))))

/* Floating point or integer comparison and floating point to integer
   conversion.  */
(match (cond_expr_convert_p @0 @2 @3 @6)
 (cond (simple_comparison@6 @0 @1) (fix_trunc@4 @2) (fix_trunc@5 @3))
  (if (INTEGRAL_TYPE_P (type) && !flag_trapping_math
       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (@2))
       && types_match (TREE_TYPE (@2), TREE_TYPE (@3))
       && !operand_equal_p (TYPE_SIZE (type),
			    TYPE_SIZE (TREE_TYPE (@0)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@2)))
       && single_use (@4)
       && single_use (@5))))

/* Floating point or integer comparison and floating point conversion
   with REAL_CST.  */
(match (cond_expr_convert_p @0 @2 @3 @6)
 (cond (simple_comparison@6 @0 @1) (REAL_CST@2) (convert@5 @3))
  (if (!flag_trapping_math
       && SCALAR_FLOAT_TYPE_P (type)
       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (@3))
       && !operand_equal_p (TYPE_SIZE (type),
			    TYPE_SIZE (TREE_TYPE (@0)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@3)))
       && single_use (@5)
       && (flag_unsafe_math_optimizations
	   || exact_real_truncate (TYPE_MODE (TREE_TYPE (@3)),
				   &TREE_REAL_CST (@2))))))

/* Floating point or integer comparison and floating point conversion
   with REAL_CST.  */
(match (cond_expr_convert_p @0 @2 @3 @6)
 (cond (simple_comparison@6 @0 @1) (convert@4 @2) (REAL_CST@3))
  (if (!flag_trapping_math
       && SCALAR_FLOAT_TYPE_P (type)
       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (@2))
       && !operand_equal_p (TYPE_SIZE (type), TYPE_SIZE (TREE_TYPE (@0)))
       && operand_equal_p (TYPE_SIZE (TREE_TYPE (@0)),
			   TYPE_SIZE (TREE_TYPE (@2)))
       && single_use (@4)
       && (flag_unsafe_math_optimizations
	   || exact_real_truncate (TYPE_MODE (TREE_TYPE (@2)),
				   &TREE_REAL_CST (@3))))))

(for bit_op (bit_and bit_ior bit_xor)
 (match (bitwise_induction_p @0 @2 @3)
  (bit_op:c
   (nop_convert1? (bit_not2?@0 (convert3? (lshift integer_onep@1 @2))))
   @3)))

(match (bitwise_induction_p @0 @2 @3)
 (bit_not
  (nop_convert1? (bit_xor@0 (convert2? (lshift integer_onep@1 @2)) @3))))

#endif

/* n - (((n > C1) ? n : C1) & -C2) ->  n & C1 for unsigned case.
   n - (((n > C1) ? n : C1) & -C2) ->  (n <= C1) ? n : (n & C1) for signed case.  */
(simplify
  (minus @0 (bit_and (max @0 INTEGER_CST@1) INTEGER_CST@2))
  (with { auto i = wi::neg (wi::to_wide (@2)); }
  /* Check if -C2 is a power of 2 and C1 = -C2 - 1.  */
    (if (wi::popcount (i) == 1
         && (wi::to_wide (@1)) == (i - 1))
      (if (TYPE_UNSIGNED (TREE_TYPE (@0)))
        (bit_and @0 @1)
      (cond (le @0 @1) @0 (bit_and @0 @1))))))

/* -x & 1 -> x & 1.  */
(simplify
 (bit_and (negate @0) integer_onep@1)
 (if (!TYPE_OVERFLOW_SANITIZED (type))
  (bit_and @0 @1)))

/* `-a` is just `a` if the type is 1bit wide or when converting
   to a 1bit type; similar to the above transformation of `(-x)&1`.
   This is used mostly with the transformation of
   `a ? ~b : b` into `(-a)^b`.
   It also can show up with bitfields.  */
(simplify
 (convert? (negate @0))
 (if (INTEGRAL_TYPE_P (type)
      && TYPE_PRECISION (type) == 1
      && !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@0)))
  (convert @0)))

/* Optimize
   c1 = VEC_PERM_EXPR (a, a, mask)
   c2 = VEC_PERM_EXPR (b, b, mask)
   c3 = c1 op c2
   -->
   c = a op b
   c3 = VEC_PERM_EXPR (c, c, mask)
   For all integer non-div operations.  */
(for op (plus minus mult bit_and bit_ior bit_xor
	 lshift rshift)
 (simplify
  (op (vec_perm @0 @0 @2) (vec_perm @1 @1 @2))
   (if (VECTOR_INTEGER_TYPE_P (type))
    (vec_perm (op@3 @0 @1) @3 @2))))

/* Similar for float arithmetic when permutation constant covers
   all vector elements.  */
(for op (plus minus mult)
 (simplify
  (op (vec_perm @0 @0 VECTOR_CST@2) (vec_perm @1 @1 VECTOR_CST@2))
   (if (VECTOR_FLOAT_TYPE_P (type)
	&& TYPE_VECTOR_SUBPARTS (type).is_constant ())
    (with
     {
       tree perm_cst = @2;
       vec_perm_builder builder;
       bool full_perm_p = false;
       if (tree_to_vec_perm_builder (&builder, perm_cst))
	 {
	   unsigned HOST_WIDE_INT nelts;

	   nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
	   /* Create a vec_perm_indices for the VECTOR_CST.  */
	   vec_perm_indices sel (builder, 1, nelts);

	   /* Check if perm indices covers all vector elements.  */
	   if (sel.encoding ().encoded_full_vector_p ())
	     {
	       auto_sbitmap seen (nelts);
	       bitmap_clear (seen);

	       unsigned HOST_WIDE_INT count = 0, i;

	       for (i = 0; i < nelts; i++)
		 {
		   if (!bitmap_set_bit (seen, sel[i].to_constant ()))
		    break;
		   count++;
		 }
	       full_perm_p = count == nelts;
	     }
	 }
      }
      (if (full_perm_p)
	(vec_perm (op@3 @0 @1) @3 @2))))))

/* Fold
     x = VEC_PERM_EXPR <a, ANY, sel0>;
     y = VEC_PERM_EXPR <ANY, b, sel0>;
     c = VEC_PERM_EXPR <x, y, sel1>;
   into
     c = VEC_PERM_EXPR <a, b, sel0>;
   if sel0 combined with sel1 denotes extracting a contiguous subvector from
   the conceptual concatenated [ a | b ].  */
(simplify
 (vec_perm (view_convert? (vec_perm @0 @4 VECTOR_CST@2))
	   (view_convert? (vec_perm @5 @1 VECTOR_CST@2))
	   VECTOR_CST@3)
 (with
  {
    bool can_fold = false;
    unsigned HOST_WIDE_INT nelts;
    vec_perm_builder builder;
    if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts)
	&& tree_to_vec_perm_builder (&builder, @2))
      {
	/* Set can_fold to true when
	   - sel0 is a vector of consecutive indices.
	   - sel1 is composed of two parts of consecutive indices [ ia | ib ],
	     selecting the elements originally in 'a' and 'b', respectively.  */
	vec_perm_indices sel0 (builder, 2, VECTOR_CST_NELTS (@2));
	unsigned int sel0_first_idx = sel0[0].to_constant ();
	unsigned int elt_size = vector_element_bits (TREE_TYPE (@0));
	unsigned int ia_size = tree_to_uhwi (TYPE_SIZE (type))
			       - elt_size * sel0_first_idx;
	unsigned int ib_start;
	if (sel0.series_p (0, 1, sel0_first_idx, 1)
	    && multiple_p (ia_size, vector_element_bits (type), &ib_start)
	    && tree_to_vec_perm_builder (&builder, @3))
	  {
	    /* Check if the ib part contains consecutive indices starting from
	       'nelts + ib_start'.  */
	    vec_perm_indices sel1 (builder, 2, VECTOR_CST_NELTS (@3));
	    can_fold = sel1.series_p (ib_start, 1, nelts + ib_start, 1);

	    /* Check if the ia part contains indices [0 ... ib_start - 1].  */
	    if (can_fold)
	      for (unsigned int i = 0; i < ib_start; i++)
		if (sel1[i].to_constant () != i)
		  {
		    can_fold = false;
		    break;
		  }
	  }
      }
  }
  (if (can_fold)
    (view_convert (vec_perm @0 @1 @2)))))

/* Simplify
     v = VEC_PERM_EXPR <op0, op1, sel>;
     res = BIT_INSERT_EXPR <v, new_elt, bit_pos>;
   by replacing either op0 or op1 with the other one when
     1) BIT_INSERT_EXPR inserts exactly one full lane of v.
     2) All other lanes come from a single source, either op0 or op1.

   The vector operand that contributes only to the overwritten lane is dead.
   Replacing it avoids materializing a filler vector.  */

(simplify
 (bit_insert (vec_perm:s @0 @1 VECTOR_CST@2) @3 INTEGER_CST@4)
 (with
  {
    bool from0 = false;
    bool from1 = false;
    tree single_op = NULL_TREE;
    unsigned elt_size = vector_element_bits (type);

    unsigned HOST_WIDE_INT nelts;
    unsigned ins_lane_idx;
    vec_perm_indices sel;

    /* Require fixed-length vectors, different operands to VEC_PERM_EXPR and
       BIT_INSERT_EXPR to insert exactly one full lane.  */
    if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts)
	&& @0 != @1
	&& tree_to_vec_perm_indices (&sel, @0, @1, @2)
	&& tree_to_uhwi (TYPE_SIZE (TREE_TYPE (@3))) == elt_size
	&& multiple_p (tree_to_uhwi (@4), elt_size, &ins_lane_idx))
      {
	for (unsigned i = 0; i < nelts; i++)
	  {
	    /* Skip the overwritten lane.  */
	    if (i == ins_lane_idx)
	      continue;

	    /* Set FROM0 / FROM1 if current lane comes from @0 / @1.  */
	    unsigned HOST_WIDE_INT elt = sel[i].to_constant ();
	    if (elt < nelts)
	      from0 = true;
	    else
	      from1 = true;
	  }

	/* If only one of FROM0 and FROM1 is true, all live lanes come from
	   a single source.  */
	if (from0 ^ from1)
	  {
	    single_op = from0 ? @0 : @1;
	    vec_perm_indices new_sel;
	    if (!tree_to_vec_perm_indices (&new_sel, single_op, single_op, @2)
		|| !can_vec_perm_const_p (TYPE_MODE (type),
					  TYPE_MODE (TREE_TYPE (single_op)),
					  new_sel,
					  false))
	      single_op = NULL_TREE;
	  }
      }
  }
  (if (single_op != NULL_TREE)
    (bit_insert (vec_perm { single_op; } { single_op; } @2) @3 @4))))

#if GIMPLE
/* Simplify (a >> 1) + (b >> 1) + ((a | b) & 1) to .AVG_CEIL (a, b).
   Similar for (a | b) - ((a ^ b) >> 1).  */

(simplify
  (plus:c
    (plus (rshift @0 integer_onep@1) (rshift @2 @1))
    (bit_and (bit_ior @0 @2) integer_onep@3))
  (if (fold_before_rtl_expansion_p ()
      && VECTOR_TYPE_P (type)
      && direct_internal_fn_supported_p (IFN_AVG_CEIL, type, OPTIMIZE_FOR_BOTH))
      (IFN_AVG_CEIL @0 @2)))

(simplify
  (minus
    (bit_ior @0 @2)
    (rshift (bit_xor @0 @2) integer_onep@1))
  (if (fold_before_rtl_expansion_p ()
      && VECTOR_TYPE_P (type)
      && direct_internal_fn_supported_p (IFN_AVG_CEIL, type, OPTIMIZE_FOR_BOTH))
      (IFN_AVG_CEIL @0 @2)))
#endif

/* vec shift left insert (dup (A), A) -> dup(A) */
(simplify
 (IFN_VEC_SHL_INSERT (vec_duplicate@1 @0) @0)
  @1)

/* In this case the XOR flips bits that originate from the result of the
   right shift and do not impact the result of the left shift.   We can
   reassociate the XOR to work on the final result and simplify the rest
   to a rotate.  */
(simplify
  (bit_ior:c (lshift @0 INTEGER_CST@1)
	     (bit_xor (rshift @2 INTEGER_CST@3) INTEGER_CST@4))
   (if (type_has_mode_precision_p (type)
	&& tree_fits_uhwi_p (@1)
	&& tree_fits_uhwi_p (@3)
	&& (wi::mask (tree_to_uhwi (@1), true, TYPE_PRECISION (type)) & wi::to_wide (@4)) == 0
        && (tree_to_uhwi (@1) + tree_to_uhwi (@3)) == TYPE_PRECISION (type)
        && TYPE_UNSIGNED (type)
        && @0 == @2)
    (bit_xor (rrotate @0 @3) @4)))

/* Similarly, but in this case the XOR flips bits that originate from the
   result of the left shift.  */
(simplify
  (bit_ior:c (bit_xor (lshift @0 INTEGER_CST@1) INTEGER_CST@2)
	     (rshift @3 INTEGER_CST@4))
   (if (type_has_mode_precision_p (type)
	&& tree_fits_uhwi_p (@1)
	&& tree_fits_uhwi_p (@4)
	&& ((wi::mask (tree_to_uhwi (@1), false, TYPE_PRECISION (type)) & wi::to_wide (@2)) == 0)
        && (tree_to_uhwi (@1) + tree_to_uhwi (@4)) == TYPE_PRECISION (type)
        && TYPE_UNSIGNED (type)
        && @0 == @3)
    (bit_xor (rrotate @0 @4) @2)))


/* (T0)(1 << x) & (T1)(1 << x) -> (T)(1 << x),
 *  but keep the shift in the wider type to avoid introducing
 *  undefined behaviour.  */
(simplify
 (bit_and:c
  (convert? (lshift@2 integer_onep@1 @0))
  (convert? (lshift@3 integer_onep@4 @0)))
 (with
  {
    tree ltype0 = TREE_TYPE (@2);
    tree ltype1 = TREE_TYPE (@3);
  }
  (if (INTEGRAL_TYPE_P (type)
       && INTEGRAL_TYPE_P (ltype0)
       && INTEGRAL_TYPE_P (ltype1)
       && element_precision (type) <= element_precision (ltype0)
       && element_precision (ltype0) <= element_precision (ltype1))
       (convert:type
	  (lshift:ltype1 { @4; } @0)))))

/* Optimize extraction from a uniform vector to a representative element as
   long as the requested element is within range.  */
(simplify (IFN_VEC_EXTRACT (vec_duplicate @0) INTEGER_CST@1)
 @0)

/* Strip nop-conversions __builtin_constant_p arguments.  */
(simplify
 (BUILT_IN_CONSTANT_P (nop_convert@1 @0))
 (BUILT_IN_CONSTANT_P @0))

/* Canonicalize a conditional signed narrow clip while the signed negation
   has its original overflow semantics:

     (NT) ((UT) X > NT_MAX ? (-X) >> (PREC - 1) : X)
       -> (NT) MIN ((UT) MAX (X, 0), NT_MAX).  */
(if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
 (simplify
  (cond (gt (nop_convert@4 @0) INTEGER_CST@1)
	(convert (rshift@5
		  (nop_convert? (negate@3 (nop_convert? @0)))
		  INTEGER_CST@2))
	(convert @0))
  (if (!TREE_SIDE_EFFECTS (@0)
	&& !TYPE_UNSIGNED (TREE_TYPE (@0))
	&& TYPE_UNSIGNED (TREE_TYPE (@4))
	&& !TYPE_UNSIGNED (TREE_TYPE (@5))
	&& TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@3))
	&& !TYPE_OVERFLOW_SANITIZED (TREE_TYPE (@3)))
   (with
    {
     tree utype = TREE_TYPE (@4);
     unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0));
     unsigned otype_precision = TYPE_PRECISION (type);
     wide_int trunc_max = wi::mask (otype_precision, false, itype_precision);
     wide_int int_cst_1 = wi::to_wide (@1, itype_precision);
     wide_int int_cst_2 = wi::to_wide (@2, itype_precision);
     wide_int shift_amount = wi::uhwi (itype_precision - 1,
				  itype_precision);
    }
    (if (otype_precision < itype_precision
	 && wi::eq_p (trunc_max, int_cst_1)
	 && wi::eq_p (int_cst_2, shift_amount))
     (convert (min:utype
		(convert:utype
		 (max @0 { build_zero_cst (TREE_TYPE (@0)); }))
		@1)))))))

#if GIMPLE

/* Include the saturation alu match patterns.  */
#include "match-sat-alu.pd"

#endif

/* The boundary condition for case 10: IMM = 1:
   SAT_U_SUB = X >= IMM ? (X - IMM) : 0.
   simplify (X != 0 ? X + ~0 : 0) to X - (X != 0).  */
(simplify
 (cond (ne@1 @0 integer_zerop)
       (nop_convert1? (plus (nop_convert2?@2 @0) integer_all_onesp))
       integer_zerop)
 (if (INTEGRAL_TYPE_P (type))
   (with { tree itype = TREE_TYPE (@2); }
    (convert (minus @2 (convert:itype @1))))))

#if GIMPLE
/* Replace (x == C) ? y : MIN (x, y) -> MIN (x, y) when C >= upper_bound (y),
   and (x == C) ? y : MAX (x, y) -> MAX (x, y) when C <= lower_bound (y),
   because in both cases the guarded arm equals the min/max expression.  */
(for minmax (min max)
 (simplify
  (cond (eq @0 INTEGER_CST@1) @2 (minmax:c@3 @0 @2))
  (if (INTEGRAL_TYPE_P (type))
   (with {
     int_range_max r;
     if (!gimple_match_range_of_expr (r, @2, @2))
       r.set_varying (TREE_TYPE (@2));
     wide_int bound;
     if (minmax == MIN_EXPR)
       bound = r.upper_bound ();
     else
       bound = r.lower_bound ();
    }
   (if ((minmax == MIN_EXPR
	 && wi::ge_p (wi::to_wide (@1), bound, TYPE_SIGN (type)))
	|| (minmax == MAX_EXPR
	    && wi::le_p (wi::to_wide (@1), bound, TYPE_SIGN (type))))
    @3)))))
#endif