| /* Schedule GIMPLE vector statements. | |
| Copyright (C) 2020-2026 Free Software Foundation, Inc. | |
| This file is part of GCC. | |
| GCC is free software; you can redistribute it and/or modify it | |
| under the terms of the GNU General Public License as published by the | |
| Free Software Foundation; either version 3, or (at your option) any | |
| later version. | |
| GCC is distributed in the hope that it will be useful, but WITHOUT | |
| ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| for more details. | |
| You should have received a copy of the GNU General Public License | |
| along with GCC; see the file COPYING3. If not see | |
| <http://www.gnu.org/licenses/>. */ | |
| #include "config.h" | |
| #include "system.h" | |
| #include "coretypes.h" | |
| #include "backend.h" | |
| #include "rtl.h" | |
| #include "tree.h" | |
| #include "gimple.h" | |
| #include "tree-pass.h" | |
| #include "ssa.h" | |
| #include "expmed.h" | |
| #include "optabs-tree.h" | |
| #include "tree-eh.h" | |
| #include "gimple-iterator.h" | |
| #include "gimplify-me.h" | |
| #include "gimplify.h" | |
| #include "tree-cfg.h" | |
| #include "bitmap.h" | |
| #include "tree-ssa-dce.h" | |
| #include "memmodel.h" | |
| #include "optabs.h" | |
| #include "gimple-fold.h" | |
| #include "internal-fn.h" | |
| #include "fold-const.h" | |
| #include "gimple-range.h" | |
| /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to | |
| internal function based on vector type of selected expansion. | |
| For vec_set: | |
| VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D); | |
| => | |
| _7 = u; | |
| _8 = .VEC_SET (_7, i_4(D), _1); | |
| u = _8; | |
| For vec_extract: | |
| _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)]; | |
| => | |
| _4 = vD.2208; | |
| _3 = .VEC_EXTRACT (_4, idx_2(D)); */ | |
| static bool | |
| gimple_expand_vec_set_extract_expr (struct function *fun, | |
| gimple_stmt_iterator *gsi) | |
| { | |
| gcall *new_stmt = NULL; | |
| gassign *ass_stmt = NULL; | |
| bool cfg_changed = false; | |
| /* Only consider code == GIMPLE_ASSIGN. */ | |
| gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi)); | |
| if (!stmt) | |
| return false; | |
| bool is_extract = false; | |
| tree lhs = gimple_assign_lhs (stmt); | |
| tree rhs = gimple_assign_rhs1 (stmt); | |
| tree val, ref; | |
| if (TREE_CODE (lhs) == ARRAY_REF) | |
| { | |
| /* Assume it is a vec_set. */ | |
| val = rhs; | |
| ref = lhs; | |
| } | |
| else if (TREE_CODE (rhs) == ARRAY_REF) | |
| { | |
| /* vec_extract. */ | |
| is_extract = true; | |
| val = lhs; | |
| ref = rhs; | |
| } | |
| else | |
| return false; | |
| tree op0 = TREE_OPERAND (ref, 0); | |
| if (TREE_CODE (op0) == VIEW_CONVERT_EXPR | |
| && DECL_P (TREE_OPERAND (op0, 0)) | |
| && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0))) | |
| && (TYPE_MODE (TREE_TYPE (ref)) | |
| == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))) | |
| { | |
| tree pos = TREE_OPERAND (ref, 1); | |
| tree view_op0 = TREE_OPERAND (op0, 0); | |
| tree idx = TREE_OPERAND (ref, 1); | |
| // if index is a constant, then check the bounds | |
| poly_uint64 idx_poly; | |
| if (poly_int_tree_p (idx, &idx_poly)) | |
| { | |
| poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (view_op0)); | |
| if (known_ge (idx_poly, nelts)) | |
| return false; | |
| } | |
| else if (poly_int_tree_p (idx)) | |
| // if idx doesn't fit into poly_uint64, but is constant, it | |
| // must be out of bounds | |
| return false; | |
| machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0)); | |
| machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref)); | |
| if ((auto_var_in_fn_p (view_op0, fun->decl) | |
| || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0))) | |
| && !TREE_ADDRESSABLE (view_op0) | |
| && ((!is_extract && can_vec_set_var_idx_p (outermode)) | |
| || (is_extract | |
| && can_vec_extract_var_idx_p (outermode, extract_mode)))) | |
| { | |
| location_t loc = gimple_location (stmt); | |
| tree var_src = make_ssa_name (TREE_TYPE (view_op0)); | |
| ass_stmt = gimple_build_assign (var_src, view_op0); | |
| gimple_set_vuse (ass_stmt, gimple_vuse (stmt)); | |
| gimple_set_location (ass_stmt, loc); | |
| gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); | |
| if (!is_extract) | |
| { | |
| tree var_dst = make_ssa_name (TREE_TYPE (view_op0)); | |
| new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src, | |
| val, pos); | |
| gimple_call_set_lhs (new_stmt, var_dst); | |
| gimple_set_location (new_stmt, loc); | |
| gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT); | |
| ass_stmt = gimple_build_assign (view_op0, var_dst); | |
| gimple_set_location (ass_stmt, loc); | |
| gimple_move_vops (ass_stmt, stmt); | |
| gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); | |
| basic_block bb = gimple_bb (stmt); | |
| if (gsi_remove (gsi, true) | |
| && gimple_purge_dead_eh_edges (bb)) | |
| cfg_changed = true; | |
| *gsi = gsi_for_stmt (ass_stmt); | |
| } | |
| else | |
| { | |
| new_stmt | |
| = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos); | |
| gimple_call_set_lhs (new_stmt, lhs); | |
| gsi_replace (gsi, new_stmt, true); | |
| cfg_changed = true; | |
| } | |
| } | |
| } | |
| return cfg_changed; | |
| } | |
| /* Expand all VEC_COND_EXPR gimple assignments into calls to internal | |
| function based on type of selected expansion. */ | |
| static gimple * | |
| gimple_expand_vec_cond_expr (gimple_stmt_iterator *gsi) | |
| { | |
| tree lhs, op0a = NULL_TREE; | |
| enum tree_code code; | |
| enum tree_code tcode; | |
| /* Only consider code == GIMPLE_ASSIGN. */ | |
| gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi)); | |
| if (!stmt) | |
| return NULL; | |
| code = gimple_assign_rhs_code (stmt); | |
| if (code != VEC_COND_EXPR) | |
| return NULL; | |
| tree op0 = gimple_assign_rhs1 (stmt); | |
| tree op1 = gimple_assign_rhs2 (stmt); | |
| tree op2 = gimple_assign_rhs3 (stmt); | |
| lhs = gimple_assign_lhs (stmt); | |
| machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); | |
| /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations. | |
| Those can end up generated by folding and at least for integer mode masks | |
| we cannot expect vcond expanders to exist. We lower a ? b : c | |
| to (b & a) | (c & ~a). */ | |
| if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)) | |
| && !VECTOR_MODE_P (mode)) | |
| { | |
| gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1))); | |
| gimple_seq stmts = NULL; | |
| tree type = TREE_TYPE (lhs); | |
| location_t loc = gimple_location (stmt); | |
| tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0); | |
| tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0); | |
| tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1); | |
| tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2); | |
| gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); | |
| return gimple_build_assign (lhs, tem3); | |
| } | |
| bool can_compute_op0 = true; | |
| gcc_assert (!COMPARISON_CLASS_P (op0)); | |
| if (TREE_CODE (op0) == SSA_NAME) | |
| { | |
| gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0)); | |
| if (def_stmt) | |
| { | |
| tcode = gimple_assign_rhs_code (def_stmt); | |
| op0a = gimple_assign_rhs1 (def_stmt); | |
| tree op0_type = TREE_TYPE (op0); | |
| tree op0a_type = TREE_TYPE (op0a); | |
| if (TREE_CODE_CLASS (tcode) == tcc_comparison) | |
| can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type, | |
| tcode); | |
| gcc_assert (can_compute_op0); | |
| if (can_compute_op0 | |
| && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0))) | |
| { | |
| /* Assuming c = x CMP y. */ | |
| bool op1_minus_onep = integer_minus_onep (op1); | |
| bool op2_zerop = integer_zerop (op2); | |
| tree vtype = TREE_TYPE (lhs); | |
| machine_mode vmode = TYPE_MODE (vtype); | |
| /* Try to fold r = c ? -1 : 0 to r = c. */ | |
| if (op1_minus_onep && op2_zerop) | |
| { | |
| tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0); | |
| return gimple_build_assign (lhs, conv_op); | |
| } | |
| /* Try to fold r = c ? -1 : z to r = c | z, or | |
| r = c ? c : z. */ | |
| if (op1_minus_onep) | |
| { | |
| tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0); | |
| tree new_op1 = make_ssa_name (vtype); | |
| gassign *new_stmt = gimple_build_assign (new_op1, conv_op); | |
| gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT); | |
| if (optab_handler (ior_optab, vmode) != CODE_FOR_nothing) | |
| /* r = c | z */ | |
| return gimple_build_assign (lhs, BIT_IOR_EXPR, new_op1, | |
| op2); | |
| /* r = c ? c : z */ | |
| op1 = new_op1; | |
| } | |
| /* Try to fold r = c ? z : 0 to r = c & z, or | |
| r = c ? z : c. */ | |
| else if (op2_zerop) | |
| { | |
| tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0); | |
| tree new_op2 = make_ssa_name (vtype); | |
| gassign *new_stmt = gimple_build_assign (new_op2, conv_op); | |
| gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT); | |
| if (optab_handler (and_optab, vmode) != CODE_FOR_nothing) | |
| /* r = c | z */ | |
| return gimple_build_assign (lhs, BIT_AND_EXPR, new_op2, | |
| op1); | |
| /* r = c ? z : c */ | |
| op2 = new_op2; | |
| } | |
| bool op1_zerop = integer_zerop (op1); | |
| bool op2_minus_onep = integer_minus_onep (op2); | |
| /* Try to fold r = c ? 0 : z to r = .BIT_ANDN (z, c). */ | |
| if (op1_zerop | |
| && (direct_internal_fn_supported_p (IFN_BIT_ANDN, vtype, | |
| OPTIMIZE_FOR_BOTH))) | |
| { | |
| tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0); | |
| tree new_op = make_ssa_name (vtype); | |
| gassign *new_stmt = gimple_build_assign (new_op, conv_op); | |
| gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT); | |
| return gimple_build_call_internal (IFN_BIT_ANDN, 2, op2, | |
| new_op); | |
| } | |
| /* Try to fold r = c ? z : -1 to r = .BIT_IORN (z, c). */ | |
| else if (op2_minus_onep | |
| && (direct_internal_fn_supported_p (IFN_BIT_IORN, vtype, | |
| OPTIMIZE_FOR_BOTH))) | |
| { | |
| tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0); | |
| tree new_op = make_ssa_name (vtype); | |
| gassign *new_stmt = gimple_build_assign (new_op, conv_op); | |
| gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT); | |
| return gimple_build_call_internal (IFN_BIT_IORN, 2, op1, | |
| new_op); | |
| } | |
| } | |
| } | |
| } | |
| gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); | |
| gcc_assert (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) | |
| != CODE_FOR_nothing); | |
| return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2); | |
| } | |
| /* Duplicate COND_EXPR condition defs of STMT located in BB when they are | |
| comparisons so RTL expansion with the help of TER | |
| can perform better if conversion. */ | |
| static void | |
| maybe_duplicate_comparison (gassign *stmt, basic_block bb) | |
| { | |
| imm_use_iterator imm_iter; | |
| use_operand_p use_p; | |
| auto_vec<gassign *, 4> cond_exprs; | |
| tree lhs = gimple_assign_lhs (stmt); | |
| unsigned cnt = 0; | |
| /* This is should not be used for -O0 nor it is not useful | |
| when ter is turned off. */ | |
| if (!optimize || !flag_tree_ter) | |
| return; | |
| FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs) | |
| { | |
| if (is_gimple_debug (USE_STMT (use_p))) | |
| continue; | |
| cnt++; | |
| /* Add the use statement if it was a cond_expr. */ | |
| if (gimple_bb (USE_STMT (use_p)) == bb | |
| && is_gimple_assign (USE_STMT (use_p)) | |
| && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR | |
| && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use) | |
| cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p))); | |
| } | |
| /* If the comparison has 0 or 1 uses, no reason to do anything. */ | |
| if (cnt <= 1) | |
| return; | |
| /* If we only use the expression inside cond_exprs in that BB, we don't | |
| need to duplicate for one of them so pop the top. */ | |
| if (cond_exprs.length () == cnt) | |
| cond_exprs.pop(); | |
| while (!cond_exprs.is_empty()) | |
| { | |
| auto old_top = cond_exprs.pop(); | |
| gassign *copy = as_a <gassign *> (gimple_copy (stmt)); | |
| tree new_def = duplicate_ssa_name (lhs, copy); | |
| gimple_assign_set_lhs (copy, new_def); | |
| auto gsi2 = gsi_for_stmt (old_top); | |
| gsi_insert_before (&gsi2, copy, GSI_SAME_STMT); | |
| gimple_assign_set_rhs1 (old_top, new_def); | |
| update_stmt (old_top); | |
| } | |
| } | |
| /* match.pd function to match atomic_bit_test_and pattern which | |
| has nop_convert: | |
| _1 = __atomic_fetch_or_4 (&v, 1, 0); | |
| _2 = (int) _1; | |
| _5 = _2 & 1; | |
| */ | |
| extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *, | |
| tree (*) (tree)); | |
| extern bool gimple_nop_convert (tree, tree*, tree (*) (tree)); | |
| namespace { | |
| const pass_data pass_data_gimple_isel = | |
| { | |
| GIMPLE_PASS, /* type */ | |
| "isel", /* name */ | |
| OPTGROUP_VEC, /* optinfo_flags */ | |
| TV_NONE, /* tv_id */ | |
| PROP_cfg, /* properties_required */ | |
| 0, /* properties_provided */ | |
| 0, /* properties_destroyed */ | |
| 0, /* todo_flags_start */ | |
| TODO_update_ssa, /* todo_flags_finish */ | |
| }; | |
| class pass_gimple_isel : public gimple_opt_pass | |
| { | |
| public: | |
| pass_gimple_isel (gcc::context *ctxt) | |
| : gimple_opt_pass (pass_data_gimple_isel, ctxt) | |
| {} | |
| /* opt_pass methods: */ | |
| bool gate (function *) final override | |
| { | |
| return true; | |
| } | |
| unsigned int execute (function *fun) final override; | |
| }; // class pass_gimple_isel | |
| /* Convert | |
| _1 = __atomic_fetch_or_* (ptr_6, 1, _3); | |
| _7 = ~_1; | |
| _5 = (_Bool) _7; | |
| to | |
| _1 = __atomic_fetch_or_* (ptr_6, 1, _3); | |
| _8 = _1 & 1; | |
| _5 = _8 == 0; | |
| and convert | |
| _1 = __atomic_fetch_and_* (ptr_6, ~1, _3); | |
| _7 = ~_1; | |
| _4 = (_Bool) _7; | |
| to | |
| _1 = __atomic_fetch_and_* (ptr_6, ~1, _3); | |
| _8 = _1 & 1; | |
| _4 = (_Bool) _8; | |
| USE_STMT is the gimplt statement which uses the return value of | |
| __atomic_fetch_or_*. LHS is the return value of __atomic_fetch_or_*. | |
| MASK is the mask passed to __atomic_fetch_or_*. | |
| */ | |
| static gimple * | |
| convert_atomic_bit_not (enum internal_fn fn, gimple *use_stmt, | |
| tree lhs, tree mask) | |
| { | |
| tree and_mask; | |
| if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET) | |
| { | |
| /* MASK must be ~1. */ | |
| if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), | |
| ~HOST_WIDE_INT_1), mask, 0)) | |
| return nullptr; | |
| and_mask = build_int_cst (TREE_TYPE (lhs), 1); | |
| } | |
| else | |
| { | |
| /* MASK must be 1. */ | |
| if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), 1), mask, 0)) | |
| return nullptr; | |
| and_mask = mask; | |
| } | |
| tree use_lhs = gimple_assign_lhs (use_stmt); | |
| use_operand_p use_p; | |
| gimple *use_not_stmt; | |
| if (!single_imm_use (use_lhs, &use_p, &use_not_stmt) | |
| || !is_gimple_assign (use_not_stmt)) | |
| return nullptr; | |
| if (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_not_stmt))) | |
| return nullptr; | |
| tree use_not_lhs = gimple_assign_lhs (use_not_stmt); | |
| if (TREE_CODE (TREE_TYPE (use_not_lhs)) != BOOLEAN_TYPE) | |
| return nullptr; | |
| gimple_stmt_iterator gsi; | |
| tree var = make_ssa_name (TREE_TYPE (lhs)); | |
| /* use_stmt need to be removed after use_nop_stmt, | |
| so use_lhs can be released. */ | |
| gimple *use_stmt_removal = use_stmt; | |
| use_stmt = gimple_build_assign (var, BIT_AND_EXPR, lhs, and_mask); | |
| gsi = gsi_for_stmt (use_not_stmt); | |
| gsi_insert_before (&gsi, use_stmt, GSI_NEW_STMT); | |
| lhs = gimple_assign_lhs (use_not_stmt); | |
| gimple *g = gimple_build_assign (lhs, EQ_EXPR, var, | |
| build_zero_cst (TREE_TYPE (mask))); | |
| gsi_insert_after (&gsi, g, GSI_NEW_STMT); | |
| gsi = gsi_for_stmt (use_not_stmt); | |
| gsi_remove (&gsi, true); | |
| gsi = gsi_for_stmt (use_stmt_removal); | |
| gsi_remove (&gsi, true); | |
| return use_stmt; | |
| } | |
| /* Optimize | |
| mask_2 = 1 << cnt_1; | |
| _4 = __atomic_fetch_or_* (ptr_6, mask_2, _3); | |
| _5 = _4 & mask_2; | |
| to | |
| _4 = .ATOMIC_BIT_TEST_AND_SET (ptr_6, cnt_1, 0, _3); | |
| _5 = _4; | |
| If _5 is only used in _5 != 0 or _5 == 0 comparisons, 1 | |
| is passed instead of 0, and the builtin just returns a zero | |
| or 1 value instead of the actual bit. | |
| Similarly for __sync_fetch_and_or_* (without the ", _3" part | |
| in there), and/or if mask_2 is a power of 2 constant. | |
| Similarly for xor instead of or, use ATOMIC_BIT_TEST_AND_COMPLEMENT | |
| in that case. And similarly for and instead of or, except that | |
| the second argument to the builtin needs to be one's complement | |
| of the mask instead of mask. */ | |
| static bool | |
| optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip, | |
| enum internal_fn fn, bool has_model_arg, | |
| bool after) | |
| { | |
| gimple *call = gsi_stmt (*gsip); | |
| tree lhs = gimple_call_lhs (call); | |
| use_operand_p use_p; | |
| gimple *use_stmt; | |
| tree mask; | |
| optab optab; | |
| if (!flag_inline_atomics | |
| || optimize_debug | |
| || !gimple_call_builtin_p (call, BUILT_IN_NORMAL) | |
| || !lhs | |
| || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs) | |
| || !single_imm_use (lhs, &use_p, &use_stmt) | |
| || !is_gimple_assign (use_stmt) | |
| || !gimple_vdef (call)) | |
| return false; | |
| switch (fn) | |
| { | |
| case IFN_ATOMIC_BIT_TEST_AND_SET: | |
| optab = atomic_bit_test_and_set_optab; | |
| break; | |
| case IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT: | |
| optab = atomic_bit_test_and_complement_optab; | |
| break; | |
| case IFN_ATOMIC_BIT_TEST_AND_RESET: | |
| optab = atomic_bit_test_and_reset_optab; | |
| break; | |
| default: | |
| return false; | |
| } | |
| tree bit = nullptr; | |
| mask = gimple_call_arg (call, 1); | |
| tree_code rhs_code = gimple_assign_rhs_code (use_stmt); | |
| if (rhs_code != BIT_AND_EXPR) | |
| { | |
| if (rhs_code != NOP_EXPR && rhs_code != BIT_NOT_EXPR) | |
| return false; | |
| tree use_lhs = gimple_assign_lhs (use_stmt); | |
| if (TREE_CODE (use_lhs) == SSA_NAME | |
| && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)) | |
| return false; | |
| tree use_rhs = gimple_assign_rhs1 (use_stmt); | |
| if (lhs != use_rhs) | |
| return false; | |
| if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))) | |
| == CODE_FOR_nothing) | |
| return false; | |
| gimple *g; | |
| gimple_stmt_iterator gsi; | |
| tree var; | |
| int ibit = -1; | |
| if (rhs_code == BIT_NOT_EXPR) | |
| { | |
| g = convert_atomic_bit_not (fn, use_stmt, lhs, mask); | |
| if (!g) | |
| return false; | |
| use_stmt = g; | |
| ibit = 0; | |
| } | |
| else if (TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE) | |
| { | |
| tree and_mask; | |
| if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET) | |
| { | |
| /* MASK must be ~1. */ | |
| if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), | |
| ~HOST_WIDE_INT_1), | |
| mask, 0)) | |
| return false; | |
| /* Convert | |
| _1 = __atomic_fetch_and_* (ptr_6, ~1, _3); | |
| _4 = (_Bool) _1; | |
| to | |
| _1 = __atomic_fetch_and_* (ptr_6, ~1, _3); | |
| _5 = _1 & 1; | |
| _4 = (_Bool) _5; | |
| */ | |
| and_mask = build_int_cst (TREE_TYPE (lhs), 1); | |
| } | |
| else | |
| { | |
| and_mask = build_int_cst (TREE_TYPE (lhs), 1); | |
| if (!operand_equal_p (and_mask, mask, 0)) | |
| return false; | |
| /* Convert | |
| _1 = __atomic_fetch_or_* (ptr_6, 1, _3); | |
| _4 = (_Bool) _1; | |
| to | |
| _1 = __atomic_fetch_or_* (ptr_6, 1, _3); | |
| _5 = _1 & 1; | |
| _4 = (_Bool) _5; | |
| */ | |
| } | |
| var = make_ssa_name (TREE_TYPE (use_rhs)); | |
| replace_uses_by (use_rhs, var); | |
| g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs, | |
| and_mask); | |
| gsi = gsi_for_stmt (use_stmt); | |
| gsi_insert_before (&gsi, g, GSI_NEW_STMT); | |
| use_stmt = g; | |
| ibit = 0; | |
| } | |
| else if (TYPE_PRECISION (TREE_TYPE (use_lhs)) | |
| <= TYPE_PRECISION (TREE_TYPE (use_rhs))) | |
| { | |
| gimple *use_nop_stmt; | |
| if (!single_imm_use (use_lhs, &use_p, &use_nop_stmt) | |
| || (!is_gimple_assign (use_nop_stmt) | |
| && gimple_code (use_nop_stmt) != GIMPLE_COND)) | |
| return false; | |
| /* Handle both | |
| _4 = _5 < 0; | |
| and | |
| if (_5 < 0) | |
| */ | |
| tree use_nop_lhs = nullptr; | |
| rhs_code = ERROR_MARK; | |
| if (is_gimple_assign (use_nop_stmt)) | |
| { | |
| use_nop_lhs = gimple_assign_lhs (use_nop_stmt); | |
| rhs_code = gimple_assign_rhs_code (use_nop_stmt); | |
| } | |
| if (!use_nop_lhs || rhs_code != BIT_AND_EXPR) | |
| { | |
| /* Also handle | |
| if (_5 < 0) | |
| */ | |
| if (use_nop_lhs | |
| && TREE_CODE (use_nop_lhs) == SSA_NAME | |
| && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_nop_lhs)) | |
| return false; | |
| if (use_nop_lhs && rhs_code == BIT_NOT_EXPR) | |
| { | |
| /* Handle | |
| _7 = ~_2; | |
| */ | |
| g = convert_atomic_bit_not (fn, use_nop_stmt, lhs, | |
| mask); | |
| if (!g) | |
| return false; | |
| /* Convert | |
| _1 = __atomic_fetch_or_4 (ptr_6, 1, _3); | |
| _2 = (int) _1; | |
| _7 = ~_2; | |
| _5 = (_Bool) _7; | |
| to | |
| _1 = __atomic_fetch_or_4 (ptr_6, 1, _3); | |
| _8 = _1 & 1; | |
| _5 = _8 == 0; | |
| and convert | |
| _1 = __atomic_fetch_and_4 (ptr_6, ~1, _3); | |
| _2 = (int) _1; | |
| _7 = ~_2; | |
| _5 = (_Bool) _7; | |
| to | |
| _1 = __atomic_fetch_and_4 (ptr_6, ~1, _3); | |
| _8 = _1 & 1; | |
| _5 = _8 == 0; | |
| */ | |
| gsi = gsi_for_stmt (use_stmt); | |
| gsi_remove (&gsi, true); | |
| use_stmt = g; | |
| ibit = 0; | |
| } | |
| else | |
| { | |
| tree cmp_rhs1, cmp_rhs2; | |
| if (use_nop_lhs) | |
| { | |
| /* Handle | |
| _4 = _5 < 0; | |
| */ | |
| if (TREE_CODE (TREE_TYPE (use_nop_lhs)) | |
| != BOOLEAN_TYPE) | |
| return false; | |
| cmp_rhs1 = gimple_assign_rhs1 (use_nop_stmt); | |
| cmp_rhs2 = gimple_assign_rhs2 (use_nop_stmt); | |
| } | |
| else | |
| { | |
| /* Handle | |
| if (_5 < 0) | |
| */ | |
| rhs_code = gimple_cond_code (use_nop_stmt); | |
| cmp_rhs1 = gimple_cond_lhs (use_nop_stmt); | |
| cmp_rhs2 = gimple_cond_rhs (use_nop_stmt); | |
| } | |
| if (rhs_code != GE_EXPR && rhs_code != LT_EXPR) | |
| return false; | |
| if (TYPE_UNSIGNED (TREE_TYPE (use_lhs))) | |
| return false; | |
| if (use_lhs != cmp_rhs1) | |
| return false; | |
| if (!integer_zerop (cmp_rhs2)) | |
| return false; | |
| tree and_mask; | |
| ibit = TYPE_PRECISION (TREE_TYPE (use_lhs)) - 1; | |
| unsigned HOST_WIDE_INT highest | |
| = HOST_WIDE_INT_1U << ibit; | |
| if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET) | |
| { | |
| /* Get the signed maximum of the USE_RHS type. */ | |
| and_mask = build_int_cst (TREE_TYPE (use_rhs), | |
| highest - 1); | |
| if (!operand_equal_p (and_mask, mask, 0)) | |
| return false; | |
| /* Convert | |
| _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3); | |
| _5 = (signed int) _1; | |
| _4 = _5 < 0 or _5 >= 0; | |
| to | |
| _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3); | |
| _6 = _1 & 0x80000000; | |
| _4 = _6 != 0 or _6 == 0; | |
| and convert | |
| _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3); | |
| _5 = (signed int) _1; | |
| if (_5 < 0 or _5 >= 0) | |
| to | |
| _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3); | |
| _6 = _1 & 0x80000000; | |
| if (_6 != 0 or _6 == 0) | |
| */ | |
| and_mask = build_int_cst (TREE_TYPE (use_rhs), | |
| highest); | |
| } | |
| else | |
| { | |
| /* Get the signed minimum of the USE_RHS type. */ | |
| and_mask = build_int_cst (TREE_TYPE (use_rhs), | |
| highest); | |
| if (!operand_equal_p (and_mask, mask, 0)) | |
| return false; | |
| /* Convert | |
| _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3); | |
| _5 = (signed int) _1; | |
| _4 = _5 < 0 or _5 >= 0; | |
| to | |
| _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3); | |
| _6 = _1 & 0x80000000; | |
| _4 = _6 != 0 or _6 == 0; | |
| and convert | |
| _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3); | |
| _5 = (signed int) _1; | |
| if (_5 < 0 or _5 >= 0) | |
| to | |
| _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3); | |
| _6 = _1 & 0x80000000; | |
| if (_6 != 0 or _6 == 0) | |
| */ | |
| } | |
| var = make_ssa_name (TREE_TYPE (use_rhs)); | |
| gimple *use_stmt_removal = use_stmt; | |
| g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs, | |
| and_mask); | |
| gsi = gsi_for_stmt (use_nop_stmt); | |
| gsi_insert_before (&gsi, g, GSI_NEW_STMT); | |
| use_stmt = g; | |
| rhs_code = rhs_code == GE_EXPR ? EQ_EXPR : NE_EXPR; | |
| tree const_zero = build_zero_cst (TREE_TYPE (use_rhs)); | |
| if (use_nop_lhs) | |
| g = gimple_build_assign (use_nop_lhs, rhs_code, | |
| var, const_zero); | |
| else | |
| g = gimple_build_cond (rhs_code, var, const_zero, | |
| nullptr, nullptr); | |
| gsi_insert_after (&gsi, g, GSI_NEW_STMT); | |
| gsi = gsi_for_stmt (use_nop_stmt); | |
| gsi_remove (&gsi, true); | |
| gsi = gsi_for_stmt (use_stmt_removal); | |
| gsi_remove (&gsi, true); | |
| } | |
| } | |
| else | |
| { | |
| tree match_op[3]; | |
| gimple *g; | |
| if (!gimple_nop_atomic_bit_test_and_p (use_nop_lhs, | |
| &match_op[0], NULL) | |
| || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (match_op[2]) | |
| || !single_imm_use (match_op[2], &use_p, &g) | |
| || !is_gimple_assign (g)) | |
| return false; | |
| mask = match_op[0]; | |
| if (TREE_CODE (match_op[1]) == INTEGER_CST) | |
| { | |
| ibit = tree_log2 (match_op[1]); | |
| gcc_assert (ibit >= 0); | |
| } | |
| else | |
| { | |
| g = SSA_NAME_DEF_STMT (match_op[1]); | |
| gcc_assert (is_gimple_assign (g)); | |
| bit = gimple_assign_rhs2 (g); | |
| } | |
| /* Convert | |
| _1 = __atomic_fetch_or_4 (ptr_6, mask, _3); | |
| _2 = (int) _1; | |
| _5 = _2 & mask; | |
| to | |
| _1 = __atomic_fetch_or_4 (ptr_6, mask, _3); | |
| _6 = _1 & mask; | |
| _5 = (int) _6; | |
| and convert | |
| _1 = ~mask_7; | |
| _2 = (unsigned int) _1; | |
| _3 = __atomic_fetch_and_4 (ptr_6, _2, 0); | |
| _4 = (int) _3; | |
| _5 = _4 & mask_7; | |
| to | |
| _1 = __atomic_fetch_and_* (ptr_6, ~mask_7, _3); | |
| _12 = _3 & mask_7; | |
| _5 = (int) _12; | |
| and convert | |
| _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3); | |
| _2 = (short int) _1; | |
| _5 = _2 & mask; | |
| to | |
| _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3); | |
| _8 = _1 & mask; | |
| _5 = (short int) _8; | |
| */ | |
| gimple_seq stmts = NULL; | |
| match_op[1] = gimple_convert (&stmts, | |
| TREE_TYPE (use_rhs), | |
| match_op[1]); | |
| var = gimple_build (&stmts, BIT_AND_EXPR, | |
| TREE_TYPE (use_rhs), use_rhs, match_op[1]); | |
| gsi = gsi_for_stmt (use_stmt); | |
| gsi_remove (&gsi, true); | |
| release_defs (use_stmt); | |
| use_stmt = gimple_seq_last_stmt (stmts); | |
| gsi = gsi_for_stmt (use_nop_stmt); | |
| gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT); | |
| gimple_assign_set_rhs_with_ops (&gsi, CONVERT_EXPR, var); | |
| update_stmt (use_nop_stmt); | |
| } | |
| } | |
| else | |
| return false; | |
| if (!bit) | |
| { | |
| if (ibit < 0) | |
| gcc_unreachable (); | |
| bit = build_int_cst (TREE_TYPE (lhs), ibit); | |
| } | |
| } | |
| else if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))) | |
| == CODE_FOR_nothing) | |
| return false; | |
| tree use_lhs = gimple_assign_lhs (use_stmt); | |
| if (!use_lhs) | |
| return false; | |
| if (!bit) | |
| { | |
| if (TREE_CODE (mask) == INTEGER_CST) | |
| { | |
| if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET) | |
| mask = const_unop (BIT_NOT_EXPR, TREE_TYPE (mask), mask); | |
| mask = fold_convert (TREE_TYPE (lhs), mask); | |
| int ibit = tree_log2 (mask); | |
| if (ibit < 0) | |
| return false; | |
| bit = build_int_cst (TREE_TYPE (lhs), ibit); | |
| } | |
| else if (TREE_CODE (mask) == SSA_NAME) | |
| { | |
| gimple *g = SSA_NAME_DEF_STMT (mask); | |
| tree match_op; | |
| if (gimple_nop_convert (mask, &match_op, NULL)) | |
| { | |
| mask = match_op; | |
| if (TREE_CODE (mask) != SSA_NAME) | |
| return false; | |
| g = SSA_NAME_DEF_STMT (mask); | |
| } | |
| if (!is_gimple_assign (g)) | |
| return false; | |
| if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET) | |
| { | |
| if (gimple_assign_rhs_code (g) != BIT_NOT_EXPR) | |
| return false; | |
| mask = gimple_assign_rhs1 (g); | |
| if (TREE_CODE (mask) != SSA_NAME) | |
| return false; | |
| g = SSA_NAME_DEF_STMT (mask); | |
| } | |
| if (!is_gimple_assign (g) | |
| || gimple_assign_rhs_code (g) != LSHIFT_EXPR | |
| || !integer_onep (gimple_assign_rhs1 (g))) | |
| return false; | |
| bit = gimple_assign_rhs2 (g); | |
| } | |
| else | |
| return false; | |
| tree cmp_mask; | |
| if (gimple_assign_rhs1 (use_stmt) == lhs) | |
| cmp_mask = gimple_assign_rhs2 (use_stmt); | |
| else | |
| cmp_mask = gimple_assign_rhs1 (use_stmt); | |
| tree match_op; | |
| if (gimple_nop_convert (cmp_mask, &match_op, NULL)) | |
| cmp_mask = match_op; | |
| if (!operand_equal_p (cmp_mask, mask, 0)) | |
| return false; | |
| } | |
| bool use_bool = true; | |
| bool has_debug_uses = false; | |
| imm_use_iterator iter; | |
| gimple *g; | |
| if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)) | |
| use_bool = false; | |
| FOR_EACH_IMM_USE_STMT (g, iter, use_lhs) | |
| { | |
| enum tree_code code = ERROR_MARK; | |
| tree op0 = NULL_TREE, op1 = NULL_TREE; | |
| if (is_gimple_debug (g)) | |
| { | |
| has_debug_uses = true; | |
| continue; | |
| } | |
| else if (is_gimple_assign (g)) | |
| switch (gimple_assign_rhs_code (g)) | |
| { | |
| case COND_EXPR: | |
| op1 = gimple_assign_rhs1 (g); | |
| code = TREE_CODE (op1); | |
| if (TREE_CODE_CLASS (code) != tcc_comparison) | |
| break; | |
| op0 = TREE_OPERAND (op1, 0); | |
| op1 = TREE_OPERAND (op1, 1); | |
| break; | |
| case EQ_EXPR: | |
| case NE_EXPR: | |
| code = gimple_assign_rhs_code (g); | |
| op0 = gimple_assign_rhs1 (g); | |
| op1 = gimple_assign_rhs2 (g); | |
| break; | |
| default: | |
| break; | |
| } | |
| else if (gimple_code (g) == GIMPLE_COND) | |
| { | |
| code = gimple_cond_code (g); | |
| op0 = gimple_cond_lhs (g); | |
| op1 = gimple_cond_rhs (g); | |
| } | |
| if ((code == EQ_EXPR || code == NE_EXPR) | |
| && op0 == use_lhs | |
| && integer_zerop (op1)) | |
| { | |
| use_operand_p use_p; | |
| int n = 0; | |
| FOR_EACH_IMM_USE_ON_STMT (use_p, iter) | |
| n++; | |
| if (n == 1) | |
| continue; | |
| } | |
| use_bool = false; | |
| break; | |
| } | |
| tree new_lhs = make_ssa_name (TREE_TYPE (lhs)); | |
| tree flag = build_int_cst (TREE_TYPE (lhs), use_bool); | |
| if (has_model_arg) | |
| g = gimple_build_call_internal (fn, 5, gimple_call_arg (call, 0), | |
| bit, flag, gimple_call_arg (call, 2), | |
| gimple_call_fn (call)); | |
| else | |
| g = gimple_build_call_internal (fn, 4, gimple_call_arg (call, 0), | |
| bit, flag, gimple_call_fn (call)); | |
| gimple_call_set_lhs (g, new_lhs); | |
| gimple_set_location (g, gimple_location (call)); | |
| gimple_move_vops (g, call); | |
| bool throws = stmt_can_throw_internal (cfun, call); | |
| gimple_call_set_nothrow (as_a <gcall *> (g), | |
| gimple_call_nothrow_p (as_a <gcall *> (call))); | |
| gimple_stmt_iterator gsi = *gsip; | |
| gsi_insert_after (&gsi, g, GSI_NEW_STMT); | |
| edge e = NULL; | |
| if (throws) | |
| { | |
| maybe_clean_or_replace_eh_stmt (call, g); | |
| if (after || (use_bool && has_debug_uses)) | |
| e = find_fallthru_edge (gsi_bb (gsi)->succs); | |
| } | |
| if (after) | |
| { | |
| /* The internal function returns the value of the specified bit | |
| before the atomic operation. If we are interested in the value | |
| of the specified bit after the atomic operation (makes only sense | |
| for xor, otherwise the bit content is compile time known), | |
| we need to invert the bit. */ | |
| tree mask_convert = mask; | |
| gimple_seq stmts = NULL; | |
| if (!use_bool) | |
| mask_convert = gimple_convert (&stmts, TREE_TYPE (lhs), mask); | |
| new_lhs = gimple_build (&stmts, BIT_XOR_EXPR, TREE_TYPE (lhs), new_lhs, | |
| use_bool ? build_int_cst (TREE_TYPE (lhs), 1) | |
| : mask_convert); | |
| if (throws) | |
| { | |
| gsi_insert_seq_on_edge_immediate (e, stmts); | |
| gsi = gsi_for_stmt (gimple_seq_last (stmts)); | |
| } | |
| else | |
| gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT); | |
| } | |
| if (use_bool && has_debug_uses) | |
| { | |
| tree temp = NULL_TREE; | |
| if (!throws || after || single_pred_p (e->dest)) | |
| { | |
| temp = build_debug_expr_decl (TREE_TYPE (lhs)); | |
| tree t = build2 (LSHIFT_EXPR, TREE_TYPE (lhs), new_lhs, bit); | |
| g = gimple_build_debug_bind (temp, t, g); | |
| if (throws && !after) | |
| { | |
| gsi = gsi_after_labels (e->dest); | |
| gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
| } | |
| else | |
| gsi_insert_after (&gsi, g, GSI_NEW_STMT); | |
| } | |
| FOR_EACH_IMM_USE_STMT (g, iter, use_lhs) | |
| if (is_gimple_debug (g)) | |
| { | |
| use_operand_p use_p; | |
| if (temp == NULL_TREE) | |
| gimple_debug_bind_reset_value (g); | |
| else | |
| FOR_EACH_IMM_USE_ON_STMT (use_p, iter) | |
| SET_USE (use_p, temp); | |
| update_stmt (g); | |
| } | |
| } | |
| SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_lhs) | |
| = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs); | |
| replace_uses_by (use_lhs, new_lhs); | |
| gsi = gsi_for_stmt (use_stmt); | |
| gsi_remove (&gsi, true); | |
| release_defs (use_stmt); | |
| gsi_remove (gsip, true); | |
| release_ssa_name (lhs); | |
| return true; | |
| } | |
| /* Optimize | |
| _4 = __atomic_add_fetch_* (ptr_6, arg_2, _3); | |
| _5 = _4 == 0; | |
| to | |
| _4 = .ATOMIC_ADD_FETCH_CMP_0 (EQ_EXPR, ptr_6, arg_2, _3); | |
| _5 = _4; | |
| Similarly for __sync_add_and_fetch_* (without the ", _3" part | |
| in there). */ | |
| static bool | |
| optimize_atomic_op_fetch_cmp_0 (gimple_stmt_iterator *gsip, | |
| enum internal_fn fn, bool has_model_arg) | |
| { | |
| gimple *call = gsi_stmt (*gsip); | |
| tree lhs = gimple_call_lhs (call); | |
| use_operand_p use_p; | |
| gimple *use_stmt; | |
| if (!flag_inline_atomics | |
| || !gimple_call_builtin_p (call, BUILT_IN_NORMAL) | |
| || !lhs | |
| || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs) | |
| || !single_imm_use (lhs, &use_p, &use_stmt) | |
| || !gimple_vdef (call)) | |
| return false; | |
| optab optab; | |
| switch (fn) | |
| { | |
| case IFN_ATOMIC_ADD_FETCH_CMP_0: | |
| optab = atomic_add_fetch_cmp_0_optab; | |
| break; | |
| case IFN_ATOMIC_SUB_FETCH_CMP_0: | |
| optab = atomic_sub_fetch_cmp_0_optab; | |
| break; | |
| case IFN_ATOMIC_AND_FETCH_CMP_0: | |
| optab = atomic_and_fetch_cmp_0_optab; | |
| break; | |
| case IFN_ATOMIC_OR_FETCH_CMP_0: | |
| optab = atomic_or_fetch_cmp_0_optab; | |
| break; | |
| case IFN_ATOMIC_XOR_FETCH_CMP_0: | |
| optab = atomic_xor_fetch_cmp_0_optab; | |
| break; | |
| default: | |
| return false; | |
| } | |
| if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))) | |
| == CODE_FOR_nothing) | |
| return false; | |
| tree use_lhs = lhs; | |
| if (gimple_assign_cast_p (use_stmt)) | |
| { | |
| use_lhs = gimple_assign_lhs (use_stmt); | |
| if (!tree_nop_conversion_p (TREE_TYPE (use_lhs), TREE_TYPE (lhs)) | |
| || (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs)) | |
| && !POINTER_TYPE_P (TREE_TYPE (use_lhs))) | |
| || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs) | |
| || !single_imm_use (use_lhs, &use_p, &use_stmt)) | |
| return false; | |
| } | |
| enum tree_code code = ERROR_MARK; | |
| tree op0 = NULL_TREE, op1 = NULL_TREE; | |
| if (is_gimple_assign (use_stmt)) | |
| switch (gimple_assign_rhs_code (use_stmt)) | |
| { | |
| case COND_EXPR: | |
| op1 = gimple_assign_rhs1 (use_stmt); | |
| code = TREE_CODE (op1); | |
| if (TREE_CODE_CLASS (code) == tcc_comparison) | |
| { | |
| op0 = TREE_OPERAND (op1, 0); | |
| op1 = TREE_OPERAND (op1, 1); | |
| } | |
| break; | |
| default: | |
| code = gimple_assign_rhs_code (use_stmt); | |
| if (TREE_CODE_CLASS (code) == tcc_comparison) | |
| { | |
| op0 = gimple_assign_rhs1 (use_stmt); | |
| op1 = gimple_assign_rhs2 (use_stmt); | |
| } | |
| break; | |
| } | |
| else if (gimple_code (use_stmt) == GIMPLE_COND) | |
| { | |
| code = gimple_cond_code (use_stmt); | |
| op0 = gimple_cond_lhs (use_stmt); | |
| op1 = gimple_cond_rhs (use_stmt); | |
| } | |
| switch (code) | |
| { | |
| case LT_EXPR: | |
| case LE_EXPR: | |
| case GT_EXPR: | |
| case GE_EXPR: | |
| if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs)) | |
| || TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE | |
| || TYPE_UNSIGNED (TREE_TYPE (use_lhs))) | |
| return false; | |
| /* FALLTHRU */ | |
| case EQ_EXPR: | |
| case NE_EXPR: | |
| if (op0 == use_lhs && integer_zerop (op1)) | |
| break; | |
| return false; | |
| default: | |
| return false; | |
| } | |
| int encoded; | |
| switch (code) | |
| { | |
| /* Use special encoding of the operation. We want to also | |
| encode the mode in the first argument and for neither EQ_EXPR | |
| etc. nor EQ etc. we can rely it will fit into QImode. */ | |
| case EQ_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_EQ; break; | |
| case NE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_NE; break; | |
| case LT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LT; break; | |
| case LE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LE; break; | |
| case GT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GT; break; | |
| case GE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GE; break; | |
| default: gcc_unreachable (); | |
| } | |
| tree new_lhs = make_ssa_name (boolean_type_node); | |
| gimple *g; | |
| tree flag = build_int_cst (TREE_TYPE (lhs), encoded); | |
| if (has_model_arg) | |
| g = gimple_build_call_internal (fn, 5, flag, | |
| gimple_call_arg (call, 0), | |
| gimple_call_arg (call, 1), | |
| gimple_call_arg (call, 2), | |
| gimple_call_fn (call)); | |
| else | |
| g = gimple_build_call_internal (fn, 4, flag, | |
| gimple_call_arg (call, 0), | |
| gimple_call_arg (call, 1), | |
| gimple_call_fn (call)); | |
| gimple_call_set_lhs (g, new_lhs); | |
| gimple_set_location (g, gimple_location (call)); | |
| gimple_move_vops (g, call); | |
| bool throws = stmt_can_throw_internal (cfun, call); | |
| gimple_call_set_nothrow (as_a <gcall *> (g), | |
| gimple_call_nothrow_p (as_a <gcall *> (call))); | |
| gimple_stmt_iterator gsi = *gsip; | |
| gsi_insert_after (&gsi, g, GSI_SAME_STMT); | |
| if (throws) | |
| maybe_clean_or_replace_eh_stmt (call, g); | |
| if (is_gimple_assign (use_stmt)) | |
| switch (gimple_assign_rhs_code (use_stmt)) | |
| { | |
| case COND_EXPR: | |
| gimple_assign_set_rhs1 (use_stmt, new_lhs); | |
| break; | |
| default: | |
| gsi = gsi_for_stmt (use_stmt); | |
| if (tree ulhs = gimple_assign_lhs (use_stmt)) | |
| if (useless_type_conversion_p (TREE_TYPE (ulhs), | |
| boolean_type_node)) | |
| { | |
| gimple_assign_set_rhs_with_ops (&gsi, SSA_NAME, new_lhs); | |
| break; | |
| } | |
| gimple_assign_set_rhs_with_ops (&gsi, NOP_EXPR, new_lhs); | |
| break; | |
| } | |
| else if (gimple_code (use_stmt) == GIMPLE_COND) | |
| { | |
| gcond *use_cond = as_a <gcond *> (use_stmt); | |
| gimple_cond_set_code (use_cond, NE_EXPR); | |
| gimple_cond_set_lhs (use_cond, new_lhs); | |
| gimple_cond_set_rhs (use_cond, boolean_false_node); | |
| } | |
| update_stmt (use_stmt); | |
| if (use_lhs != lhs) | |
| { | |
| gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (use_lhs)); | |
| gsi_remove (&gsi, true); | |
| release_ssa_name (use_lhs); | |
| } | |
| gsi_remove (gsip, true); | |
| release_ssa_name (lhs); | |
| return true; | |
| } | |
| /* Process builtin CALL located at GSI. | |
| Currently it is only fgr atomic functions optimizations from above. */ | |
| static void | |
| gimple_isel_builtin_call (gcall *call, gimple_stmt_iterator *gsi) | |
| { | |
| /* Don't handle these in non optimization mode or optimize debug mode. */ | |
| if (!optimize || optimize_debug) | |
| return; | |
| if (!gimple_call_builtin_p (call, BUILT_IN_NORMAL)) | |
| return; | |
| tree callee = gimple_call_fndecl (call); | |
| switch (DECL_FUNCTION_CODE (callee)) | |
| { | |
| #define CASE_ATOMIC(NAME) \ | |
| case BUILT_IN_##NAME##_1: \ | |
| case BUILT_IN_##NAME##_2: \ | |
| case BUILT_IN_##NAME##_4: \ | |
| case BUILT_IN_##NAME##_8: \ | |
| case BUILT_IN_##NAME##_16 | |
| #define CASE_ATOMIC_CMP0(ATOMIC, SYNC) \ | |
| CASE_ATOMIC(ATOMIC_##ATOMIC): \ | |
| optimize_atomic_op_fetch_cmp_0 (gsi, \ | |
| IFN_ATOMIC_##ATOMIC##_CMP_0, \ | |
| true); \ | |
| break; \ | |
| CASE_ATOMIC(SYNC_##SYNC): \ | |
| optimize_atomic_op_fetch_cmp_0 (gsi, \ | |
| IFN_ATOMIC_##ATOMIC##_CMP_0, \ | |
| false); \ | |
| break; | |
| CASE_ATOMIC_CMP0(ADD_FETCH, ADD_AND_FETCH) | |
| CASE_ATOMIC_CMP0(SUB_FETCH, SUB_AND_FETCH) | |
| CASE_ATOMIC_CMP0(AND_FETCH, AND_AND_FETCH) | |
| CASE_ATOMIC_CMP0(OR_FETCH, OR_AND_FETCH) | |
| #define CASE_ATOMIC_BIT_TEST_AND(ATOMIC, SYNC, FN, AFTER) \ | |
| CASE_ATOMIC(ATOMIC_##ATOMIC): \ | |
| optimize_atomic_bit_test_and (gsi, \ | |
| IFN_ATOMIC_BIT_TEST_AND_##FN, \ | |
| true, AFTER); \ | |
| break; \ | |
| CASE_ATOMIC(SYNC_##SYNC): \ | |
| optimize_atomic_bit_test_and (gsi, \ | |
| IFN_ATOMIC_BIT_TEST_AND_##FN, \ | |
| false, AFTER); \ | |
| break; | |
| CASE_ATOMIC_BIT_TEST_AND(FETCH_OR, FETCH_AND_OR, SET, false) | |
| CASE_ATOMIC_BIT_TEST_AND(FETCH_XOR, FETCH_AND_XOR, COMPLEMENT, false) | |
| CASE_ATOMIC_BIT_TEST_AND(FETCH_AND, FETCH_AND_AND, RESET, false) | |
| CASE_ATOMIC(ATOMIC_XOR_FETCH): | |
| if (optimize_atomic_bit_test_and | |
| (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, true, true)) | |
| break; | |
| optimize_atomic_op_fetch_cmp_0 (gsi, | |
| IFN_ATOMIC_XOR_FETCH_CMP_0, | |
| true); | |
| break; | |
| CASE_ATOMIC(SYNC_XOR_AND_FETCH): | |
| if (optimize_atomic_bit_test_and | |
| (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, false, true)) | |
| break; | |
| optimize_atomic_op_fetch_cmp_0 (gsi, | |
| IFN_ATOMIC_XOR_FETCH_CMP_0, | |
| false); | |
| break; | |
| default:; | |
| } | |
| } | |
| /* Iterate all gimple statements and perform pre RTL expansion | |
| GIMPLE massaging to improve instruction selection. */ | |
| unsigned int | |
| pass_gimple_isel::execute (struct function *fun) | |
| { | |
| gimple_stmt_iterator gsi; | |
| basic_block bb; | |
| bool cfg_changed = false; | |
| if (optimize) | |
| enable_ranger (fun); | |
| FOR_EACH_BB_FN (bb, fun) | |
| { | |
| for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) | |
| { | |
| /* Give the target first try at replacing the instruction. */ | |
| cfg_changed |= targetm.instruction_selection (fun, &gsi); | |
| /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function | |
| calls mapping to supported optabs. */ | |
| gimple *g = gimple_expand_vec_cond_expr (&gsi); | |
| if (g != NULL) | |
| { | |
| tree lhs = gimple_assign_lhs (gsi_stmt (gsi)); | |
| gimple_set_lhs (g, lhs); | |
| gsi_replace (&gsi, g, false); | |
| } | |
| /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */ | |
| cfg_changed |= gimple_expand_vec_set_extract_expr (fun, &gsi); | |
| if (gsi_end_p (gsi)) | |
| break; | |
| if (gcall *call = dyn_cast <gcall*>(*gsi)) | |
| { | |
| gimple_isel_builtin_call (call, &gsi); | |
| continue; | |
| } | |
| gassign *stmt = dyn_cast <gassign *> (*gsi); | |
| if (!stmt) | |
| continue; | |
| tree_code code = gimple_assign_rhs_code (stmt); | |
| if (TREE_CODE_CLASS (code) == tcc_comparison) | |
| maybe_duplicate_comparison (stmt, bb); | |
| } | |
| } | |
| if (optimize) | |
| disable_ranger (fun); | |
| return cfg_changed ? TODO_cleanup_cfg : 0; | |
| } | |
| } // anon namespace | |
| gimple_opt_pass * | |
| make_pass_gimple_isel (gcc::context *ctxt) | |
| { | |
| return new pass_gimple_isel (ctxt); | |
| } | |