| /* Statement Analysis and Transformation for Vectorization |
| Copyright (C) 2003-2017 Free Software Foundation, Inc. |
| Contributed by Dorit Naishlos <dorit@il.ibm.com> |
| and Ira Rosen <irar@il.ibm.com> |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it under |
| the terms of the GNU General Public License as published by the Free |
| Software Foundation; either version 3, or (at your option) any later |
| version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "target.h" |
| #include "rtl.h" |
| #include "tree.h" |
| #include "gimple.h" |
| #include "ssa.h" |
| #include "optabs-tree.h" |
| #include "insn-config.h" |
| #include "recog.h" /* FIXME: for insn_data */ |
| #include "cgraph.h" |
| #include "dumpfile.h" |
| #include "alias.h" |
| #include "fold-const.h" |
| #include "stor-layout.h" |
| #include "tree-eh.h" |
| #include "gimplify.h" |
| #include "gimple-iterator.h" |
| #include "gimplify-me.h" |
| #include "tree-cfg.h" |
| #include "tree-ssa-loop-manip.h" |
| #include "cfgloop.h" |
| #include "tree-ssa-loop.h" |
| #include "tree-scalar-evolution.h" |
| #include "tree-vectorizer.h" |
| #include "builtins.h" |
| #include "internal-fn.h" |
| |
| /* For lang_hooks.types.type_for_mode. */ |
| #include "langhooks.h" |
| |
| /* Says whether a statement is a load, a store of a vectorized statement |
| result, or a store of an invariant value. */ |
| enum vec_load_store_type { |
| VLS_LOAD, |
| VLS_STORE, |
| VLS_STORE_INVARIANT |
| }; |
| |
| /* Return the vectorized type for the given statement. */ |
| |
| tree |
| stmt_vectype (struct _stmt_vec_info *stmt_info) |
| { |
| return STMT_VINFO_VECTYPE (stmt_info); |
| } |
| |
| /* Return TRUE iff the given statement is in an inner loop relative to |
| the loop being vectorized. */ |
| bool |
| stmt_in_inner_loop_p (struct _stmt_vec_info *stmt_info) |
| { |
| gimple *stmt = STMT_VINFO_STMT (stmt_info); |
| basic_block bb = gimple_bb (stmt); |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| struct loop* loop; |
| |
| if (!loop_vinfo) |
| return false; |
| |
| loop = LOOP_VINFO_LOOP (loop_vinfo); |
| |
| return (bb->loop_father == loop->inner); |
| } |
| |
| /* Record the cost of a statement, either by directly informing the |
| target model or by saving it in a vector for later processing. |
| Return a preliminary estimate of the statement's cost. */ |
| |
| unsigned |
| record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, |
| enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, |
| int misalign, enum vect_cost_model_location where) |
| { |
| if (body_cost_vec) |
| { |
| tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE; |
| stmt_info_for_cost si = { count, kind, |
| stmt_info ? STMT_VINFO_STMT (stmt_info) : NULL, |
| misalign }; |
| body_cost_vec->safe_push (si); |
| return (unsigned) |
| (builtin_vectorization_cost (kind, vectype, misalign) * count); |
| } |
| else |
| return add_stmt_cost (stmt_info->vinfo->target_cost_data, |
| count, kind, stmt_info, misalign, where); |
| } |
| |
| /* Return a variable of type ELEM_TYPE[NELEMS]. */ |
| |
| static tree |
| create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems) |
| { |
| return create_tmp_var (build_array_type_nelts (elem_type, nelems), |
| "vect_array"); |
| } |
| |
| /* ARRAY is an array of vectors created by create_vector_array. |
| Return an SSA_NAME for the vector in index N. The reference |
| is part of the vectorization of STMT and the vector is associated |
| with scalar destination SCALAR_DEST. */ |
| |
| static tree |
| read_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree scalar_dest, |
| tree array, unsigned HOST_WIDE_INT n) |
| { |
| tree vect_type, vect, vect_name, array_ref; |
| gimple *new_stmt; |
| |
| gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE); |
| vect_type = TREE_TYPE (TREE_TYPE (array)); |
| vect = vect_create_destination_var (scalar_dest, vect_type); |
| array_ref = build4 (ARRAY_REF, vect_type, array, |
| build_int_cst (size_type_node, n), |
| NULL_TREE, NULL_TREE); |
| |
| new_stmt = gimple_build_assign (vect, array_ref); |
| vect_name = make_ssa_name (vect, new_stmt); |
| gimple_assign_set_lhs (new_stmt, vect_name); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| |
| return vect_name; |
| } |
| |
| /* ARRAY is an array of vectors created by create_vector_array. |
| Emit code to store SSA_NAME VECT in index N of the array. |
| The store is part of the vectorization of STMT. */ |
| |
| static void |
| write_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree vect, |
| tree array, unsigned HOST_WIDE_INT n) |
| { |
| tree array_ref; |
| gimple *new_stmt; |
| |
| array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array, |
| build_int_cst (size_type_node, n), |
| NULL_TREE, NULL_TREE); |
| |
| new_stmt = gimple_build_assign (array_ref, vect); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| } |
| |
| /* PTR is a pointer to an array of type TYPE. Return a representation |
| of *PTR. The memory reference replaces those in FIRST_DR |
| (and its group). */ |
| |
| static tree |
| create_array_ref (tree type, tree ptr, tree alias_ptr_type) |
| { |
| tree mem_ref; |
| |
| mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0)); |
| /* Arrays have the same alignment as their type. */ |
| set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0); |
| return mem_ref; |
| } |
| |
| /* Utility functions used by vect_mark_stmts_to_be_vectorized. */ |
| |
| /* Function vect_mark_relevant. |
| |
| Mark STMT as "relevant for vectorization" and add it to WORKLIST. */ |
| |
| static void |
| vect_mark_relevant (vec<gimple *> *worklist, gimple *stmt, |
| enum vect_relevant relevant, bool live_p) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info); |
| bool save_live_p = STMT_VINFO_LIVE_P (stmt_info); |
| gimple *pattern_stmt; |
| |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "mark relevant %d, live %d: ", relevant, live_p); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); |
| } |
| |
| /* If this stmt is an original stmt in a pattern, we might need to mark its |
| related pattern stmt instead of the original stmt. However, such stmts |
| may have their own uses that are not in any pattern, in such cases the |
| stmt itself should be marked. */ |
| if (STMT_VINFO_IN_PATTERN_P (stmt_info)) |
| { |
| /* This is the last stmt in a sequence that was detected as a |
| pattern that can potentially be vectorized. Don't mark the stmt |
| as relevant/live because it's not going to be vectorized. |
| Instead mark the pattern-stmt that replaces it. */ |
| |
| pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "last stmt in pattern. don't mark" |
| " relevant/live.\n"); |
| stmt_info = vinfo_for_stmt (pattern_stmt); |
| gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt); |
| save_relevant = STMT_VINFO_RELEVANT (stmt_info); |
| save_live_p = STMT_VINFO_LIVE_P (stmt_info); |
| stmt = pattern_stmt; |
| } |
| |
| STMT_VINFO_LIVE_P (stmt_info) |= live_p; |
| if (relevant > STMT_VINFO_RELEVANT (stmt_info)) |
| STMT_VINFO_RELEVANT (stmt_info) = relevant; |
| |
| if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant |
| && STMT_VINFO_LIVE_P (stmt_info) == save_live_p) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "already marked relevant/live.\n"); |
| return; |
| } |
| |
| worklist->safe_push (stmt); |
| } |
| |
| |
| /* Function is_simple_and_all_uses_invariant |
| |
| Return true if STMT is simple and all uses of it are invariant. */ |
| |
| bool |
| is_simple_and_all_uses_invariant (gimple *stmt, loop_vec_info loop_vinfo) |
| { |
| tree op; |
| gimple *def_stmt; |
| ssa_op_iter iter; |
| |
| if (!is_gimple_assign (stmt)) |
| return false; |
| |
| FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE) |
| { |
| enum vect_def_type dt = vect_uninitialized_def; |
| |
| if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "use not simple.\n"); |
| return false; |
| } |
| |
| if (dt != vect_external_def && dt != vect_constant_def) |
| return false; |
| } |
| return true; |
| } |
| |
| /* Function vect_stmt_relevant_p. |
| |
| Return true if STMT in loop that is represented by LOOP_VINFO is |
| "relevant for vectorization". |
| |
| A stmt is considered "relevant for vectorization" if: |
| - it has uses outside the loop. |
| - it has vdefs (it alters memory). |
| - control stmts in the loop (except for the exit condition). |
| |
| CHECKME: what other side effects would the vectorizer allow? */ |
| |
| static bool |
| vect_stmt_relevant_p (gimple *stmt, loop_vec_info loop_vinfo, |
| enum vect_relevant *relevant, bool *live_p) |
| { |
| struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); |
| ssa_op_iter op_iter; |
| imm_use_iterator imm_iter; |
| use_operand_p use_p; |
| def_operand_p def_p; |
| |
| *relevant = vect_unused_in_scope; |
| *live_p = false; |
| |
| /* cond stmt other than loop exit cond. */ |
| if (is_ctrl_stmt (stmt) |
| && STMT_VINFO_TYPE (vinfo_for_stmt (stmt)) |
| != loop_exit_ctrl_vec_info_type) |
| *relevant = vect_used_in_scope; |
| |
| /* changing memory. */ |
| if (gimple_code (stmt) != GIMPLE_PHI) |
| if (gimple_vdef (stmt) |
| && !gimple_clobber_p (stmt)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vec_stmt_relevant_p: stmt has vdefs.\n"); |
| *relevant = vect_used_in_scope; |
| } |
| |
| /* uses outside the loop. */ |
| FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF) |
| { |
| FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p)) |
| { |
| basic_block bb = gimple_bb (USE_STMT (use_p)); |
| if (!flow_bb_inside_loop_p (loop, bb)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vec_stmt_relevant_p: used out of loop.\n"); |
| |
| if (is_gimple_debug (USE_STMT (use_p))) |
| continue; |
| |
| /* We expect all such uses to be in the loop exit phis |
| (because of loop closed form) */ |
| gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI); |
| gcc_assert (bb == single_exit (loop)->dest); |
| |
| *live_p = true; |
| } |
| } |
| } |
| |
| if (*live_p && *relevant == vect_unused_in_scope |
| && !is_simple_and_all_uses_invariant (stmt, loop_vinfo)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vec_stmt_relevant_p: stmt live but not relevant.\n"); |
| *relevant = vect_used_only_live; |
| } |
| |
| return (*live_p || *relevant); |
| } |
| |
| |
| /* Function exist_non_indexing_operands_for_use_p |
| |
| USE is one of the uses attached to STMT. Check if USE is |
| used in STMT for anything other than indexing an array. */ |
| |
| static bool |
| exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) |
| { |
| tree operand; |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| |
| /* USE corresponds to some operand in STMT. If there is no data |
| reference in STMT, then any operand that corresponds to USE |
| is not indexing an array. */ |
| if (!STMT_VINFO_DATA_REF (stmt_info)) |
| return true; |
| |
| /* STMT has a data_ref. FORNOW this means that its of one of |
| the following forms: |
| -1- ARRAY_REF = var |
| -2- var = ARRAY_REF |
| (This should have been verified in analyze_data_refs). |
| |
| 'var' in the second case corresponds to a def, not a use, |
| so USE cannot correspond to any operands that are not used |
| for array indexing. |
| |
| Therefore, all we need to check is if STMT falls into the |
| first case, and whether var corresponds to USE. */ |
| |
| if (!gimple_assign_copy_p (stmt)) |
| { |
| if (is_gimple_call (stmt) |
| && gimple_call_internal_p (stmt)) |
| switch (gimple_call_internal_fn (stmt)) |
| { |
| case IFN_MASK_STORE: |
| operand = gimple_call_arg (stmt, 3); |
| if (operand == use) |
| return true; |
| /* FALLTHRU */ |
| case IFN_MASK_LOAD: |
| operand = gimple_call_arg (stmt, 2); |
| if (operand == use) |
| return true; |
| break; |
| default: |
| break; |
| } |
| return false; |
| } |
| |
| if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME) |
| return false; |
| operand = gimple_assign_rhs1 (stmt); |
| if (TREE_CODE (operand) != SSA_NAME) |
| return false; |
| |
| if (operand == use) |
| return true; |
| |
| return false; |
| } |
| |
| |
| /* |
| Function process_use. |
| |
| Inputs: |
| - a USE in STMT in a loop represented by LOOP_VINFO |
| - RELEVANT - enum value to be set in the STMT_VINFO of the stmt |
| that defined USE. This is done by calling mark_relevant and passing it |
| the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant). |
| - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't |
| be performed. |
| |
| Outputs: |
| Generally, LIVE_P and RELEVANT are used to define the liveness and |
| relevance info of the DEF_STMT of this USE: |
| STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p |
| STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant |
| Exceptions: |
| - case 1: If USE is used only for address computations (e.g. array indexing), |
| which does not need to be directly vectorized, then the liveness/relevance |
| of the respective DEF_STMT is left unchanged. |
| - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we |
| skip DEF_STMT cause it had already been processed. |
| - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will |
| be modified accordingly. |
| |
| Return true if everything is as expected. Return false otherwise. */ |
| |
| static bool |
| process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, |
| enum vect_relevant relevant, vec<gimple *> *worklist, |
| bool force) |
| { |
| struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); |
| stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); |
| stmt_vec_info dstmt_vinfo; |
| basic_block bb, def_bb; |
| gimple *def_stmt; |
| enum vect_def_type dt; |
| |
| /* case 1: we are only interested in uses that need to be vectorized. Uses |
| that are used for address computation are not considered relevant. */ |
| if (!force && !exist_non_indexing_operands_for_use_p (use, stmt)) |
| return true; |
| |
| if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &dt)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "not vectorized: unsupported use in stmt.\n"); |
| return false; |
| } |
| |
| if (!def_stmt || gimple_nop_p (def_stmt)) |
| return true; |
| |
| def_bb = gimple_bb (def_stmt); |
| if (!flow_bb_inside_loop_p (loop, def_bb)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, "def_stmt is out of loop.\n"); |
| return true; |
| } |
| |
| /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT). |
| DEF_STMT must have already been processed, because this should be the |
| only way that STMT, which is a reduction-phi, was put in the worklist, |
| as there should be no other uses for DEF_STMT in the loop. So we just |
| check that everything is as expected, and we are done. */ |
| dstmt_vinfo = vinfo_for_stmt (def_stmt); |
| bb = gimple_bb (stmt); |
| if (gimple_code (stmt) == GIMPLE_PHI |
| && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def |
| && gimple_code (def_stmt) != GIMPLE_PHI |
| && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def |
| && bb->loop_father == def_bb->loop_father) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "reduc-stmt defining reduc-phi in the same nest.\n"); |
| if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo)) |
| dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo)); |
| gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction); |
| gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo) |
| || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope); |
| return true; |
| } |
| |
| /* case 3a: outer-loop stmt defining an inner-loop stmt: |
| outer-loop-header-bb: |
| d = def_stmt |
| inner-loop: |
| stmt # use (d) |
| outer-loop-tail-bb: |
| ... */ |
| if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "outer-loop def-stmt defining inner-loop stmt.\n"); |
| |
| switch (relevant) |
| { |
| case vect_unused_in_scope: |
| relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ? |
| vect_used_in_scope : vect_unused_in_scope; |
| break; |
| |
| case vect_used_in_outer_by_reduction: |
| gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def); |
| relevant = vect_used_by_reduction; |
| break; |
| |
| case vect_used_in_outer: |
| gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def); |
| relevant = vect_used_in_scope; |
| break; |
| |
| case vect_used_in_scope: |
| break; |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| /* case 3b: inner-loop stmt defining an outer-loop stmt: |
| outer-loop-header-bb: |
| ... |
| inner-loop: |
| d = def_stmt |
| outer-loop-tail-bb (or outer-loop-exit-bb in double reduction): |
| stmt # use (d) */ |
| else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "inner-loop def-stmt defining outer-loop stmt.\n"); |
| |
| switch (relevant) |
| { |
| case vect_unused_in_scope: |
| relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def |
| || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ? |
| vect_used_in_outer_by_reduction : vect_unused_in_scope; |
| break; |
| |
| case vect_used_by_reduction: |
| case vect_used_only_live: |
| relevant = vect_used_in_outer_by_reduction; |
| break; |
| |
| case vect_used_in_scope: |
| relevant = vect_used_in_outer; |
| break; |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| vect_mark_relevant (worklist, def_stmt, relevant, false); |
| return true; |
| } |
| |
| |
| /* Function vect_mark_stmts_to_be_vectorized. |
| |
| Not all stmts in the loop need to be vectorized. For example: |
| |
| for i... |
| for j... |
| 1. T0 = i + j |
| 2. T1 = a[T0] |
| |
| 3. j = j + 1 |
| |
| Stmt 1 and 3 do not need to be vectorized, because loop control and |
| addressing of vectorized data-refs are handled differently. |
| |
| This pass detects such stmts. */ |
| |
| bool |
| vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) |
| { |
| struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); |
| basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); |
| unsigned int nbbs = loop->num_nodes; |
| gimple_stmt_iterator si; |
| gimple *stmt; |
| unsigned int i; |
| stmt_vec_info stmt_vinfo; |
| basic_block bb; |
| gimple *phi; |
| bool live_p; |
| enum vect_relevant relevant; |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "=== vect_mark_stmts_to_be_vectorized ===\n"); |
| |
| auto_vec<gimple *, 64> worklist; |
| |
| /* 1. Init worklist. */ |
| for (i = 0; i < nbbs; i++) |
| { |
| bb = bbs[i]; |
| for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si)) |
| { |
| phi = gsi_stmt (si); |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0); |
| } |
| |
| if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p)) |
| vect_mark_relevant (&worklist, phi, relevant, live_p); |
| } |
| for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) |
| { |
| stmt = gsi_stmt (si); |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, "init: stmt relevant? "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); |
| } |
| |
| if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p)) |
| vect_mark_relevant (&worklist, stmt, relevant, live_p); |
| } |
| } |
| |
| /* 2. Process_worklist */ |
| while (worklist.length () > 0) |
| { |
| use_operand_p use_p; |
| ssa_op_iter iter; |
| |
| stmt = worklist.pop (); |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, "worklist: examine stmt: "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); |
| } |
| |
| /* Examine the USEs of STMT. For each USE, mark the stmt that defines it |
| (DEF_STMT) as relevant/irrelevant according to the relevance property |
| of STMT. */ |
| stmt_vinfo = vinfo_for_stmt (stmt); |
| relevant = STMT_VINFO_RELEVANT (stmt_vinfo); |
| |
| /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is |
| propagated as is to the DEF_STMTs of its USEs. |
| |
| One exception is when STMT has been identified as defining a reduction |
| variable; in this case we set the relevance to vect_used_by_reduction. |
| This is because we distinguish between two kinds of relevant stmts - |
| those that are used by a reduction computation, and those that are |
| (also) used by a regular computation. This allows us later on to |
| identify stmts that are used solely by a reduction, and therefore the |
| order of the results that they produce does not have to be kept. */ |
| |
| switch (STMT_VINFO_DEF_TYPE (stmt_vinfo)) |
| { |
| case vect_reduction_def: |
| gcc_assert (relevant != vect_unused_in_scope); |
| if (relevant != vect_unused_in_scope |
| && relevant != vect_used_in_scope |
| && relevant != vect_used_by_reduction |
| && relevant != vect_used_only_live) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "unsupported use of reduction.\n"); |
| return false; |
| } |
| break; |
| |
| case vect_nested_cycle: |
| if (relevant != vect_unused_in_scope |
| && relevant != vect_used_in_outer_by_reduction |
| && relevant != vect_used_in_outer) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "unsupported use of nested cycle.\n"); |
| |
| return false; |
| } |
| break; |
| |
| case vect_double_reduction_def: |
| if (relevant != vect_unused_in_scope |
| && relevant != vect_used_by_reduction |
| && relevant != vect_used_only_live) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "unsupported use of double reduction.\n"); |
| |
| return false; |
| } |
| break; |
| |
| default: |
| break; |
| } |
| |
| if (is_pattern_stmt_p (stmt_vinfo)) |
| { |
| /* Pattern statements are not inserted into the code, so |
| FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we |
| have to scan the RHS or function arguments instead. */ |
| if (is_gimple_assign (stmt)) |
| { |
| enum tree_code rhs_code = gimple_assign_rhs_code (stmt); |
| tree op = gimple_assign_rhs1 (stmt); |
| |
| i = 1; |
| if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op)) |
| { |
| if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo, |
| relevant, &worklist, false) |
| || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo, |
| relevant, &worklist, false)) |
| return false; |
| i = 2; |
| } |
| for (; i < gimple_num_ops (stmt); i++) |
| { |
| op = gimple_op (stmt, i); |
| if (TREE_CODE (op) == SSA_NAME |
| && !process_use (stmt, op, loop_vinfo, relevant, |
| &worklist, false)) |
| return false; |
| } |
| } |
| else if (is_gimple_call (stmt)) |
| { |
| for (i = 0; i < gimple_call_num_args (stmt); i++) |
| { |
| tree arg = gimple_call_arg (stmt, i); |
| if (!process_use (stmt, arg, loop_vinfo, relevant, |
| &worklist, false)) |
| return false; |
| } |
| } |
| } |
| else |
| FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE) |
| { |
| tree op = USE_FROM_PTR (use_p); |
| if (!process_use (stmt, op, loop_vinfo, relevant, |
| &worklist, false)) |
| return false; |
| } |
| |
| if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) |
| { |
| gather_scatter_info gs_info; |
| if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info)) |
| gcc_unreachable (); |
| if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant, |
| &worklist, true)) |
| return false; |
| } |
| } /* while worklist */ |
| |
| return true; |
| } |
| |
| |
| /* Function vect_model_simple_cost. |
| |
| Models cost for simple operations, i.e. those that only emit ncopies of a |
| single op. Right now, this does not account for multiple insns that could |
| be generated for the single vector op. We will handle that shortly. */ |
| |
| void |
| vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies, |
| enum vect_def_type *dt, |
| stmt_vector_for_cost *prologue_cost_vec, |
| stmt_vector_for_cost *body_cost_vec) |
| { |
| int i; |
| int inside_cost = 0, prologue_cost = 0; |
| |
| /* The SLP costs were already calculated during SLP tree build. */ |
| if (PURE_SLP_STMT (stmt_info)) |
| return; |
| |
| /* FORNOW: Assuming maximum 2 args per stmts. */ |
| for (i = 0; i < 2; i++) |
| if (dt[i] == vect_constant_def || dt[i] == vect_external_def) |
| prologue_cost += record_stmt_cost (prologue_cost_vec, 1, vector_stmt, |
| stmt_info, 0, vect_prologue); |
| |
| /* Pass the inside-of-loop statements to the target-specific cost model. */ |
| inside_cost = record_stmt_cost (body_cost_vec, ncopies, vector_stmt, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_simple_cost: inside_cost = %d, " |
| "prologue_cost = %d .\n", inside_cost, prologue_cost); |
| } |
| |
| |
| /* Model cost for type demotion and promotion operations. PWR is normally |
| zero for single-step promotions and demotions. It will be one if |
| two-step promotion/demotion is required, and so on. Each additional |
| step doubles the number of instructions required. */ |
| |
| static void |
| vect_model_promotion_demotion_cost (stmt_vec_info stmt_info, |
| enum vect_def_type *dt, int pwr) |
| { |
| int i, tmp; |
| int inside_cost = 0, prologue_cost = 0; |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); |
| void *target_cost_data; |
| |
| /* The SLP costs were already calculated during SLP tree build. */ |
| if (PURE_SLP_STMT (stmt_info)) |
| return; |
| |
| if (loop_vinfo) |
| target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo); |
| else |
| target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo); |
| |
| for (i = 0; i < pwr + 1; i++) |
| { |
| tmp = (STMT_VINFO_TYPE (stmt_info) == type_promotion_vec_info_type) ? |
| (i + 1) : i; |
| inside_cost += add_stmt_cost (target_cost_data, vect_pow2 (tmp), |
| vec_promote_demote, stmt_info, 0, |
| vect_body); |
| } |
| |
| /* FORNOW: Assuming maximum 2 args per stmts. */ |
| for (i = 0; i < 2; i++) |
| if (dt[i] == vect_constant_def || dt[i] == vect_external_def) |
| prologue_cost += add_stmt_cost (target_cost_data, 1, vector_stmt, |
| stmt_info, 0, vect_prologue); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_promotion_demotion_cost: inside_cost = %d, " |
| "prologue_cost = %d .\n", inside_cost, prologue_cost); |
| } |
| |
| /* Function vect_model_store_cost |
| |
| Models cost for stores. In the case of grouped accesses, one access |
| has the overhead of the grouped access attributed to it. */ |
| |
| void |
| vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, |
| vect_memory_access_type memory_access_type, |
| enum vect_def_type dt, slp_tree slp_node, |
| stmt_vector_for_cost *prologue_cost_vec, |
| stmt_vector_for_cost *body_cost_vec) |
| { |
| unsigned int inside_cost = 0, prologue_cost = 0; |
| struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); |
| gimple *first_stmt = STMT_VINFO_STMT (stmt_info); |
| bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info); |
| |
| if (dt == vect_constant_def || dt == vect_external_def) |
| prologue_cost += record_stmt_cost (prologue_cost_vec, 1, scalar_to_vec, |
| stmt_info, 0, vect_prologue); |
| |
| /* Grouped stores update all elements in the group at once, |
| so we want the DR for the first statement. */ |
| if (!slp_node && grouped_access_p) |
| { |
| first_stmt = GROUP_FIRST_ELEMENT (stmt_info); |
| dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); |
| } |
| |
| /* True if we should include any once-per-group costs as well as |
| the cost of the statement itself. For SLP we only get called |
| once per group anyhow. */ |
| bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info)); |
| |
| /* We assume that the cost of a single store-lanes instruction is |
| equivalent to the cost of GROUP_SIZE separate stores. If a grouped |
| access is instead being provided by a permute-and-store operation, |
| include the cost of the permutes. */ |
| if (first_stmt_p |
| && memory_access_type == VMAT_CONTIGUOUS_PERMUTE) |
| { |
| /* Uses a high and low interleave or shuffle operations for each |
| needed permute. */ |
| int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); |
| int nstmts = ncopies * ceil_log2 (group_size) * group_size; |
| inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_store_cost: strided group_size = %d .\n", |
| group_size); |
| } |
| |
| tree vectype = STMT_VINFO_VECTYPE (stmt_info); |
| /* Costs of the stores. */ |
| if (memory_access_type == VMAT_ELEMENTWISE) |
| /* N scalar stores plus extracting the elements. */ |
| inside_cost += record_stmt_cost (body_cost_vec, |
| ncopies * TYPE_VECTOR_SUBPARTS (vectype), |
| scalar_store, stmt_info, 0, vect_body); |
| else |
| vect_get_store_cost (dr, ncopies, &inside_cost, body_cost_vec); |
| |
| if (memory_access_type == VMAT_ELEMENTWISE |
| || memory_access_type == VMAT_STRIDED_SLP) |
| inside_cost += record_stmt_cost (body_cost_vec, |
| ncopies * TYPE_VECTOR_SUBPARTS (vectype), |
| vec_to_scalar, stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_store_cost: inside_cost = %d, " |
| "prologue_cost = %d .\n", inside_cost, prologue_cost); |
| } |
| |
| |
| /* Calculate cost of DR's memory access. */ |
| void |
| vect_get_store_cost (struct data_reference *dr, int ncopies, |
| unsigned int *inside_cost, |
| stmt_vector_for_cost *body_cost_vec) |
| { |
| int alignment_support_scheme = vect_supportable_dr_alignment (dr, false); |
| gimple *stmt = DR_STMT (dr); |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| |
| switch (alignment_support_scheme) |
| { |
| case dr_aligned: |
| { |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, |
| vector_store, stmt_info, 0, |
| vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_store_cost: aligned.\n"); |
| break; |
| } |
| |
| case dr_unaligned_supported: |
| { |
| /* Here, we assign an additional cost for the unaligned store. */ |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, |
| unaligned_store, stmt_info, |
| DR_MISALIGNMENT (dr), vect_body); |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_store_cost: unaligned supported by " |
| "hardware.\n"); |
| break; |
| } |
| |
| case dr_unaligned_unsupported: |
| { |
| *inside_cost = VECT_MAX_COST; |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "vect_model_store_cost: unsupported access.\n"); |
| break; |
| } |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| |
| /* Function vect_model_load_cost |
| |
| Models cost for loads. In the case of grouped accesses, one access has |
| the overhead of the grouped access attributed to it. Since unaligned |
| accesses are supported for loads, we also account for the costs of the |
| access scheme chosen. */ |
| |
| void |
| vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, |
| vect_memory_access_type memory_access_type, |
| slp_tree slp_node, |
| stmt_vector_for_cost *prologue_cost_vec, |
| stmt_vector_for_cost *body_cost_vec) |
| { |
| gimple *first_stmt = STMT_VINFO_STMT (stmt_info); |
| struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); |
| unsigned int inside_cost = 0, prologue_cost = 0; |
| bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info); |
| |
| /* Grouped loads read all elements in the group at once, |
| so we want the DR for the first statement. */ |
| if (!slp_node && grouped_access_p) |
| { |
| first_stmt = GROUP_FIRST_ELEMENT (stmt_info); |
| dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); |
| } |
| |
| /* True if we should include any once-per-group costs as well as |
| the cost of the statement itself. For SLP we only get called |
| once per group anyhow. */ |
| bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info)); |
| |
| /* We assume that the cost of a single load-lanes instruction is |
| equivalent to the cost of GROUP_SIZE separate loads. If a grouped |
| access is instead being provided by a load-and-permute operation, |
| include the cost of the permutes. */ |
| if (first_stmt_p |
| && memory_access_type == VMAT_CONTIGUOUS_PERMUTE) |
| { |
| /* Uses an even and odd extract operations or shuffle operations |
| for each needed permute. */ |
| int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); |
| int nstmts = ncopies * ceil_log2 (group_size) * group_size; |
| inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: strided group_size = %d .\n", |
| group_size); |
| } |
| |
| /* The loads themselves. */ |
| if (memory_access_type == VMAT_ELEMENTWISE) |
| { |
| /* N scalar loads plus gathering them into a vector. */ |
| tree vectype = STMT_VINFO_VECTYPE (stmt_info); |
| inside_cost += record_stmt_cost (body_cost_vec, |
| ncopies * TYPE_VECTOR_SUBPARTS (vectype), |
| scalar_load, stmt_info, 0, vect_body); |
| } |
| else |
| vect_get_load_cost (dr, ncopies, first_stmt_p, |
| &inside_cost, &prologue_cost, |
| prologue_cost_vec, body_cost_vec, true); |
| if (memory_access_type == VMAT_ELEMENTWISE |
| || memory_access_type == VMAT_STRIDED_SLP) |
| inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_construct, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: inside_cost = %d, " |
| "prologue_cost = %d .\n", inside_cost, prologue_cost); |
| } |
| |
| |
| /* Calculate cost of DR's memory access. */ |
| void |
| vect_get_load_cost (struct data_reference *dr, int ncopies, |
| bool add_realign_cost, unsigned int *inside_cost, |
| unsigned int *prologue_cost, |
| stmt_vector_for_cost *prologue_cost_vec, |
| stmt_vector_for_cost *body_cost_vec, |
| bool record_prologue_costs) |
| { |
| int alignment_support_scheme = vect_supportable_dr_alignment (dr, false); |
| gimple *stmt = DR_STMT (dr); |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| |
| switch (alignment_support_scheme) |
| { |
| case dr_aligned: |
| { |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: aligned.\n"); |
| |
| break; |
| } |
| case dr_unaligned_supported: |
| { |
| /* Here, we assign an additional cost for the unaligned load. */ |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, |
| unaligned_load, stmt_info, |
| DR_MISALIGNMENT (dr), vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: unaligned supported by " |
| "hardware.\n"); |
| |
| break; |
| } |
| case dr_explicit_realign: |
| { |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2, |
| vector_load, stmt_info, 0, vect_body); |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, |
| vec_perm, stmt_info, 0, vect_body); |
| |
| /* FIXME: If the misalignment remains fixed across the iterations of |
| the containing loop, the following cost should be added to the |
| prologue costs. */ |
| if (targetm.vectorize.builtin_mask_for_load) |
| *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: explicit realign\n"); |
| |
| break; |
| } |
| case dr_explicit_realign_optimized: |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: unaligned software " |
| "pipelined.\n"); |
| |
| /* Unaligned software pipeline has a load of an address, an initial |
| load, and possibly a mask operation to "prime" the loop. However, |
| if this is an access in a group of loads, which provide grouped |
| access, then the above cost should only be considered for one |
| access in the group. Inside the loop, there is a load op |
| and a realignment op. */ |
| |
| if (add_realign_cost && record_prologue_costs) |
| { |
| *prologue_cost += record_stmt_cost (prologue_cost_vec, 2, |
| vector_stmt, stmt_info, |
| 0, vect_prologue); |
| if (targetm.vectorize.builtin_mask_for_load) |
| *prologue_cost += record_stmt_cost (prologue_cost_vec, 1, |
| vector_stmt, stmt_info, |
| 0, vect_prologue); |
| } |
| |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load, |
| stmt_info, 0, vect_body); |
| *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm, |
| stmt_info, 0, vect_body); |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_model_load_cost: explicit realign optimized" |
| "\n"); |
| |
| break; |
| } |
| |
| case dr_unaligned_unsupported: |
| { |
| *inside_cost = VECT_MAX_COST; |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "vect_model_load_cost: unsupported access.\n"); |
| break; |
| } |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in |
| the loop preheader for the vectorized stmt STMT. */ |
| |
| static void |
| vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi) |
| { |
| if (gsi) |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| else |
| { |
| stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); |
| |
| if (loop_vinfo) |
| { |
| struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); |
| basic_block new_bb; |
| edge pe; |
| |
| if (nested_in_vect_loop_p (loop, stmt)) |
| loop = loop->inner; |
| |
| pe = loop_preheader_edge (loop); |
| new_bb = gsi_insert_on_edge_immediate (pe, new_stmt); |
| gcc_assert (!new_bb); |
| } |
| else |
| { |
| bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo); |
| basic_block bb; |
| gimple_stmt_iterator gsi_bb_start; |
| |
| gcc_assert (bb_vinfo); |
| bb = BB_VINFO_BB (bb_vinfo); |
| gsi_bb_start = gsi_after_labels (bb); |
| gsi_insert_before (&gsi_bb_start, new_stmt, GSI_SAME_STMT); |
| } |
| } |
| |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "created new init_stmt: "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, new_stmt, 0); |
| } |
| } |
| |
| /* Function vect_init_vector. |
| |
| Insert a new stmt (INIT_STMT) that initializes a new variable of type |
| TYPE with the value VAL. If TYPE is a vector type and VAL does not have |
| vector type a vector with all elements equal to VAL is created first. |
| Place the initialization at BSI if it is not NULL. Otherwise, place the |
| initialization at the loop preheader. |
| Return the DEF of INIT_STMT. |
| It will be used in the vectorization of STMT. */ |
| |
| tree |
| vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi) |
| { |
| gimple *init_stmt; |
| tree new_temp; |
| |
| /* We abuse this function to push sth to a SSA name with initial 'val'. */ |
| if (! useless_type_conversion_p (type, TREE_TYPE (val))) |
| { |
| gcc_assert (TREE_CODE (type) == VECTOR_TYPE); |
| if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val))) |
| { |
| /* Scalar boolean value should be transformed into |
| all zeros or all ones value before building a vector. */ |
| if (VECTOR_BOOLEAN_TYPE_P (type)) |
| { |
| tree true_val = build_all_ones_cst (TREE_TYPE (type)); |
| tree false_val = build_zero_cst (TREE_TYPE (type)); |
| |
| if (CONSTANT_CLASS_P (val)) |
| val = integer_zerop (val) ? false_val : true_val; |
| else |
| { |
| new_temp = make_ssa_name (TREE_TYPE (type)); |
| init_stmt = gimple_build_assign (new_temp, COND_EXPR, |
| val, true_val, false_val); |
| vect_init_vector_1 (stmt, init_stmt, gsi); |
| val = new_temp; |
| } |
| } |
| else if (CONSTANT_CLASS_P (val)) |
| val = fold_convert (TREE_TYPE (type), val); |
| else |
| { |
| new_temp = make_ssa_name (TREE_TYPE (type)); |
| if (! INTEGRAL_TYPE_P (TREE_TYPE (val))) |
| init_stmt = gimple_build_assign (new_temp, |
| fold_build1 (VIEW_CONVERT_EXPR, |
| TREE_TYPE (type), |
| val)); |
| else |
| init_stmt = gimple_build_assign (new_temp, NOP_EXPR, val); |
| vect_init_vector_1 (stmt, init_stmt, gsi); |
| val = new_temp; |
| } |
| } |
| val = build_vector_from_val (type, val); |
| } |
| |
| new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_"); |
| init_stmt = gimple_build_assign (new_temp, val); |
| vect_init_vector_1 (stmt, init_stmt, gsi); |
| return new_temp; |
| } |
| |
| /* Function vect_get_vec_def_for_operand_1. |
| |
| For a defining stmt DEF_STMT of a scalar stmt, return a vector def with type |
| DT that will be used in the vectorized stmt. */ |
| |
| tree |
| vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) |
| { |
| tree vec_oprnd; |
| gimple *vec_stmt; |
| stmt_vec_info def_stmt_info = NULL; |
| |
| switch (dt) |
| { |
| /* operand is a constant or a loop invariant. */ |
| case vect_constant_def: |
| case vect_external_def: |
| /* Code should use vect_get_vec_def_for_operand. */ |
| gcc_unreachable (); |
| |
| /* operand is defined inside the loop. */ |
| case vect_internal_def: |
| { |
| /* Get the def from the vectorized stmt. */ |
| def_stmt_info = vinfo_for_stmt (def_stmt); |
| |
| vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info); |
| /* Get vectorized pattern statement. */ |
| if (!vec_stmt |
| && STMT_VINFO_IN_PATTERN_P (def_stmt_info) |
| && !STMT_VINFO_RELEVANT (def_stmt_info)) |
| vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt ( |
| STMT_VINFO_RELATED_STMT (def_stmt_info))); |
| gcc_assert (vec_stmt); |
| if (gimple_code (vec_stmt) == GIMPLE_PHI) |
| vec_oprnd = PHI_RESULT (vec_stmt); |
| else if (is_gimple_call (vec_stmt)) |
| vec_oprnd = gimple_call_lhs (vec_stmt); |
| else |
| vec_oprnd = gimple_assign_lhs (vec_stmt); |
| return vec_oprnd; |
| } |
| |
| /* operand is defined by a loop header phi - reduction */ |
| case vect_reduction_def: |
| case vect_double_reduction_def: |
| case vect_nested_cycle: |
| /* Code should use get_initial_def_for_reduction. */ |
| gcc_unreachable (); |
| |
| /* operand is defined by loop-header phi - induction. */ |
| case vect_induction_def: |
| { |
| gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI); |
| |
| /* Get the def from the vectorized stmt. */ |
| def_stmt_info = vinfo_for_stmt (def_stmt); |
| vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info); |
| if (gimple_code (vec_stmt) == GIMPLE_PHI) |
| vec_oprnd = PHI_RESULT (vec_stmt); |
| else |
| vec_oprnd = gimple_get_lhs (vec_stmt); |
| return vec_oprnd; |
| } |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| |
| /* Function vect_get_vec_def_for_operand. |
| |
| OP is an operand in STMT. This function returns a (vector) def that will be |
| used in the vectorized stmt for STMT. |
| |
| In the case that OP is an SSA_NAME which is defined in the loop, then |
| STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def. |
| |
| In case OP is an invariant or constant, a new stmt that creates a vector def |
| needs to be introduced. VECTYPE may be used to specify a required type for |
| vector invariant. */ |
| |
| tree |
| vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype) |
| { |
| gimple *def_stmt; |
| enum vect_def_type dt; |
| bool is_simple_use; |
| stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); |
| |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "vect_get_vec_def_for_operand: "); |
| dump_generic_expr (MSG_NOTE, TDF_SLIM, op); |
| dump_printf (MSG_NOTE, "\n"); |
| } |
| |
| is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt); |
| gcc_assert (is_simple_use); |
| if (def_stmt && dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0); |
| } |
| |
| if (dt == vect_constant_def || dt == vect_external_def) |
| { |
| tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo); |
| tree vector_type; |
| |
| if (vectype) |
| vector_type = vectype; |
| else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op)) |
| && VECTOR_BOOLEAN_TYPE_P (stmt_vectype)) |
| vector_type = build_same_sized_truth_vector_type (stmt_vectype); |
| else |
| vector_type = get_vectype_for_scalar_type (TREE_TYPE (op)); |
| |
| gcc_assert (vector_type); |
| return vect_init_vector (stmt, op, vector_type, NULL); |
| } |
| else |
| return vect_get_vec_def_for_operand_1 (def_stmt, dt); |
| } |
| |
| |
| /* Function vect_get_vec_def_for_stmt_copy |
| |
| Return a vector-def for an operand. This function is used when the |
| vectorized stmt to be created (by the caller to this function) is a "copy" |
| created in case the vectorized result cannot fit in one vector, and several |
| copies of the vector-stmt are required. In this case the vector-def is |
| retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field |
| of the stmt that defines VEC_OPRND. |
| DT is the type of the vector def VEC_OPRND. |
| |
| Context: |
| In case the vectorization factor (VF) is bigger than the number |
| of elements that can fit in a vectype (nunits), we have to generate |
| more than one vector stmt to vectorize the scalar stmt. This situation |
| arises when there are multiple data-types operated upon in the loop; the |
| smallest data-type determines the VF, and as a result, when vectorizing |
| stmts operating on wider types we need to create 'VF/nunits' "copies" of the |
| vector stmt (each computing a vector of 'nunits' results, and together |
| computing 'VF' results in each iteration). This function is called when |
| vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in |
| which VF=16 and nunits=4, so the number of copies required is 4): |
| |
| scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT |
| |
| S1: x = load VS1.0: vx.0 = memref0 VS1.1 |
| VS1.1: vx.1 = memref1 VS1.2 |
| VS1.2: vx.2 = memref2 VS1.3 |
| VS1.3: vx.3 = memref3 |
| |
| S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1 |
| VSnew.1: vz1 = vx.1 + ... VSnew.2 |
| VSnew.2: vz2 = vx.2 + ... VSnew.3 |
| VSnew.3: vz3 = vx.3 + ... |
| |
| The vectorization of S1 is explained in vectorizable_load. |
| The vectorization of S2: |
| To create the first vector-stmt out of the 4 copies - VSnew.0 - |
| the function 'vect_get_vec_def_for_operand' is called to |
| get the relevant vector-def for each operand of S2. For operand x it |
| returns the vector-def 'vx.0'. |
| |
| To create the remaining copies of the vector-stmt (VSnew.j), this |
| function is called to get the relevant vector-def for each operand. It is |
| obtained from the respective VS1.j stmt, which is recorded in the |
| STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND. |
| |
| For example, to obtain the vector-def 'vx.1' in order to create the |
| vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'. |
| Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the |
| STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1', |
| and return its def ('vx.1'). |
| Overall, to create the above sequence this function will be called 3 times: |
| vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0); |
| vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1); |
| vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */ |
| |
| tree |
| vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd) |
| { |
| gimple *vec_stmt_for_operand; |
| stmt_vec_info def_stmt_info; |
| |
| /* Do nothing; can reuse same def. */ |
| if (dt == vect_external_def || dt == vect_constant_def ) |
| return vec_oprnd; |
| |
| vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd); |
| def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand); |
| gcc_assert (def_stmt_info); |
| vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info); |
| gcc_assert (vec_stmt_for_operand); |
| if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI) |
| vec_oprnd = PHI_RESULT (vec_stmt_for_operand); |
| else |
| vec_oprnd = gimple_get_lhs (vec_stmt_for_operand); |
| return vec_oprnd; |
| } |
| |
| |
| /* Get vectorized definitions for the operands to create a copy of an original |
| stmt. See vect_get_vec_def_for_stmt_copy () for details. */ |
| |
| static void |
| vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt, |
| vec<tree> *vec_oprnds0, |
| vec<tree> *vec_oprnds1) |
| { |
| tree vec_oprnd = vec_oprnds0->pop (); |
| |
| vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd); |
| vec_oprnds0->quick_push (vec_oprnd); |
| |
| if (vec_oprnds1 && vec_oprnds1->length ()) |
| { |
| vec_oprnd = vec_oprnds1->pop (); |
| vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd); |
| vec_oprnds1->quick_push (vec_oprnd); |
| } |
| } |
| |
| |
| /* Get vectorized definitions for OP0 and OP1. |
| REDUC_INDEX is the index of reduction operand in case of reduction, |
| and -1 otherwise. */ |
| |
| void |
| vect_get_vec_defs (tree op0, tree op1, gimple *stmt, |
| vec<tree> *vec_oprnds0, |
| vec<tree> *vec_oprnds1, |
| slp_tree slp_node, int reduc_index) |
| { |
| if (slp_node) |
| { |
| int nops = (op1 == NULL_TREE) ? 1 : 2; |
| auto_vec<tree> ops (nops); |
| auto_vec<vec<tree> > vec_defs (nops); |
| |
| ops.quick_push (op0); |
| if (op1) |
| ops.quick_push (op1); |
| |
| vect_get_slp_defs (ops, slp_node, &vec_defs, reduc_index); |
| |
| *vec_oprnds0 = vec_defs[0]; |
| if (op1) |
| *vec_oprnds1 = vec_defs[1]; |
| } |
| else |
| { |
| tree vec_oprnd; |
| |
| vec_oprnds0->create (1); |
| vec_oprnd = vect_get_vec_def_for_operand (op0, stmt); |
| vec_oprnds0->quick_push (vec_oprnd); |
| |
| if (op1) |
| { |
| vec_oprnds1->create (1); |
| vec_oprnd = vect_get_vec_def_for_operand (op1, stmt); |
| vec_oprnds1->quick_push (vec_oprnd); |
| } |
| } |
| } |
| |
| |
| /* Function vect_finish_stmt_generation. |
| |
| Insert a new stmt. */ |
| |
| void |
| vect_finish_stmt_generation (gimple *stmt, gimple *vec_stmt, |
| gimple_stmt_iterator *gsi) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| vec_info *vinfo = stmt_info->vinfo; |
| |
| gcc_assert (gimple_code (stmt) != GIMPLE_LABEL); |
| |
| if (!gsi_end_p (*gsi) |
| && gimple_has_mem_ops (vec_stmt)) |
| { |
| gimple *at_stmt = gsi_stmt (*gsi); |
| tree vuse = gimple_vuse (at_stmt); |
| if (vuse && TREE_CODE (vuse) == SSA_NAME) |
| { |
| tree vdef = gimple_vdef (at_stmt); |
| gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt)); |
| /* If we have an SSA vuse and insert a store, update virtual |
| SSA form to avoid triggering the renamer. Do so only |
| if we can easily see all uses - which is what almost always |
| happens with the way vectorized stmts are inserted. */ |
| if ((vdef && TREE_CODE (vdef) == SSA_NAME) |
| && ((is_gimple_assign (vec_stmt) |
| && !is_gimple_reg (gimple_assign_lhs (vec_stmt))) |
| || (is_gimple_call (vec_stmt) |
| && !(gimple_call_flags (vec_stmt) |
| & (ECF_CONST|ECF_PURE|ECF_NOVOPS))))) |
| { |
| tree new_vdef = copy_ssa_name (vuse, vec_stmt); |
| gimple_set_vdef (vec_stmt, new_vdef); |
| SET_USE (gimple_vuse_op (at_stmt), new_vdef); |
| } |
| } |
| } |
| gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT); |
| |
| set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, vinfo)); |
| |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: "); |
| dump_gimple_stmt (MSG_NOTE, TDF_SLIM, vec_stmt, 0); |
| } |
| |
| gimple_set_location (vec_stmt, gimple_location (stmt)); |
| |
| /* While EH edges will generally prevent vectorization, stmt might |
| e.g. be in a must-not-throw region. Ensure newly created stmts |
| that could throw are part of the same region. */ |
| int lp_nr = lookup_stmt_eh_lp (stmt); |
| if (lp_nr != 0 && stmt_could_throw_p (vec_stmt)) |
| add_stmt_to_eh_lp (vec_stmt, lp_nr); |
| } |
| |
| /* We want to vectorize a call to combined function CFN with function |
| decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN |
| as the types of all inputs. Check whether this is possible using |
| an internal function, returning its code if so or IFN_LAST if not. */ |
| |
| static internal_fn |
| vectorizable_internal_function (combined_fn cfn, tree fndecl, |
| tree vectype_out, tree vectype_in) |
| { |
| internal_fn ifn; |
| if (internal_fn_p (cfn)) |
| ifn = as_internal_fn (cfn); |
| else |
| ifn = associated_internal_fn (fndecl); |
| if (ifn != IFN_LAST && direct_internal_fn_p (ifn)) |
| { |
| const direct_internal_fn_info &info = direct_internal_fn (ifn); |
| if (info.vectorizable) |
| { |
| tree type0 = (info.type0 < 0 ? vectype_out : vectype_in); |
| tree type1 = (info.type1 < 0 ? vectype_out : vectype_in); |
| if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1), |
| OPTIMIZE_FOR_SPEED)) |
| return ifn; |
| } |
| } |
| return IFN_LAST; |
| } |
| |
| |
| static tree permute_vec_elements (tree, tree, tree, gimple *, |
| gimple_stmt_iterator *); |
| |
| /* STMT is a non-strided load or store, meaning that it accesses |
| elements with a known constant step. Return -1 if that step |
| is negative, 0 if it is zero, and 1 if it is greater than zero. */ |
| |
| static int |
| compare_step_with_zero (gimple *stmt) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| tree step; |
| if (loop_vinfo && nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) |
| step = STMT_VINFO_DR_STEP (stmt_info); |
| else |
| step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); |
| return tree_int_cst_compare (step, size_zero_node); |
| } |
| |
| /* If the target supports a permute mask that reverses the elements in |
| a vector of type VECTYPE, return that mask, otherwise return null. */ |
| |
| static tree |
| perm_mask_for_reverse (tree vectype) |
| { |
| int i, nunits; |
| unsigned char *sel; |
| |
| nunits = TYPE_VECTOR_SUBPARTS (vectype); |
| sel = XALLOCAVEC (unsigned char, nunits); |
| |
| for (i = 0; i < nunits; ++i) |
| sel[i] = nunits - 1 - i; |
| |
| if (!can_vec_perm_p (TYPE_MODE (vectype), false, sel)) |
| return NULL_TREE; |
| return vect_gen_perm_mask_checked (vectype, sel); |
| } |
| |
| /* A subroutine of get_load_store_type, with a subset of the same |
| arguments. Handle the case where STMT is part of a grouped load |
| or store. |
| |
| For stores, the statements in the group are all consecutive |
| and there is no gap at the end. For loads, the statements in the |
| group might not be consecutive; there can be gaps between statements |
| as well as at the end. */ |
| |
| static bool |
| get_group_load_store_type (gimple *stmt, tree vectype, bool slp, |
| vec_load_store_type vls_type, |
| vect_memory_access_type *memory_access_type) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| vec_info *vinfo = stmt_info->vinfo; |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; |
| gimple *first_stmt = GROUP_FIRST_ELEMENT (stmt_info); |
| unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); |
| bool single_element_p = (stmt == first_stmt |
| && !GROUP_NEXT_ELEMENT (stmt_info)); |
| unsigned HOST_WIDE_INT gap = GROUP_GAP (vinfo_for_stmt (first_stmt)); |
| unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype); |
| |
| /* True if the vectorized statements would access beyond the last |
| statement in the group. */ |
| bool overrun_p = false; |
| |
| /* True if we can cope with such overrun by peeling for gaps, so that |
| there is at least one final scalar iteration after the vector loop. */ |
| bool can_overrun_p = (vls_type == VLS_LOAD && loop_vinfo && !loop->inner); |
| |
| /* There can only be a gap at the end of the group if the stride is |
| known at compile time. */ |
| gcc_assert (!STMT_VINFO_STRIDED_P (stmt_info) || gap == 0); |
| |
| /* Stores can't yet have gaps. */ |
| gcc_assert (slp || vls_type == VLS_LOAD || gap == 0); |
| |
| if (slp) |
| { |
| if (STMT_VINFO_STRIDED_P (stmt_info)) |
| { |
| /* Try to use consecutive accesses of GROUP_SIZE elements, |
| separated by the stride, until we have a complete vector. |
| Fall back to scalar accesses if that isn't possible. */ |
| if (nunits % group_size == 0) |
| *memory_access_type = VMAT_STRIDED_SLP; |
| else |
| *memory_access_type = VMAT_ELEMENTWISE; |
| } |
| else |
| { |
| overrun_p = loop_vinfo && gap != 0; |
| if (overrun_p && vls_type != VLS_LOAD) |
| { |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "Grouped store with gaps requires" |
| " non-consecutive accesses\n"); |
| return false; |
| } |
| /* If the access is aligned an overrun is fine. */ |
| if (overrun_p |
| && aligned_access_p |
| (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))) |
| overrun_p = false; |
| if (overrun_p && !can_overrun_p) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "Peeling for outer loop is not supported\n"); |
| return false; |
| } |
| *memory_access_type = VMAT_CONTIGUOUS; |
| } |
| } |
| else |
| { |
| /* We can always handle this case using elementwise accesses, |
| but see if something more efficient is available. */ |
| *memory_access_type = VMAT_ELEMENTWISE; |
| |
| /* If there is a gap at the end of the group then these optimizations |
| would access excess elements in the last iteration. */ |
| bool would_overrun_p = (gap != 0); |
| /* If the access is aligned an overrun is fine, but only if the |
| overrun is not inside an unused vector (if the gap is as large |
| or larger than a vector). */ |
| if (would_overrun_p |
| && gap < nunits |
| && aligned_access_p |
| (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))) |
| would_overrun_p = false; |
| if (!STMT_VINFO_STRIDED_P (stmt_info) |
| && (can_overrun_p || !would_overrun_p) |
| && compare_step_with_zero (stmt) > 0) |
| { |
| /* First try using LOAD/STORE_LANES. */ |
| if (vls_type == VLS_LOAD |
| ? vect_load_lanes_supported (vectype, group_size) |
| : vect_store_lanes_supported (vectype, group_size)) |
| { |
| *memory_access_type = VMAT_LOAD_STORE_LANES; |
| overrun_p = would_overrun_p; |
| } |
| |
| /* If that fails, try using permuting loads. */ |
| if (*memory_access_type == VMAT_ELEMENTWISE |
| && (vls_type == VLS_LOAD |
| ? vect_grouped_load_supported (vectype, single_element_p, |
| group_size) |
| : vect_grouped_store_supported (vectype, group_size))) |
| { |
| *memory_access_type = VMAT_CONTIGUOUS_PERMUTE; |
| overrun_p = would_overrun_p; |
| } |
| } |
| } |
| |
| if (vls_type != VLS_LOAD && first_stmt == stmt) |
| { |
| /* STMT is the leader of the group. Check the operands of all the |
| stmts of the group. */ |
| gimple *next_stmt = GROUP_NEXT_ELEMENT (stmt_info); |
| while (next_stmt) |
| { |
| gcc_assert (gimple_assign_single_p (next_stmt)); |
| tree op = gimple_assign_rhs1 (next_stmt); |
| gimple *def_stmt; |
| enum vect_def_type dt; |
| if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "use not simple.\n"); |
| return false; |
| } |
| next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); |
| } |
| } |
| |
| if (overrun_p) |
| { |
| gcc_assert (can_overrun_p); |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "Data access with gaps requires scalar " |
| "epilogue loop\n"); |
| LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true; |
| } |
| |
| return true; |
| } |
| |
| /* A subroutine of get_load_store_type, with a subset of the same |
| arguments. Handle the case where STMT is a load or store that |
| accesses consecutive elements with a negative step. */ |
| |
| static vect_memory_access_type |
| get_negative_load_store_type (gimple *stmt, tree vectype, |
| vec_load_store_type vls_type, |
| unsigned int ncopies) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); |
| dr_alignment_support alignment_support_scheme; |
| |
| if (ncopies > 1) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "multiple types with negative step.\n"); |
| return VMAT_ELEMENTWISE; |
| } |
| |
| alignment_support_scheme = vect_supportable_dr_alignment (dr, false); |
| if (alignment_support_scheme != dr_aligned |
| && alignment_support_scheme != dr_unaligned_supported) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "negative step but alignment required.\n"); |
| return VMAT_ELEMENTWISE; |
| } |
| |
| if (vls_type == VLS_STORE_INVARIANT) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, |
| "negative step with invariant source;" |
| " no permute needed.\n"); |
| return VMAT_CONTIGUOUS_DOWN; |
| } |
| |
| if (!perm_mask_for_reverse (vectype)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "negative step and reversing not supported.\n"); |
| return VMAT_ELEMENTWISE; |
| } |
| |
| return VMAT_CONTIGUOUS_REVERSE; |
| } |
| |
| /* Analyze load or store statement STMT of type VLS_TYPE. Return true |
| if there is a memory access type that the vectorized form can use, |
| storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers |
| or scatters, fill in GS_INFO accordingly. |
| |
| SLP says whether we're performing SLP rather than loop vectorization. |
| VECTYPE is the vector type that the vectorized statements will use. |
| NCOPIES is the number of vector statements that will be needed. */ |
| |
| static bool |
| get_load_store_type (gimple *stmt, tree vectype, bool slp, |
| vec_load_store_type vls_type, unsigned int ncopies, |
| vect_memory_access_type *memory_access_type, |
| gather_scatter_info *gs_info) |
| { |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| vec_info *vinfo = stmt_info->vinfo; |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) |
| { |
| *memory_access_type = VMAT_GATHER_SCATTER; |
| gimple *def_stmt; |
| if (!vect_check_gather_scatter (stmt, loop_vinfo, gs_info)) |
| gcc_unreachable (); |
| else if (!vect_is_simple_use (gs_info->offset, vinfo, &def_stmt, |
| &gs_info->offset_dt, |
| &gs_info->offset_vectype)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "%s index use not simple.\n", |
| vls_type == VLS_LOAD ? "gather" : "scatter"); |
| return false; |
| } |
| } |
| else if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) |
| { |
| if (!get_group_load_store_type (stmt, vectype, slp, vls_type, |
| memory_access_type)) |
| return false; |
| } |
| else if (STMT_VINFO_STRIDED_P (stmt_info)) |
| { |
| gcc_assert (!slp); |
| *memory_access_type = VMAT_ELEMENTWISE; |
| } |
| else |
| { |
| int cmp = compare_step_with_zero (stmt); |
| if (cmp < 0) |
| *memory_access_type = get_negative_load_store_type |
| (stmt, vectype, vls_type, ncopies); |
| else if (cmp == 0) |
| { |
| gcc_assert (vls_type == VLS_LOAD); |
| *memory_access_type = VMAT_INVARIANT; |
| } |
| else |
| *memory_access_type = VMAT_CONTIGUOUS; |
| } |
| |
| /* FIXME: At the moment the cost model seems to underestimate the |
| cost of using elementwise accesses. This check preserves the |
| traditional behavior until that can be fixed. */ |
| if (*memory_access_type == VMAT_ELEMENTWISE |
| && !STMT_VINFO_STRIDED_P (stmt_info)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "not falling back to elementwise accesses\n"); |
| return false; |
| } |
| return true; |
| } |
| |
| /* Function vectorizable_mask_load_store. |
| |
| Check if STMT performs a conditional load or store that can be vectorized. |
| If VEC_STMT is also passed, vectorize the STMT: create a vectorized |
| stmt to replace it, put it in VEC_STMT, and insert it at GSI. |
| Return FALSE if not a vectorizable STMT, TRUE otherwise. */ |
| |
| static bool |
| vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, |
| gimple **vec_stmt, slp_tree slp_node) |
| { |
| tree vec_dest = NULL; |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| stmt_vec_info prev_stmt_info; |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); |
| bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt); |
| struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); |
| tree vectype = STMT_VINFO_VECTYPE (stmt_info); |
| tree rhs_vectype = NULL_TREE; |
| tree mask_vectype; |
| tree elem_type; |
| gimple *new_stmt; |
| tree dummy; |
| tree dataref_ptr = NULL_TREE; |
| gimple *ptr_incr; |
| int nunits = TYPE_VECTOR_SUBPARTS (vectype); |
| int ncopies; |
| int i, j; |
| bool inv_p; |
| gather_scatter_info gs_info; |
| vec_load_store_type vls_type; |
| tree mask; |
| gimple *def_stmt; |
| enum vect_def_type dt; |
| |
| if (slp_node != NULL) |
| return false; |
| |
| ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits; |
| gcc_assert (ncopies >= 1); |
| |
| mask = gimple_call_arg (stmt, 2); |
| |
| if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask))) |
| return false; |
| |
| /* FORNOW. This restriction should be relaxed. */ |
| if (nested_in_vect_loop && ncopies > 1) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "multiple types in nested loop."); |
| return false; |
| } |
| |
| if (!STMT_VINFO_RELEVANT_P (stmt_info)) |
| return false; |
| |
| if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def |
| && ! vec_stmt) |
| return false; |
| |
| if (!STMT_VINFO_DATA_REF (stmt_info)) |
| return false; |
| |
| elem_type = TREE_TYPE (vectype); |
| |
| if (TREE_CODE (mask) != SSA_NAME) |
| return false; |
| |
| if (!vect_is_simple_use (mask, loop_vinfo, &def_stmt, &dt, &mask_vectype)) |
| return false; |
| |
| if (!mask_vectype) |
| mask_vectype = get_mask_type_for_scalar_type (TREE_TYPE (vectype)); |
| |
| if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype) |
| || TYPE_VECTOR_SUBPARTS (mask_vectype) != TYPE_VECTOR_SUBPARTS (vectype)) |
| return false; |
| |
| if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE) |
| { |
| tree rhs = gimple_call_arg (stmt, 3); |
| if (!vect_is_simple_use (rhs, loop_vinfo, &def_stmt, &dt, &rhs_vectype)) |
| return false; |
| if (dt == vect_constant_def || dt == vect_external_def) |
| vls_type = VLS_STORE_INVARIANT; |
| else |
| vls_type = VLS_STORE; |
| } |
| else |
| vls_type = VLS_LOAD; |
| |
| vect_memory_access_type memory_access_type; |
| if (!get_load_store_type (stmt, vectype, false, vls_type, ncopies, |
| &memory_access_type, &gs_info)) |
| return false; |
| |
| if (memory_access_type == VMAT_GATHER_SCATTER) |
| { |
| tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); |
| tree masktype |
| = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist)))); |
| if (TREE_CODE (masktype) == INTEGER_TYPE) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "masked gather with integer mask not supported."); |
| return false; |
| } |
| } |
| else if (memory_access_type != VMAT_CONTIGUOUS) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "unsupported access type for masked %s.\n", |
| vls_type == VLS_LOAD ? "load" : "store"); |
| return false; |
| } |
| else if (!VECTOR_MODE_P (TYPE_MODE (vectype)) |
| || !can_vec_mask_load_store_p (TYPE_MODE (vectype), |
| TYPE_MODE (mask_vectype), |
| vls_type == VLS_LOAD) |
| || (rhs_vectype |
| && !useless_type_conversion_p (vectype, rhs_vectype))) |
| return false; |
| |
| if (!vec_stmt) /* transformation not required. */ |
| { |
| STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type; |
| STMT_VINFO_TYPE (stmt_info) = call_vec_info_type; |
| if (vls_type == VLS_LOAD) |
| vect_model_load_cost (stmt_info, ncopies, memory_access_type, |
| NULL, NULL, NULL); |
| else |
| vect_model_store_cost (stmt_info, ncopies, memory_access_type, |
| dt, NULL, NULL, NULL); |
| return true; |
| } |
| gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)); |
| |
| /** Transform. **/ |
| |
| if (memory_access_type == VMAT_GATHER_SCATTER) |
| { |
| tree vec_oprnd0 = NULL_TREE, op; |
| tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl)); |
| tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; |
| tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale; |
| tree perm_mask = NULL_TREE, prev_res = NULL_TREE; |
| tree mask_perm_mask = NULL_TREE; |
| edge pe = loop_preheader_edge (loop); |
| gimple_seq seq; |
| basic_block new_bb; |
| enum { NARROW, NONE, WIDEN } modifier; |
| int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype); |
| |
| rettype = TREE_TYPE (TREE_TYPE (gs_info.decl)); |
| srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); |
| ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); |
| idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); |
| masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); |
| scaletype = TREE_VALUE (arglist); |
| gcc_checking_assert (types_compatible_p (srctype, rettype) |
| && types_compatible_p (srctype, masktype)); |
| |
| if (nunits == gather_off_nunits) |
| modifier = NONE; |
| else if (nunits == gather_off_nunits / 2) |
| { |
| unsigned char *sel = XALLOCAVEC (unsigned char, gather_off_nunits); |
| modifier = WIDEN; |
| |
| for (i = 0; i < gather_off_nunits; ++i) |
| sel[i] = i | nunits; |
| |
| perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel); |
| } |
| else if (nunits == gather_off_nunits * 2) |
| { |
| unsigned char *sel = XALLOCAVEC (unsigned char, nunits); |
| modifier = NARROW; |
| |
| for (i = 0; i < nunits; ++i) |
| sel[i] = i < gather_off_nunits |
| ? i : i + nunits - gather_off_nunits; |
| |
| perm_mask = vect_gen_perm_mask_checked (vectype, sel); |
| ncopies *= 2; |
| for (i = 0; i < nunits; ++i) |
| sel[i] = i | gather_off_nunits; |
| mask_perm_mask = vect_gen_perm_mask_checked (masktype, sel); |
| } |
| else |
| gcc_unreachable (); |
| |
| vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype); |
| |
| ptr = fold_convert (ptrtype, gs_info.base); |
| if (!is_gimple_min_invariant (ptr)) |
| { |
| ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); |
| new_bb = gsi_insert_seq_on_edge_immediate (pe, seq); |
| gcc_assert (!new_bb); |
| } |
| |
| scale = build_int_cst (scaletype, gs_info.scale); |
| |
| prev_stmt_info = NULL; |
| for (j = 0; j < ncopies; ++j) |
| { |
| if (modifier == WIDEN && (j & 1)) |
| op = permute_vec_elements (vec_oprnd0, vec_oprnd0, |
| perm_mask, stmt, gsi); |
| else if (j == 0) |
| op = vec_oprnd0 |
| = vect_get_vec_def_for_operand (gs_info.offset, stmt); |
| else |
| op = vec_oprnd0 |
| = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0); |
| |
| if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) |
| { |
| gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)) |
| == TYPE_VECTOR_SUBPARTS (idxtype)); |
| var = vect_get_new_ssa_name (idxtype, vect_simple_var); |
| op = build1 (VIEW_CONVERT_EXPR, idxtype, op); |
| new_stmt |
| = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| op = var; |
| } |
| |
| if (mask_perm_mask && (j & 1)) |
| mask_op = permute_vec_elements (mask_op, mask_op, |
| mask_perm_mask, stmt, gsi); |
| else |
| { |
| if (j == 0) |
| vec_mask = vect_get_vec_def_for_operand (mask, stmt); |
| else |
| { |
| vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt); |
| vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask); |
| } |
| |
| mask_op = vec_mask; |
| if (!useless_type_conversion_p (masktype, TREE_TYPE (vec_mask))) |
| { |
| gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op)) |
| == TYPE_VECTOR_SUBPARTS (masktype)); |
| var = vect_get_new_ssa_name (masktype, vect_simple_var); |
| mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op); |
| new_stmt |
| = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_op); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| mask_op = var; |
| } |
| } |
| |
| new_stmt |
| = gimple_build_call (gs_info.decl, 5, mask_op, ptr, op, mask_op, |
| scale); |
| |
| if (!useless_type_conversion_p (vectype, rettype)) |
| { |
| gcc_assert (TYPE_VECTOR_SUBPARTS (vectype) |
| == TYPE_VECTOR_SUBPARTS (rettype)); |
| op = vect_get_new_ssa_name (rettype, vect_simple_var); |
| gimple_call_set_lhs (new_stmt, op); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| var = make_ssa_name (vec_dest); |
| op = build1 (VIEW_CONVERT_EXPR, vectype, op); |
| new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); |
| } |
| else |
| { |
| var = make_ssa_name (vec_dest, new_stmt); |
| gimple_call_set_lhs (new_stmt, var); |
| } |
| |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| |
| if (modifier == NARROW) |
| { |
| if ((j & 1) == 0) |
| { |
| prev_res = var; |
| continue; |
| } |
| var = permute_vec_elements (prev_res, var, |
| perm_mask, stmt, gsi); |
| new_stmt = SSA_NAME_DEF_STMT (var); |
| } |
| |
| if (prev_stmt_info == NULL) |
| STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| |
| /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed |
| from the IL. */ |
| if (STMT_VINFO_RELATED_STMT (stmt_info)) |
| { |
| stmt = STMT_VINFO_RELATED_STMT (stmt_info); |
| stmt_info = vinfo_for_stmt (stmt); |
| } |
| tree lhs = gimple_call_lhs (stmt); |
| new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); |
| set_vinfo_for_stmt (new_stmt, stmt_info); |
| set_vinfo_for_stmt (stmt, NULL); |
| STMT_VINFO_STMT (stmt_info) = new_stmt; |
| gsi_replace (gsi, new_stmt, true); |
| return true; |
| } |
| else if (vls_type != VLS_LOAD) |
| { |
| tree vec_rhs = NULL_TREE, vec_mask = NULL_TREE; |
| prev_stmt_info = NULL; |
| LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true; |
| for (i = 0; i < ncopies; i++) |
| { |
| unsigned align, misalign; |
| |
| if (i == 0) |
| { |
| tree rhs = gimple_call_arg (stmt, 3); |
| vec_rhs = vect_get_vec_def_for_operand (rhs, stmt); |
| vec_mask = vect_get_vec_def_for_operand (mask, stmt); |
| /* We should have catched mismatched types earlier. */ |
| gcc_assert (useless_type_conversion_p (vectype, |
| TREE_TYPE (vec_rhs))); |
| dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL, |
| NULL_TREE, &dummy, gsi, |
| &ptr_incr, false, &inv_p); |
| gcc_assert (!inv_p); |
| } |
| else |
| { |
| vect_is_simple_use (vec_rhs, loop_vinfo, &def_stmt, &dt); |
| vec_rhs = vect_get_vec_def_for_stmt_copy (dt, vec_rhs); |
| vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt); |
| vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask); |
| dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, |
| TYPE_SIZE_UNIT (vectype)); |
| } |
| |
| align = TYPE_ALIGN_UNIT (vectype); |
| if (aligned_access_p (dr)) |
| misalign = 0; |
| else if (DR_MISALIGNMENT (dr) == -1) |
| { |
| align = TYPE_ALIGN_UNIT (elem_type); |
| misalign = 0; |
| } |
| else |
| misalign = DR_MISALIGNMENT (dr); |
| set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, |
| misalign); |
| tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), |
| misalign ? least_bit_hwi (misalign) : align); |
| new_stmt |
| = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr, |
| ptr, vec_mask, vec_rhs); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| if (i == 0) |
| STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| } |
| else |
| { |
| tree vec_mask = NULL_TREE; |
| prev_stmt_info = NULL; |
| vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype); |
| for (i = 0; i < ncopies; i++) |
| { |
| unsigned align, misalign; |
| |
| if (i == 0) |
| { |
| vec_mask = vect_get_vec_def_for_operand (mask, stmt); |
| dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL, |
| NULL_TREE, &dummy, gsi, |
| &ptr_incr, false, &inv_p); |
| gcc_assert (!inv_p); |
| } |
| else |
| { |
| vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt); |
| vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask); |
| dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, |
| TYPE_SIZE_UNIT (vectype)); |
| } |
| |
| align = TYPE_ALIGN_UNIT (vectype); |
| if (aligned_access_p (dr)) |
| misalign = 0; |
| else if (DR_MISALIGNMENT (dr) == -1) |
| { |
| align = TYPE_ALIGN_UNIT (elem_type); |
| misalign = 0; |
| } |
| else |
| misalign = DR_MISALIGNMENT (dr); |
| set_ptr_info_alignment (get_ptr_info (dataref_ptr), align, |
| misalign); |
| tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), |
| misalign ? least_bit_hwi (misalign) : align); |
| new_stmt |
| = gimple_build_call_internal (IFN_MASK_LOAD, 3, dataref_ptr, |
| ptr, vec_mask); |
| gimple_call_set_lhs (new_stmt, make_ssa_name (vec_dest)); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| if (i == 0) |
| STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| } |
| |
| if (vls_type == VLS_LOAD) |
| { |
| /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed |
| from the IL. */ |
| if (STMT_VINFO_RELATED_STMT (stmt_info)) |
| { |
| stmt = STMT_VINFO_RELATED_STMT (stmt_info); |
| stmt_info = vinfo_for_stmt (stmt); |
| } |
| tree lhs = gimple_call_lhs (stmt); |
| new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); |
| set_vinfo_for_stmt (new_stmt, stmt_info); |
| set_vinfo_for_stmt (stmt, NULL); |
| STMT_VINFO_STMT (stmt_info) = new_stmt; |
| gsi_replace (gsi, new_stmt, true); |
| } |
| |
| return true; |
| } |
| |
| /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */ |
| |
| static bool |
| vectorizable_bswap (gimple *stmt, gimple_stmt_iterator *gsi, |
| gimple **vec_stmt, slp_tree slp_node, |
| tree vectype_in, enum vect_def_type *dt) |
| { |
| tree op, vectype; |
| stmt_vec_info stmt_info = vinfo_for_stmt (stmt); |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| unsigned ncopies, nunits; |
| |
| op = gimple_call_arg (stmt, 0); |
| vectype = STMT_VINFO_VECTYPE (stmt_info); |
| nunits = TYPE_VECTOR_SUBPARTS (vectype); |
| |
| /* Multiple types in SLP are handled by creating the appropriate number of |
| vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in |
| case of SLP. */ |
| if (slp_node) |
| ncopies = 1; |
| else |
| ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits; |
| |
| gcc_assert (ncopies >= 1); |
| |
| tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in); |
| if (! char_vectype) |
| return false; |
| |
| unsigned char *elts |
| = XALLOCAVEC (unsigned char, TYPE_VECTOR_SUBPARTS (char_vectype)); |
| unsigned char *elt = elts; |
| unsigned word_bytes = TYPE_VECTOR_SUBPARTS (char_vectype) / nunits; |
| for (unsigned i = 0; i < nunits; ++i) |
| for (unsigned j = 0; j < word_bytes; ++j) |
| *elt++ = (i + 1) * word_bytes - j - 1; |
| |
| if (! can_vec_perm_p (TYPE_MODE (char_vectype), false, elts)) |
| return false; |
| |
| if (! vec_stmt) |
| { |
| STMT_VINFO_TYPE (stmt_info) = call_vec_info_type; |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_bswap ===" |
| "\n"); |
| if (! PURE_SLP_STMT (stmt_info)) |
| { |
| add_stmt_cost (stmt_info->vinfo->target_cost_data, |
| 1, vector_stmt, stmt_info, 0, vect_prologue); |
| add_stmt_cost (stmt_info->vinfo->target_cost_data, |
| ncopies, vec_perm, stmt_info, 0, vect_body); |
| } |
| return true; |
| } |
| |
| tree *telts = XALLOCAVEC (tree, TYPE_VECTOR_SUBPARTS (char_vectype)); |
| for (unsigned i = 0; i < TYPE_VECTOR_SUBPARTS (char_vectype); ++i) |
| telts[i] = build_int_cst (char_type_node, elts[i]); |
| tree bswap_vconst = build_vector (char_vectype, telts); |
| |
| /* Transform. */ |
| vec<tree> vec_oprnds = vNULL; |
| gimple *new_stmt = NULL; |
| stmt_vec_info prev_stmt_info = NULL; |
| for (unsigned j = 0; j < ncopies; j++) |
| { |
| /* Handle uses. */ |
| if (j == 0) |
| vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node, -1); |
| else |
| vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL); |
| |
| /* Arguments are ready. create the new vector stmt. */ |
| unsigned i; |
| tree vop; |
| FOR_EACH_VEC_ELT (vec_oprnds, i, vop) |
| { |
| tree tem = make_ssa_name (char_vectype); |
| new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR, |
| char_vectype, vop)); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| tree tem2 = make_ssa_name (char_vectype); |
| new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR, |
| tem, tem, bswap_vconst); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| tem = make_ssa_name (vectype); |
| new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR, |
| vectype, tem2)); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| if (slp_node) |
| SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); |
| } |
| |
| if (slp_node) |
| continue; |
| |
| if (j == 0) |
| STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| |
| vec_oprnds.release (); |
| return true; |
| } |
| |
| /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have |
| integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT |
| in a single step. On success, store the binary pack code in |
| *CONVERT_CODE. */ |
| |
| static bool |
| simple_integer_narrowing (tree vectype_out, tree vectype_in, |
| tree_code *convert_code) |
| { |
| if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out)) |
| || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in))) |
| return false; |
| |
| tree_code code; |
| int multi_step_cvt = 0; |
| auto_vec <tree, 8> interm_types; |
| if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in, |
| &code, &multi_step_cvt, |
| &interm_types) |
| || multi_step_cvt) |
| return false; |
| |
| *convert_code = code; |
| return true; |
| } |
| |
| /* Function vectorizable_call. |
| |
| Check if GS performs a function call that can be vectorized. |
| If VEC_STMT is also passed, vectorize the STMT: create a vectorized |
| stmt to replace it, put it in VEC_STMT, and insert it at BSI. |
| Return FALSE if not a vectorizable STMT, TRUE otherwise. */ |
| |
| static bool |
| vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, gimple **vec_stmt, |
| slp_tree slp_node) |
| { |
| gcall *stmt; |
| tree vec_dest; |
| tree scalar_dest; |
| tree op, type; |
| tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE; |
| stmt_vec_info stmt_info = vinfo_for_stmt (gs), prev_stmt_info; |
| tree vectype_out, vectype_in; |
| int nunits_in; |
| int nunits_out; |
| loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); |
| bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); |
| vec_info *vinfo = stmt_info->vinfo; |
| tree fndecl, new_temp, rhs_type; |
| gimple *def_stmt; |
| enum vect_def_type dt[3] |
| = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type}; |
| gimple *new_stmt = NULL; |
| int ncopies, j; |
| vec<tree> vargs = vNULL; |
| enum { NARROW, NONE, WIDEN } modifier; |
| size_t i, nargs; |
| tree lhs; |
| |
| if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) |
| return false; |
| |
| if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def |
| && ! vec_stmt) |
| return false; |
| |
| /* Is GS a vectorizable call? */ |
| stmt = dyn_cast <gcall *> (gs); |
| if (!stmt) |
| return false; |
| |
| if (gimple_call_internal_p (stmt) |
| && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD |
| || gimple_call_internal_fn (stmt) == IFN_MASK_STORE)) |
| return vectorizable_mask_load_store (stmt, gsi, vec_stmt, |
| slp_node); |
| |
| if (gimple_call_lhs (stmt) == NULL_TREE |
| || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME) |
| return false; |
| |
| gcc_checking_assert (!stmt_can_throw_internal (stmt)); |
| |
| vectype_out = STMT_VINFO_VECTYPE (stmt_info); |
| |
| /* Process function arguments. */ |
| rhs_type = NULL_TREE; |
| vectype_in = NULL_TREE; |
| nargs = gimple_call_num_args (stmt); |
| |
| /* Bail out if the function has more than three arguments, we do not have |
| interesting builtin functions to vectorize with more than two arguments |
| except for fma. No arguments is also not good. */ |
| if (nargs == 0 || nargs > 3) |
| return false; |
| |
| /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */ |
| if (gimple_call_internal_p (stmt) |
| && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE) |
| { |
| nargs = 0; |
| rhs_type = unsigned_type_node; |
| } |
| |
| for (i = 0; i < nargs; i++) |
| { |
| tree opvectype; |
| |
| op = gimple_call_arg (stmt, i); |
| |
| /* We can only handle calls with arguments of the same type. */ |
| if (rhs_type |
| && !types_compatible_p (rhs_type, TREE_TYPE (op))) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "argument types differ.\n"); |
| return false; |
| } |
| if (!rhs_type) |
| rhs_type = TREE_TYPE (op); |
| |
| if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[i], &opvectype)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "use not simple.\n"); |
| return false; |
| } |
| |
| if (!vectype_in) |
| vectype_in = opvectype; |
| else if (opvectype |
| && opvectype != vectype_in) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "argument vector types differ.\n"); |
| return false; |
| } |
| } |
| /* If all arguments are external or constant defs use a vector type with |
| the same size as the output vector type. */ |
| if (!vectype_in) |
| vectype_in = get_same_sized_vectype (rhs_type, vectype_out); |
| if (vec_stmt) |
| gcc_assert (vectype_in); |
| if (!vectype_in) |
| { |
| if (dump_enabled_p ()) |
| { |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "no vectype for scalar type "); |
| dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type); |
| dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); |
| } |
| |
| return false; |
| } |
| |
| /* FORNOW */ |
| nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); |
| nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); |
| if (nunits_in == nunits_out / 2) |
| modifier = NARROW; |
| else if (nunits_out == nunits_in) |
| modifier = NONE; |
| else if (nunits_out == nunits_in / 2) |
| modifier = WIDEN; |
| else |
| return false; |
| |
| /* We only handle functions that do not read or clobber memory. */ |
| if (gimple_vuse (stmt)) |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "function reads from or writes to memory.\n"); |
| return false; |
| } |
| |
| /* For now, we only vectorize functions if a target specific builtin |
| is available. TODO -- in some cases, it might be profitable to |
| insert the calls for pieces of the vector, in order to be able |
| to vectorize other operations in the loop. */ |
| fndecl = NULL_TREE; |
| internal_fn ifn = IFN_LAST; |
| combined_fn cfn = gimple_call_combined_fn (stmt); |
| tree callee = gimple_call_fndecl (stmt); |
| |
| /* First try using an internal function. */ |
| tree_code convert_code = ERROR_MARK; |
| if (cfn != CFN_LAST |
| && (modifier == NONE |
| || (modifier == NARROW |
| && simple_integer_narrowing (vectype_out, vectype_in, |
| &convert_code)))) |
| ifn = vectorizable_internal_function (cfn, callee, vectype_out, |
| vectype_in); |
| |
| /* If that fails, try asking for a target-specific built-in function. */ |
| if (ifn == IFN_LAST) |
| { |
| if (cfn != CFN_LAST) |
| fndecl = targetm.vectorize.builtin_vectorized_function |
| (cfn, vectype_out, vectype_in); |
| else if (callee) |
| fndecl = targetm.vectorize.builtin_md_vectorized_function |
| (callee, vectype_out, vectype_in); |
| } |
| |
| if (ifn == IFN_LAST && !fndecl) |
| { |
| if (cfn == CFN_GOMP_SIMD_LANE |
| && !slp_node |
| && loop_vinfo |
| && LOOP_VINFO_LOOP (loop_vinfo)->simduid |
| && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME |
| && LOOP_VINFO_LOOP (loop_vinfo)->simduid |
| == SSA_NAME_VAR (gimple_call_arg (stmt, 0))) |
| { |
| /* We can handle IFN_GOMP_SIMD_LANE by returning a |
| { 0, 1, 2, ... vf - 1 } vector. */ |
| gcc_assert (nargs == 0); |
| } |
| else if (modifier == NONE |
| && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16) |
| || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32) |
| || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64))) |
| return vectorizable_bswap (stmt, gsi, vec_stmt, slp_node, |
| vectype_in, dt); |
| else |
| { |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, |
| "function is not vectorizable.\n"); |
| return false; |
| } |
| } |
| |
| if (slp_node) |
| ncopies = 1; |
| else if (modifier == NARROW && ifn == IFN_LAST) |
| ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out; |
| else |
| ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in; |
| |
| /* Sanity check: make sure that at least one copy of the vectorized stmt |
| needs to be generated. */ |
| gcc_assert (ncopies >= 1); |
| |
| if (!vec_stmt) /* transformation not required. */ |
| { |
| STMT_VINFO_TYPE (stmt_info) = call_vec_info_type; |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_call ===" |
| "\n"); |
| vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); |
| if (ifn != IFN_LAST && modifier == NARROW && !slp_node) |
| add_stmt_cost (stmt_info->vinfo->target_cost_data, ncopies / 2, |
| vec_promote_demote, stmt_info, 0, vect_body); |
| |
| return true; |
| } |
| |
| /** Transform. **/ |
| |
| if (dump_enabled_p ()) |
| dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n"); |
| |
| /* Handle def. */ |
| scalar_dest = gimple_call_lhs (stmt); |
| vec_dest = vect_create_destination_var (scalar_dest, vectype_out); |
| |
| prev_stmt_info = NULL; |
| if (modifier == NONE || ifn != IFN_LAST) |
| { |
| tree prev_res = NULL_TREE; |
| for (j = 0; j < ncopies; ++j) |
| { |
| /* Build argument list for the vectorized call. */ |
| if (j == 0) |
| vargs.create (nargs); |
| else |
| vargs.truncate (0); |
| |
| if (slp_node) |
| { |
| auto_vec<vec<tree> > vec_defs (nargs); |
| vec<tree> vec_oprnds0; |
| |
| for (i = 0; i < nargs; i++) |
| vargs.quick_push (gimple_call_arg (stmt, i)); |
| vect_get_slp_defs (vargs, slp_node, &vec_defs, -1); |
| vec_oprnds0 = vec_defs[0]; |
| |
| /* Arguments are ready. Create the new vector stmt. */ |
| FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0) |
| { |
| size_t k; |
| for (k = 0; k < nargs; k++) |
| { |
| vec<tree> vec_oprndsk = vec_defs[k]; |
| vargs[k] = vec_oprndsk[i]; |
| } |
| if (modifier == NARROW) |
| { |
| tree half_res = make_ssa_name (vectype_in); |
| new_stmt = gimple_build_call_internal_vec (ifn, vargs); |
| gimple_call_set_lhs (new_stmt, half_res); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| if ((i & 1) == 0) |
| { |
| prev_res = half_res; |
| continue; |
| } |
| new_temp = make_ssa_name (vec_dest); |
| new_stmt = gimple_build_assign (new_temp, convert_code, |
| prev_res, half_res); |
| } |
| else |
| { |
| if (ifn != IFN_LAST) |
| new_stmt = gimple_build_call_internal_vec (ifn, vargs); |
| else |
| new_stmt = gimple_build_call_vec (fndecl, vargs); |
| new_temp = make_ssa_name (vec_dest, new_stmt); |
| gimple_call_set_lhs (new_stmt, new_temp); |
| } |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); |
| } |
| |
| for (i = 0; i < nargs; i++) |
| { |
| vec<tree> vec_oprndsi = vec_defs[i]; |
| vec_oprndsi.release (); |
| } |
| continue; |
| } |
| |
| for (i = 0; i < nargs; i++) |
| { |
| op = gimple_call_arg (stmt, i); |
| if (j == 0) |
| vec_oprnd0 |
| = vect_get_vec_def_for_operand (op, stmt); |
| else |
| { |
| vec_oprnd0 = gimple_call_arg (new_stmt, i); |
| vec_oprnd0 |
| = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0); |
| } |
| |
| vargs.quick_push (vec_oprnd0); |
| } |
| |
| if (gimple_call_internal_p (stmt) |
| && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE) |
| { |
| tree *v = XALLOCAVEC (tree, nunits_out); |
| int k; |
| for (k = 0; k < nunits_out; ++k) |
| v[k] = build_int_cst (unsigned_type_node, j * nunits_out + k); |
| tree cst = build_vector (vectype_out, v); |
| tree new_var |
| = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_"); |
| gimple *init_stmt = gimple_build_assign (new_var, cst); |
| vect_init_vector_1 (stmt, init_stmt, NULL); |
| new_temp = make_ssa_name (vec_dest); |
| new_stmt = gimple_build_assign (new_temp, new_var); |
| } |
| else if (modifier == NARROW) |
| { |
| tree half_res = make_ssa_name (vectype_in); |
| new_stmt = gimple_build_call_internal_vec (ifn, vargs); |
| gimple_call_set_lhs (new_stmt, half_res); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| if ((j & 1) == 0) |
| { |
| prev_res = half_res; |
| continue; |
| } |
| new_temp = make_ssa_name (vec_dest); |
| new_stmt = gimple_build_assign (new_temp, convert_code, |
| prev_res, half_res); |
| } |
| else |
| { |
| if (ifn != IFN_LAST) |
| new_stmt = gimple_build_call_internal_vec (ifn, vargs); |
| else |
| new_stmt = gimple_build_call_vec (fndecl, vargs); |
| new_temp = make_ssa_name (vec_dest, new_stmt); |
| gimple_call_set_lhs (new_stmt, new_temp); |
| } |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| |
| if (j == (modifier == NARROW ? 1 : 0)) |
| STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| } |
| else if (modifier == NARROW) |
| { |
| for (j = 0; j < ncopies; ++j) |
| { |
| /* Build argument list for the vectorized call. */ |
| if (j == 0) |
| vargs.create (nargs * 2); |
| else |
| vargs.truncate (0); |
| |
| if (slp_node) |
| { |
| auto_vec<vec<tree> > vec_defs (nargs); |
| vec<tree> vec_oprnds0; |
| |
| for (i = 0; i < nargs; i++) |
| vargs.quick_push (gimple_call_arg (stmt, i)); |
| vect_get_slp_defs (vargs, slp_node, &vec_defs, -1); |
| vec_oprnds0 = vec_defs[0]; |
| |
| /* Arguments are ready. Create the new vector stmt. */ |
| for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2) |
| { |
| size_t k; |
| vargs.truncate (0); |
| for (k = 0; k < nargs; k++) |
| { |
| vec<tree> vec_oprndsk = vec_defs[k]; |
| vargs.quick_push (vec_oprndsk[i]); |
| vargs.quick_push (vec_oprndsk[i + 1]); |
| } |
| if (ifn != IFN_LAST) |
| new_stmt = gimple_build_call_internal_vec (ifn, vargs); |
| else |
| new_stmt = gimple_build_call_vec (fndecl, vargs); |
| new_temp = make_ssa_name (vec_dest, new_stmt); |
| gimple_call_set_lhs (new_stmt, new_temp); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); |
| } |
| |
| for (i = 0; i < nargs; i++) |
| { |
| vec<tree> vec_oprndsi = vec_defs[i]; |
| vec_oprndsi.release (); |
| } |
| continue; |
| } |
| |
| for (i = 0; i < nargs; i++) |
| { |
| op = gimple_call_arg (stmt, i); |
| if (j == 0) |
| { |
| vec_oprnd0 |
| = vect_get_vec_def_for_operand (op, stmt); |
| vec_oprnd1 |
| = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0); |
| } |
| else |
| { |
| vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1); |
| vec_oprnd0 |
| = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1); |
| vec_oprnd1 |
| = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0); |
| } |
| |
| vargs.quick_push (vec_oprnd0); |
| vargs.quick_push (vec_oprnd1); |
| } |
| |
| new_stmt = gimple_build_call_vec (fndecl, vargs); |
| new_temp = make_ssa_name (vec_dest, new_stmt); |
| gimple_call_set_lhs (new_stmt, new_temp); |
| vect_finish_stmt_generation (stmt, new_stmt, gsi); |
| |
| if (j == 0) |
| STMT_VINFO_VEC_STMT (stmt_info) = new_stmt; |
| else |
| STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; |
| |
| prev_stmt_info = vinfo_for_stmt (new_stmt); |
| } |
| |
| *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info); |
| } |
| else |
| /* No current target implements this case. */ |
| return false; |
| |
| vargs.release (); |
| |
| /* The call in STMT might prevent it from being removed in dce. |
| We however cannot remove it here, due to the way the ssa name |
| it defines is mapped to the new definition. So just replace |
| rhs of the statement with something harmless. */ |
| |
| if (slp_node) |
| return true; |
| |
| type = TREE_TYPE (scalar_dest); |
| if (is_pattern_stmt_p (stmt_info)) |
| lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info)); |
| else |
| lhs = gimple_call_lhs (stmt); |
| |
| new_stmt = gimple_build_assign (lhs, build_zero_cst (type)); |
| set_vinfo_for_stmt (new_stmt, stmt_info); |
| set_vinfo_for_stmt (stmt, NULL); |
| STMT_VINFO_STMT (stmt_info) = new_stmt; |
| gsi_replace (gsi, new_stmt, false); |
| |
| return true; |
| } |
| |
| |
| struct simd_call_arg_info |
| { |
| tree vectype; |
| tree op; |
| HOST_WIDE_INT linear_step; |
| enum vect_def_type dt; |
| unsigned int align; |
| bool simd_lane_linear; |
| }; |
| |
| /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME, |
| is linear within simd lane (but not within whole loop), note it in |
| *ARGINFO. */ |
| |
| static void |
| vect_simd_lane_linear (tree op, struct loop *loop, |
| struct simd_call_arg_info *arginfo) |
| { |
| gimple *def_stmt = SSA_NAME_DEF_STMT (op); |
| |
| if (!is_gimple_assign (def_stmt) |
| || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR |
| || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt))) |
| return; |
| |
| tree base = gimple_assign_rhs1 (def_stmt); |
| HOST_WIDE_INT linear_step = 0; |
| tree v = gimple_assign_rhs2 (def_stmt); |
| while (TREE_CODE (v) == SSA_NAME) |
| { |
| tree t; |
| def_stmt = SSA_NAME_DEF_STMT (v); |
| if (is_gimple_assign (def_stmt)) |
| switch (gimple_assign_rhs_code (def_stmt)) |
| { |
| case PLUS_EXPR: |
| t = gimple_assign_rhs2 (def_stmt); |
| if (linear_step || TREE_CODE (t) != INTEGER_CST) |
| return; |
| base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t); |
| v = gimple_assign_rhs1 (def_stmt); |
| continue; |
| case MULT_EXPR: |
| t = gimple_assign_rhs2 (def_stmt); |
| if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t)) |
| return; |
| linear_step = tree_to_shwi (t); |
| v = gimple_assign_rhs1 (def_stmt); |
| continue; |
| CASE_CONVERT: |
| t = gimple_assign_rhs1 (def_stmt); |
| if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE |
| || (TYPE_PRECISION (TREE_TYPE (v)) |
| < TYPE_PRECISION (TREE_TYPE (t)))) |
| return; |
| if (!linear_step) |
| linear_step = 1; |
| v = t; |
| continue; |
| default: |
| return; |
| } |
| else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE) |
| && loop->simduid |
| && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME |
| && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0)) |
| == loop->simduid)) |
| { |
| if (!linear_step) |
| linear_step = 1; |
| arginfo->linear_step = linear_step; |
| arginfo->op = base; |
| arginfo->simd_lane_linear = true; |
| return; |
| } |
| } |
| } |
| |
|