| /* Straight-line strength reduction. |
| Copyright (C) 2012-2021 Free Software Foundation, Inc. |
| Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com> |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it under |
| the terms of the GNU General Public License as published by the Free |
| Software Foundation; either version 3, or (at your option) any later |
| version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| /* There are many algorithms for performing strength reduction on |
| loops. This is not one of them. IVOPTS handles strength reduction |
| of induction variables just fine. This pass is intended to pick |
| up the crumbs it leaves behind, by considering opportunities for |
| strength reduction along dominator paths. |
| |
| Strength reduction addresses explicit multiplies, and certain |
| multiplies implicit in addressing expressions. It would also be |
| possible to apply strength reduction to divisions and modulos, |
| but such opportunities are relatively uncommon. |
| |
| Strength reduction is also currently restricted to integer operations. |
| If desired, it could be extended to floating-point operations under |
| control of something like -funsafe-math-optimizations. */ |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "rtl.h" |
| #include "tree.h" |
| #include "gimple.h" |
| #include "cfghooks.h" |
| #include "tree-pass.h" |
| #include "ssa.h" |
| #include "expmed.h" |
| #include "gimple-pretty-print.h" |
| #include "fold-const.h" |
| #include "gimple-iterator.h" |
| #include "gimplify-me.h" |
| #include "stor-layout.h" |
| #include "cfgloop.h" |
| #include "tree-cfg.h" |
| #include "domwalk.h" |
| #include "tree-ssa-address.h" |
| #include "tree-affine.h" |
| #include "tree-eh.h" |
| #include "builtins.h" |
| |
| /* Information about a strength reduction candidate. Each statement |
| in the candidate table represents an expression of one of the |
| following forms (the special case of CAND_REF will be described |
| later): |
| |
| (CAND_MULT) S1: X = (B + i) * S |
| (CAND_ADD) S1: X = B + (i * S) |
| |
| Here X and B are SSA names, i is an integer constant, and S is |
| either an SSA name or a constant. We call B the "base," i the |
| "index", and S the "stride." |
| |
| Any statement S0 that dominates S1 and is of the form: |
| |
| (CAND_MULT) S0: Y = (B + i') * S |
| (CAND_ADD) S0: Y = B + (i' * S) |
| |
| is called a "basis" for S1. In both cases, S1 may be replaced by |
| |
| S1': X = Y + (i - i') * S, |
| |
| where (i - i') * S is folded to the extent possible. |
| |
| All gimple statements are visited in dominator order, and each |
| statement that may contribute to one of the forms of S1 above is |
| given at least one entry in the candidate table. Such statements |
| include addition, pointer addition, subtraction, multiplication, |
| negation, copies, and nontrivial type casts. If a statement may |
| represent more than one expression of the forms of S1 above, |
| multiple "interpretations" are stored in the table and chained |
| together. Examples: |
| |
| * An add of two SSA names may treat either operand as the base. |
| * A multiply of two SSA names, likewise. |
| * A copy or cast may be thought of as either a CAND_MULT with |
| i = 0 and S = 1, or as a CAND_ADD with i = 0 or S = 0. |
| |
| Candidate records are allocated from an obstack. They are addressed |
| both from a hash table keyed on S1, and from a vector of candidate |
| pointers arranged in predominator order. |
| |
| Opportunity note |
| ---------------- |
| Currently we don't recognize: |
| |
| S0: Y = (S * i') - B |
| S1: X = (S * i) - B |
| |
| as a strength reduction opportunity, even though this S1 would |
| also be replaceable by the S1' above. This can be added if it |
| comes up in practice. |
| |
| Strength reduction in addressing |
| -------------------------------- |
| There is another kind of candidate known as CAND_REF. A CAND_REF |
| describes a statement containing a memory reference having |
| complex addressing that might benefit from strength reduction. |
| Specifically, we are interested in references for which |
| get_inner_reference returns a base address, offset, and bitpos as |
| follows: |
| |
| base: MEM_REF (T1, C1) |
| offset: MULT_EXPR (PLUS_EXPR (T2, C2), C3) |
| bitpos: C4 * BITS_PER_UNIT |
| |
| Here T1 and T2 are arbitrary trees, and C1, C2, C3, C4 are |
| arbitrary integer constants. Note that C2 may be zero, in which |
| case the offset will be MULT_EXPR (T2, C3). |
| |
| When this pattern is recognized, the original memory reference |
| can be replaced with: |
| |
| MEM_REF (POINTER_PLUS_EXPR (T1, MULT_EXPR (T2, C3)), |
| C1 + (C2 * C3) + C4) |
| |
| which distributes the multiply to allow constant folding. When |
| two or more addressing expressions can be represented by MEM_REFs |
| of this form, differing only in the constants C1, C2, and C4, |
| making this substitution produces more efficient addressing during |
| the RTL phases. When there are not at least two expressions with |
| the same values of T1, T2, and C3, there is nothing to be gained |
| by the replacement. |
| |
| Strength reduction of CAND_REFs uses the same infrastructure as |
| that used by CAND_MULTs and CAND_ADDs. We record T1 in the base (B) |
| field, MULT_EXPR (T2, C3) in the stride (S) field, and |
| C1 + (C2 * C3) + C4 in the index (i) field. A basis for a CAND_REF |
| is thus another CAND_REF with the same B and S values. When at |
| least two CAND_REFs are chained together using the basis relation, |
| each of them is replaced as above, resulting in improved code |
| generation for addressing. |
| |
| Conditional candidates |
| ====================== |
| |
| Conditional candidates are best illustrated with an example. |
| Consider the code sequence: |
| |
| (1) x_0 = ...; |
| (2) a_0 = x_0 * 5; MULT (B: x_0; i: 0; S: 5) |
| if (...) |
| (3) x_1 = x_0 + 1; ADD (B: x_0, i: 1; S: 1) |
| (4) x_2 = PHI <x_0, x_1>; PHI (B: x_0, i: 0, S: 1) |
| (5) x_3 = x_2 + 1; ADD (B: x_2, i: 1, S: 1) |
| (6) a_1 = x_3 * 5; MULT (B: x_2, i: 1; S: 5) |
| |
| Here strength reduction is complicated by the uncertain value of x_2. |
| A legitimate transformation is: |
| |
| (1) x_0 = ...; |
| (2) a_0 = x_0 * 5; |
| if (...) |
| { |
| (3) [x_1 = x_0 + 1;] |
| (3a) t_1 = a_0 + 5; |
| } |
| (4) [x_2 = PHI <x_0, x_1>;] |
| (4a) t_2 = PHI <a_0, t_1>; |
| (5) [x_3 = x_2 + 1;] |
| (6r) a_1 = t_2 + 5; |
| |
| where the bracketed instructions may go dead. |
| |
| To recognize this opportunity, we have to observe that statement (6) |
| has a "hidden basis" (2). The hidden basis is unlike a normal basis |
| in that the statement and the hidden basis have different base SSA |
| names (x_2 and x_0, respectively). The relationship is established |
| when a statement's base name (x_2) is defined by a phi statement (4), |
| each argument of which (x_0, x_1) has an identical "derived base name." |
| If the argument is defined by a candidate (as x_1 is by (3)) that is a |
| CAND_ADD having a stride of 1, the derived base name of the argument is |
| the base name of the candidate (x_0). Otherwise, the argument itself |
| is its derived base name (as is the case with argument x_0). |
| |
| The hidden basis for statement (6) is the nearest dominating candidate |
| whose base name is the derived base name (x_0) of the feeding phi (4), |
| and whose stride is identical to that of the statement. We can then |
| create the new "phi basis" (4a) and feeding adds along incoming arcs (3a), |
| allowing the final replacement of (6) by the strength-reduced (6r). |
| |
| To facilitate this, a new kind of candidate (CAND_PHI) is introduced. |
| A CAND_PHI is not a candidate for replacement, but is maintained in the |
| candidate table to ease discovery of hidden bases. Any phi statement |
| whose arguments share a common derived base name is entered into the |
| table with the derived base name, an (arbitrary) index of zero, and a |
| stride of 1. A statement with a hidden basis can then be detected by |
| simply looking up its feeding phi definition in the candidate table, |
| extracting the derived base name, and searching for a basis in the |
| usual manner after substituting the derived base name. |
| |
| Note that the transformation is only valid when the original phi and |
| the statements that define the phi's arguments are all at the same |
| position in the loop hierarchy. */ |
| |
| |
| /* Index into the candidate vector, offset by 1. VECs are zero-based, |
| while cand_idx's are one-based, with zero indicating null. */ |
| typedef unsigned cand_idx; |
| |
| /* The kind of candidate. */ |
| enum cand_kind |
| { |
| CAND_MULT, |
| CAND_ADD, |
| CAND_REF, |
| CAND_PHI |
| }; |
| |
| class slsr_cand_d |
| { |
| public: |
| /* The candidate statement S1. */ |
| gimple *cand_stmt; |
| |
| /* The base expression B: often an SSA name, but not always. */ |
| tree base_expr; |
| |
| /* The stride S. */ |
| tree stride; |
| |
| /* The index constant i. */ |
| widest_int index; |
| |
| /* The type of the candidate. This is normally the type of base_expr, |
| but casts may have occurred when combining feeding instructions. |
| A candidate can only be a basis for candidates of the same final type. |
| (For CAND_REFs, this is the type to be used for operand 1 of the |
| replacement MEM_REF.) */ |
| tree cand_type; |
| |
| /* The type to be used to interpret the stride field when the stride |
| is not a constant. Normally the same as the type of the recorded |
| stride, but when the stride has been cast we need to maintain that |
| knowledge in order to make legal substitutions without losing |
| precision. When the stride is a constant, this will be sizetype. */ |
| tree stride_type; |
| |
| /* The kind of candidate (CAND_MULT, etc.). */ |
| enum cand_kind kind; |
| |
| /* Index of this candidate in the candidate vector. */ |
| cand_idx cand_num; |
| |
| /* Index of the next candidate record for the same statement. |
| A statement may be useful in more than one way (e.g., due to |
| commutativity). So we can have multiple "interpretations" |
| of a statement. */ |
| cand_idx next_interp; |
| |
| /* Index of the first candidate record in a chain for the same |
| statement. */ |
| cand_idx first_interp; |
| |
| /* Index of the basis statement S0, if any, in the candidate vector. */ |
| cand_idx basis; |
| |
| /* First candidate for which this candidate is a basis, if one exists. */ |
| cand_idx dependent; |
| |
| /* Next candidate having the same basis as this one. */ |
| cand_idx sibling; |
| |
| /* If this is a conditional candidate, the CAND_PHI candidate |
| that defines the base SSA name B. */ |
| cand_idx def_phi; |
| |
| /* Savings that can be expected from eliminating dead code if this |
| candidate is replaced. */ |
| int dead_savings; |
| |
| /* For PHI candidates, use a visited flag to keep from processing the |
| same PHI twice from multiple paths. */ |
| int visited; |
| |
| /* We sometimes have to cache a phi basis with a phi candidate to |
| avoid processing it twice. Valid only if visited==1. */ |
| tree cached_basis; |
| }; |
| |
| typedef class slsr_cand_d slsr_cand, *slsr_cand_t; |
| typedef const class slsr_cand_d *const_slsr_cand_t; |
| |
| /* Pointers to candidates are chained together as part of a mapping |
| from base expressions to the candidates that use them. */ |
| |
| struct cand_chain_d |
| { |
| /* Base expression for the chain of candidates: often, but not |
| always, an SSA name. */ |
| tree base_expr; |
| |
| /* Pointer to a candidate. */ |
| slsr_cand_t cand; |
| |
| /* Chain pointer. */ |
| struct cand_chain_d *next; |
| |
| }; |
| |
| typedef struct cand_chain_d cand_chain, *cand_chain_t; |
| typedef const struct cand_chain_d *const_cand_chain_t; |
| |
| /* Information about a unique "increment" associated with candidates |
| having an SSA name for a stride. An increment is the difference |
| between the index of the candidate and the index of its basis, |
| i.e., (i - i') as discussed in the module commentary. |
| |
| When we are not going to generate address arithmetic we treat |
| increments that differ only in sign as the same, allowing sharing |
| of the cost of initializers. The absolute value of the increment |
| is stored in the incr_info. */ |
| |
| class incr_info_d |
| { |
| public: |
| /* The increment that relates a candidate to its basis. */ |
| widest_int incr; |
| |
| /* How many times the increment occurs in the candidate tree. */ |
| unsigned count; |
| |
| /* Cost of replacing candidates using this increment. Negative and |
| zero costs indicate replacement should be performed. */ |
| int cost; |
| |
| /* If this increment is profitable but is not -1, 0, or 1, it requires |
| an initializer T_0 = stride * incr to be found or introduced in the |
| nearest common dominator of all candidates. This field holds T_0 |
| for subsequent use. */ |
| tree initializer; |
| |
| /* If the initializer was found to already exist, this is the block |
| where it was found. */ |
| basic_block init_bb; |
| }; |
| |
| typedef class incr_info_d incr_info, *incr_info_t; |
| |
| /* Candidates are maintained in a vector. If candidate X dominates |
| candidate Y, then X appears before Y in the vector; but the |
| converse does not necessarily hold. */ |
| static vec<slsr_cand_t> cand_vec; |
| |
| enum cost_consts |
| { |
| COST_NEUTRAL = 0, |
| COST_INFINITE = 1000 |
| }; |
| |
| enum stride_status |
| { |
| UNKNOWN_STRIDE = 0, |
| KNOWN_STRIDE = 1 |
| }; |
| |
| enum phi_adjust_status |
| { |
| NOT_PHI_ADJUST = 0, |
| PHI_ADJUST = 1 |
| }; |
| |
| enum count_phis_status |
| { |
| DONT_COUNT_PHIS = 0, |
| COUNT_PHIS = 1 |
| }; |
| |
| /* Constrain how many PHI nodes we will visit for a conditional |
| candidate (depth and breadth). */ |
| const int MAX_SPREAD = 16; |
| |
| /* Pointer map embodying a mapping from statements to candidates. */ |
| static hash_map<gimple *, slsr_cand_t> *stmt_cand_map; |
| |
| /* Obstack for candidates. */ |
| static struct obstack cand_obstack; |
| |
| /* Obstack for candidate chains. */ |
| static struct obstack chain_obstack; |
| |
| /* An array INCR_VEC of incr_infos is used during analysis of related |
| candidates having an SSA name for a stride. INCR_VEC_LEN describes |
| its current length. MAX_INCR_VEC_LEN is used to avoid costly |
| pathological cases. */ |
| static incr_info_t incr_vec; |
| static unsigned incr_vec_len; |
| const int MAX_INCR_VEC_LEN = 16; |
| |
| /* For a chain of candidates with unknown stride, indicates whether or not |
| we must generate pointer arithmetic when replacing statements. */ |
| static bool address_arithmetic_p; |
| |
| /* Forward function declarations. */ |
| static slsr_cand_t base_cand_from_table (tree); |
| static tree introduce_cast_before_cand (slsr_cand_t, tree, tree); |
| static bool legal_cast_p_1 (tree, tree); |
| |
| /* Produce a pointer to the IDX'th candidate in the candidate vector. */ |
| |
| static slsr_cand_t |
| lookup_cand (cand_idx idx) |
| { |
| return cand_vec[idx]; |
| } |
| |
| /* Helper for hashing a candidate chain header. */ |
| |
| struct cand_chain_hasher : nofree_ptr_hash <cand_chain> |
| { |
| static inline hashval_t hash (const cand_chain *); |
| static inline bool equal (const cand_chain *, const cand_chain *); |
| }; |
| |
| inline hashval_t |
| cand_chain_hasher::hash (const cand_chain *p) |
| { |
| tree base_expr = p->base_expr; |
| return iterative_hash_expr (base_expr, 0); |
| } |
| |
| inline bool |
| cand_chain_hasher::equal (const cand_chain *chain1, const cand_chain *chain2) |
| { |
| return operand_equal_p (chain1->base_expr, chain2->base_expr, 0); |
| } |
| |
| /* Hash table embodying a mapping from base exprs to chains of candidates. */ |
| static hash_table<cand_chain_hasher> *base_cand_map; |
| |
| /* Pointer map used by tree_to_aff_combination_expand. */ |
| static hash_map<tree, name_expansion *> *name_expansions; |
| /* Pointer map embodying a mapping from bases to alternative bases. */ |
| static hash_map<tree, tree> *alt_base_map; |
| |
| /* Given BASE, use the tree affine combiniation facilities to |
| find the underlying tree expression for BASE, with any |
| immediate offset excluded. |
| |
| N.B. we should eliminate this backtracking with better forward |
| analysis in a future release. */ |
| |
| static tree |
| get_alternative_base (tree base) |
| { |
| tree *result = alt_base_map->get (base); |
| |
| if (result == NULL) |
| { |
| tree expr; |
| aff_tree aff; |
| |
| tree_to_aff_combination_expand (base, TREE_TYPE (base), |
| &aff, &name_expansions); |
| aff.offset = 0; |
| expr = aff_combination_to_tree (&aff); |
| |
| gcc_assert (!alt_base_map->put (base, base == expr ? NULL : expr)); |
| |
| return expr == base ? NULL : expr; |
| } |
| |
| return *result; |
| } |
| |
| /* Look in the candidate table for a CAND_PHI that defines BASE and |
| return it if found; otherwise return NULL. */ |
| |
| static cand_idx |
| find_phi_def (tree base) |
| { |
| slsr_cand_t c; |
| |
| if (TREE_CODE (base) != SSA_NAME) |
| return 0; |
| |
| c = base_cand_from_table (base); |
| |
| if (!c || c->kind != CAND_PHI |
| || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_phi_result (c->cand_stmt))) |
| return 0; |
| |
| return c->cand_num; |
| } |
| |
| /* Determine whether all uses of NAME are directly or indirectly |
| used by STMT. That is, we want to know whether if STMT goes |
| dead, the definition of NAME also goes dead. */ |
| static bool |
| uses_consumed_by_stmt (tree name, gimple *stmt, unsigned recurse = 0) |
| { |
| gimple *use_stmt; |
| imm_use_iterator iter; |
| bool retval = true; |
| |
| FOR_EACH_IMM_USE_STMT (use_stmt, iter, name) |
| { |
| if (use_stmt == stmt || is_gimple_debug (use_stmt)) |
| continue; |
| |
| if (!is_gimple_assign (use_stmt) |
| || !gimple_get_lhs (use_stmt) |
| || !is_gimple_reg (gimple_get_lhs (use_stmt)) |
| || recurse >= 10 |
| || !uses_consumed_by_stmt (gimple_get_lhs (use_stmt), stmt, |
| recurse + 1)) |
| { |
| retval = false; |
| break; |
| } |
| } |
| |
| return retval; |
| } |
| |
| /* Helper routine for find_basis_for_candidate. May be called twice: |
| once for the candidate's base expr, and optionally again either for |
| the candidate's phi definition or for a CAND_REF's alternative base |
| expression. */ |
| |
| static slsr_cand_t |
| find_basis_for_base_expr (slsr_cand_t c, tree base_expr) |
| { |
| cand_chain mapping_key; |
| cand_chain_t chain; |
| slsr_cand_t basis = NULL; |
| |
| // Limit potential of N^2 behavior for long candidate chains. |
| int iters = 0; |
| int max_iters = param_max_slsr_candidate_scan; |
| |
| mapping_key.base_expr = base_expr; |
| chain = base_cand_map->find (&mapping_key); |
| |
| for (; chain && iters < max_iters; chain = chain->next, ++iters) |
| { |
| slsr_cand_t one_basis = chain->cand; |
| |
| if (one_basis->kind != c->kind |
| || one_basis->cand_stmt == c->cand_stmt |
| || !operand_equal_p (one_basis->stride, c->stride, 0) |
| || !types_compatible_p (one_basis->cand_type, c->cand_type) |
| || !types_compatible_p (one_basis->stride_type, c->stride_type) |
| || !dominated_by_p (CDI_DOMINATORS, |
| gimple_bb (c->cand_stmt), |
| gimple_bb (one_basis->cand_stmt))) |
| continue; |
| |
| tree lhs = gimple_assign_lhs (one_basis->cand_stmt); |
| if (lhs && TREE_CODE (lhs) == SSA_NAME |
| && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)) |
| continue; |
| |
| if (!basis || basis->cand_num < one_basis->cand_num) |
| basis = one_basis; |
| } |
| |
| return basis; |
| } |
| |
| /* Use the base expr from candidate C to look for possible candidates |
| that can serve as a basis for C. Each potential basis must also |
| appear in a block that dominates the candidate statement and have |
| the same stride and type. If more than one possible basis exists, |
| the one with highest index in the vector is chosen; this will be |
| the most immediately dominating basis. */ |
| |
| static int |
| find_basis_for_candidate (slsr_cand_t c) |
| { |
| slsr_cand_t basis = find_basis_for_base_expr (c, c->base_expr); |
| |
| /* If a candidate doesn't have a basis using its base expression, |
| it may have a basis hidden by one or more intervening phis. */ |
| if (!basis && c->def_phi) |
| { |
| basic_block basis_bb, phi_bb; |
| slsr_cand_t phi_cand = lookup_cand (c->def_phi); |
| basis = find_basis_for_base_expr (c, phi_cand->base_expr); |
| |
| if (basis) |
| { |
| /* A hidden basis must dominate the phi-definition of the |
| candidate's base name. */ |
| phi_bb = gimple_bb (phi_cand->cand_stmt); |
| basis_bb = gimple_bb (basis->cand_stmt); |
| |
| if (phi_bb == basis_bb |
| || !dominated_by_p (CDI_DOMINATORS, phi_bb, basis_bb)) |
| { |
| basis = NULL; |
| c->basis = 0; |
| } |
| |
| /* If we found a hidden basis, estimate additional dead-code |
| savings if the phi and its feeding statements can be removed. */ |
| tree feeding_var = gimple_phi_result (phi_cand->cand_stmt); |
| if (basis && uses_consumed_by_stmt (feeding_var, c->cand_stmt)) |
| c->dead_savings += phi_cand->dead_savings; |
| } |
| } |
| |
| if (flag_expensive_optimizations && !basis && c->kind == CAND_REF) |
| { |
| tree alt_base_expr = get_alternative_base (c->base_expr); |
| if (alt_base_expr) |
| basis = find_basis_for_base_expr (c, alt_base_expr); |
| } |
| |
| if (basis) |
| { |
| c->sibling = basis->dependent; |
| basis->dependent = c->cand_num; |
| return basis->cand_num; |
| } |
| |
| return 0; |
| } |
| |
| /* Record a mapping from BASE to C, indicating that C may potentially serve |
| as a basis using that base expression. BASE may be the same as |
| C->BASE_EXPR; alternatively BASE can be a different tree that share the |
| underlining expression of C->BASE_EXPR. */ |
| |
| static void |
| record_potential_basis (slsr_cand_t c, tree base) |
| { |
| cand_chain_t node; |
| cand_chain **slot; |
| |
| gcc_assert (base); |
| |
| node = (cand_chain_t) obstack_alloc (&chain_obstack, sizeof (cand_chain)); |
| node->base_expr = base; |
| node->cand = c; |
| node->next = NULL; |
| slot = base_cand_map->find_slot (node, INSERT); |
| |
| if (*slot) |
| { |
| cand_chain_t head = (cand_chain_t) (*slot); |
| node->next = head->next; |
| head->next = node; |
| } |
| else |
| *slot = node; |
| } |
| |
| /* Allocate storage for a new candidate and initialize its fields. |
| Attempt to find a basis for the candidate. |
| |
| For CAND_REF, an alternative base may also be recorded and used |
| to find a basis. This helps cases where the expression hidden |
| behind BASE (which is usually an SSA_NAME) has immediate offset, |
| e.g. |
| |
| a2[i][j] = 1; |
| a2[i + 20][j] = 2; */ |
| |
| static slsr_cand_t |
| alloc_cand_and_find_basis (enum cand_kind kind, gimple *gs, tree base, |
| const widest_int &index, tree stride, tree ctype, |
| tree stype, unsigned savings) |
| { |
| slsr_cand_t c = (slsr_cand_t) obstack_alloc (&cand_obstack, |
| sizeof (slsr_cand)); |
| c->cand_stmt = gs; |
| c->base_expr = base; |
| c->stride = stride; |
| c->index = index; |
| c->cand_type = ctype; |
| c->stride_type = stype; |
| c->kind = kind; |
| c->cand_num = cand_vec.length (); |
| c->next_interp = 0; |
| c->first_interp = c->cand_num; |
| c->dependent = 0; |
| c->sibling = 0; |
| c->def_phi = kind == CAND_MULT ? find_phi_def (base) : 0; |
| c->dead_savings = savings; |
| c->visited = 0; |
| c->cached_basis = NULL_TREE; |
| |
| cand_vec.safe_push (c); |
| |
| if (kind == CAND_PHI) |
| c->basis = 0; |
| else |
| c->basis = find_basis_for_candidate (c); |
| |
| record_potential_basis (c, base); |
| if (flag_expensive_optimizations && kind == CAND_REF) |
| { |
| tree alt_base = get_alternative_base (base); |
| if (alt_base) |
| record_potential_basis (c, alt_base); |
| } |
| |
| return c; |
| } |
| |
| /* Determine the target cost of statement GS when compiling according |
| to SPEED. */ |
| |
| static int |
| stmt_cost (gimple *gs, bool speed) |
| { |
| tree lhs, rhs1, rhs2; |
| machine_mode lhs_mode; |
| |
| gcc_assert (is_gimple_assign (gs)); |
| lhs = gimple_assign_lhs (gs); |
| rhs1 = gimple_assign_rhs1 (gs); |
| lhs_mode = TYPE_MODE (TREE_TYPE (lhs)); |
| |
| switch (gimple_assign_rhs_code (gs)) |
| { |
| case MULT_EXPR: |
| rhs2 = gimple_assign_rhs2 (gs); |
| |
| if (tree_fits_shwi_p (rhs2)) |
| return mult_by_coeff_cost (tree_to_shwi (rhs2), lhs_mode, speed); |
| |
| gcc_assert (TREE_CODE (rhs1) != INTEGER_CST); |
| return mul_cost (speed, lhs_mode); |
| |
| case PLUS_EXPR: |
| case POINTER_PLUS_EXPR: |
| case MINUS_EXPR: |
| return add_cost (speed, lhs_mode); |
| |
| case NEGATE_EXPR: |
| return neg_cost (speed, lhs_mode); |
| |
| CASE_CONVERT: |
| return convert_cost (lhs_mode, TYPE_MODE (TREE_TYPE (rhs1)), speed); |
| |
| /* Note that we don't assign costs to copies that in most cases |
| will go away. */ |
| case SSA_NAME: |
| return 0; |
| |
| default: |
| ; |
| } |
| |
| gcc_unreachable (); |
| return 0; |
| } |
| |
| /* Look up the defining statement for BASE_IN and return a pointer |
| to its candidate in the candidate table, if any; otherwise NULL. |
| Only CAND_ADD and CAND_MULT candidates are returned. */ |
| |
| static slsr_cand_t |
| base_cand_from_table (tree base_in) |
| { |
| slsr_cand_t *result; |
| |
| gimple *def = SSA_NAME_DEF_STMT (base_in); |
| if (!def) |
| return (slsr_cand_t) NULL; |
| |
| result = stmt_cand_map->get (def); |
| |
| if (result && (*result)->kind != CAND_REF) |
| return *result; |
| |
| return (slsr_cand_t) NULL; |
| } |
| |
| /* Add an entry to the statement-to-candidate mapping. */ |
| |
| static void |
| add_cand_for_stmt (gimple *gs, slsr_cand_t c) |
| { |
| gcc_assert (!stmt_cand_map->put (gs, c)); |
| } |
| |
| /* Given PHI which contains a phi statement, determine whether it |
| satisfies all the requirements of a phi candidate. If so, create |
| a candidate. Note that a CAND_PHI never has a basis itself, but |
| is used to help find a basis for subsequent candidates. */ |
| |
| static void |
| slsr_process_phi (gphi *phi, bool speed) |
| { |
| unsigned i; |
| tree arg0_base = NULL_TREE, base_type; |
| slsr_cand_t c; |
| class loop *cand_loop = gimple_bb (phi)->loop_father; |
| unsigned savings = 0; |
| |
| /* A CAND_PHI requires each of its arguments to have the same |
| derived base name. (See the module header commentary for a |
| definition of derived base names.) Furthermore, all feeding |
| definitions must be in the same position in the loop hierarchy |
| as PHI. */ |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| slsr_cand_t arg_cand; |
| tree arg = gimple_phi_arg_def (phi, i); |
| tree derived_base_name = NULL_TREE; |
| gimple *arg_stmt = NULL; |
| basic_block arg_bb = NULL; |
| |
| if (TREE_CODE (arg) != SSA_NAME) |
| return; |
| |
| arg_cand = base_cand_from_table (arg); |
| |
| if (arg_cand) |
| { |
| while (arg_cand->kind != CAND_ADD && arg_cand->kind != CAND_PHI) |
| { |
| if (!arg_cand->next_interp) |
| return; |
| |
| arg_cand = lookup_cand (arg_cand->next_interp); |
| } |
| |
| if (!integer_onep (arg_cand->stride)) |
| return; |
| |
| derived_base_name = arg_cand->base_expr; |
| arg_stmt = arg_cand->cand_stmt; |
| arg_bb = gimple_bb (arg_stmt); |
| |
| /* Gather potential dead code savings if the phi statement |
| can be removed later on. */ |
| if (uses_consumed_by_stmt (arg, phi)) |
| { |
| if (gimple_code (arg_stmt) == GIMPLE_PHI) |
| savings += arg_cand->dead_savings; |
| else |
| savings += stmt_cost (arg_stmt, speed); |
| } |
| } |
| else if (SSA_NAME_IS_DEFAULT_DEF (arg)) |
| { |
| derived_base_name = arg; |
| arg_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); |
| } |
| |
| if (!arg_bb || arg_bb->loop_father != cand_loop) |
| return; |
| |
| if (i == 0) |
| arg0_base = derived_base_name; |
| else if (!operand_equal_p (derived_base_name, arg0_base, 0)) |
| return; |
| } |
| |
| /* Create the candidate. "alloc_cand_and_find_basis" is named |
| misleadingly for this case, as no basis will be sought for a |
| CAND_PHI. */ |
| base_type = TREE_TYPE (arg0_base); |
| |
| c = alloc_cand_and_find_basis (CAND_PHI, phi, arg0_base, |
| 0, integer_one_node, base_type, |
| sizetype, savings); |
| |
| /* Add the candidate to the statement-candidate mapping. */ |
| add_cand_for_stmt (phi, c); |
| } |
| |
| /* Given PBASE which is a pointer to tree, look up the defining |
| statement for it and check whether the candidate is in the |
| form of: |
| |
| X = B + (1 * S), S is integer constant |
| X = B + (i * S), S is integer one |
| |
| If so, set PBASE to the candidate's base_expr and return double |
| int (i * S). |
| Otherwise, just return double int zero. */ |
| |
| static widest_int |
| backtrace_base_for_ref (tree *pbase) |
| { |
| tree base_in = *pbase; |
| slsr_cand_t base_cand; |
| |
| STRIP_NOPS (base_in); |
| |
| /* Strip off widening conversion(s) to handle cases where |
| e.g. 'B' is widened from an 'int' in order to calculate |
| a 64-bit address. */ |
| if (CONVERT_EXPR_P (base_in) |
| && legal_cast_p_1 (TREE_TYPE (base_in), |
| TREE_TYPE (TREE_OPERAND (base_in, 0)))) |
| base_in = get_unwidened (base_in, NULL_TREE); |
| |
| if (TREE_CODE (base_in) != SSA_NAME) |
| return 0; |
| |
| base_cand = base_cand_from_table (base_in); |
| |
| while (base_cand && base_cand->kind != CAND_PHI) |
| { |
| if (base_cand->kind == CAND_ADD |
| && base_cand->index == 1 |
| && TREE_CODE (base_cand->stride) == INTEGER_CST) |
| { |
| /* X = B + (1 * S), S is integer constant. */ |
| *pbase = base_cand->base_expr; |
| return wi::to_widest (base_cand->stride); |
| } |
| else if (base_cand->kind == CAND_ADD |
| && TREE_CODE (base_cand->stride) == INTEGER_CST |
| && integer_onep (base_cand->stride)) |
| { |
| /* X = B + (i * S), S is integer one. */ |
| *pbase = base_cand->base_expr; |
| return base_cand->index; |
| } |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| |
| return 0; |
| } |
| |
| /* Look for the following pattern: |
| |
| *PBASE: MEM_REF (T1, C1) |
| |
| *POFFSET: MULT_EXPR (T2, C3) [C2 is zero] |
| or |
| MULT_EXPR (PLUS_EXPR (T2, C2), C3) |
| or |
| MULT_EXPR (MINUS_EXPR (T2, -C2), C3) |
| |
| *PINDEX: C4 * BITS_PER_UNIT |
| |
| If not present, leave the input values unchanged and return FALSE. |
| Otherwise, modify the input values as follows and return TRUE: |
| |
| *PBASE: T1 |
| *POFFSET: MULT_EXPR (T2, C3) |
| *PINDEX: C1 + (C2 * C3) + C4 |
| |
| When T2 is recorded by a CAND_ADD in the form of (T2' + C5), it |
| will be further restructured to: |
| |
| *PBASE: T1 |
| *POFFSET: MULT_EXPR (T2', C3) |
| *PINDEX: C1 + (C2 * C3) + C4 + (C5 * C3) */ |
| |
| static bool |
| restructure_reference (tree *pbase, tree *poffset, widest_int *pindex, |
| tree *ptype) |
| { |
| tree base = *pbase, offset = *poffset; |
| widest_int index = *pindex; |
| tree mult_op0, t1, t2, type; |
| widest_int c1, c2, c3, c4, c5; |
| offset_int mem_offset; |
| |
| if (!base |
| || !offset |
| || TREE_CODE (base) != MEM_REF |
| || !mem_ref_offset (base).is_constant (&mem_offset) |
| || TREE_CODE (offset) != MULT_EXPR |
| || TREE_CODE (TREE_OPERAND (offset, 1)) != INTEGER_CST |
| || wi::umod_floor (index, BITS_PER_UNIT) != 0) |
| return false; |
| |
| t1 = TREE_OPERAND (base, 0); |
| c1 = widest_int::from (mem_offset, SIGNED); |
| type = TREE_TYPE (TREE_OPERAND (base, 1)); |
| |
| mult_op0 = TREE_OPERAND (offset, 0); |
| c3 = wi::to_widest (TREE_OPERAND (offset, 1)); |
| |
| if (TREE_CODE (mult_op0) == PLUS_EXPR) |
| |
| if (TREE_CODE (TREE_OPERAND (mult_op0, 1)) == INTEGER_CST) |
| { |
| t2 = TREE_OPERAND (mult_op0, 0); |
| c2 = wi::to_widest (TREE_OPERAND (mult_op0, 1)); |
| } |
| else |
| return false; |
| |
| else if (TREE_CODE (mult_op0) == MINUS_EXPR) |
| |
| if (TREE_CODE (TREE_OPERAND (mult_op0, 1)) == INTEGER_CST) |
| { |
| t2 = TREE_OPERAND (mult_op0, 0); |
| c2 = -wi::to_widest (TREE_OPERAND (mult_op0, 1)); |
| } |
| else |
| return false; |
| |
| else |
| { |
| t2 = mult_op0; |
| c2 = 0; |
| } |
| |
| c4 = index >> LOG2_BITS_PER_UNIT; |
| c5 = backtrace_base_for_ref (&t2); |
| |
| *pbase = t1; |
| *poffset = fold_build2 (MULT_EXPR, sizetype, fold_convert (sizetype, t2), |
| wide_int_to_tree (sizetype, c3)); |
| *pindex = c1 + c2 * c3 + c4 + c5 * c3; |
| *ptype = type; |
| |
| return true; |
| } |
| |
| /* Given GS which contains a data reference, create a CAND_REF entry in |
| the candidate table and attempt to find a basis. */ |
| |
| static void |
| slsr_process_ref (gimple *gs) |
| { |
| tree ref_expr, base, offset, type; |
| poly_int64 bitsize, bitpos; |
| machine_mode mode; |
| int unsignedp, reversep, volatilep; |
| slsr_cand_t c; |
| |
| if (gimple_vdef (gs)) |
| ref_expr = gimple_assign_lhs (gs); |
| else |
| ref_expr = gimple_assign_rhs1 (gs); |
| |
| if (!handled_component_p (ref_expr) |
| || TREE_CODE (ref_expr) == BIT_FIELD_REF |
| || (TREE_CODE (ref_expr) == COMPONENT_REF |
| && DECL_BIT_FIELD (TREE_OPERAND (ref_expr, 1)))) |
| return; |
| |
| base = get_inner_reference (ref_expr, &bitsize, &bitpos, &offset, &mode, |
| &unsignedp, &reversep, &volatilep); |
| HOST_WIDE_INT cbitpos; |
| if (reversep || !bitpos.is_constant (&cbitpos)) |
| return; |
| widest_int index = cbitpos; |
| |
| if (!restructure_reference (&base, &offset, &index, &type)) |
| return; |
| |
| c = alloc_cand_and_find_basis (CAND_REF, gs, base, index, offset, |
| type, sizetype, 0); |
| |
| /* Add the candidate to the statement-candidate mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| |
| /* Create a candidate entry for a statement GS, where GS multiplies |
| two SSA names BASE_IN and STRIDE_IN. Propagate any known information |
| about the two SSA names into the new candidate. Return the new |
| candidate. */ |
| |
| static slsr_cand_t |
| create_mul_ssa_cand (gimple *gs, tree base_in, tree stride_in, bool speed) |
| { |
| tree base = NULL_TREE, stride = NULL_TREE, ctype = NULL_TREE; |
| tree stype = NULL_TREE; |
| widest_int index; |
| unsigned savings = 0; |
| slsr_cand_t c; |
| slsr_cand_t base_cand = base_cand_from_table (base_in); |
| |
| /* Look at all interpretations of the base candidate, if necessary, |
| to find information to propagate into this candidate. */ |
| while (base_cand && !base && base_cand->kind != CAND_PHI) |
| { |
| |
| if (base_cand->kind == CAND_MULT && integer_onep (base_cand->stride)) |
| { |
| /* Y = (B + i') * 1 |
| X = Y * Z |
| ================ |
| X = (B + i') * Z */ |
| base = base_cand->base_expr; |
| index = base_cand->index; |
| stride = stride_in; |
| ctype = base_cand->cand_type; |
| stype = TREE_TYPE (stride_in); |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| else if (base_cand->kind == CAND_ADD |
| && TREE_CODE (base_cand->stride) == INTEGER_CST) |
| { |
| /* Y = B + (i' * S), S constant |
| X = Y * Z |
| ============================ |
| X = B + ((i' * S) * Z) */ |
| base = base_cand->base_expr; |
| index = base_cand->index * wi::to_widest (base_cand->stride); |
| stride = stride_in; |
| ctype = base_cand->cand_type; |
| stype = TREE_TYPE (stride_in); |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| |
| if (!base) |
| { |
| /* No interpretations had anything useful to propagate, so |
| produce X = (Y + 0) * Z. */ |
| base = base_in; |
| index = 0; |
| stride = stride_in; |
| ctype = TREE_TYPE (base_in); |
| stype = TREE_TYPE (stride_in); |
| } |
| |
| c = alloc_cand_and_find_basis (CAND_MULT, gs, base, index, stride, |
| ctype, stype, savings); |
| return c; |
| } |
| |
| /* Create a candidate entry for a statement GS, where GS multiplies |
| SSA name BASE_IN by constant STRIDE_IN. Propagate any known |
| information about BASE_IN into the new candidate. Return the new |
| candidate. */ |
| |
| static slsr_cand_t |
| create_mul_imm_cand (gimple *gs, tree base_in, tree stride_in, bool speed) |
| { |
| tree base = NULL_TREE, stride = NULL_TREE, ctype = NULL_TREE; |
| widest_int index, temp; |
| unsigned savings = 0; |
| slsr_cand_t c; |
| slsr_cand_t base_cand = base_cand_from_table (base_in); |
| |
| /* Look at all interpretations of the base candidate, if necessary, |
| to find information to propagate into this candidate. */ |
| while (base_cand && !base && base_cand->kind != CAND_PHI) |
| { |
| if (base_cand->kind == CAND_MULT |
| && TREE_CODE (base_cand->stride) == INTEGER_CST) |
| { |
| /* Y = (B + i') * S, S constant |
| X = Y * c |
| ============================ |
| X = (B + i') * (S * c) */ |
| temp = wi::to_widest (base_cand->stride) * wi::to_widest (stride_in); |
| if (wi::fits_to_tree_p (temp, TREE_TYPE (stride_in))) |
| { |
| base = base_cand->base_expr; |
| index = base_cand->index; |
| stride = wide_int_to_tree (TREE_TYPE (stride_in), temp); |
| ctype = base_cand->cand_type; |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| } |
| else if (base_cand->kind == CAND_ADD && integer_onep (base_cand->stride)) |
| { |
| /* Y = B + (i' * 1) |
| X = Y * c |
| =========================== |
| X = (B + i') * c */ |
| base = base_cand->base_expr; |
| index = base_cand->index; |
| stride = stride_in; |
| ctype = base_cand->cand_type; |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| else if (base_cand->kind == CAND_ADD |
| && base_cand->index == 1 |
| && TREE_CODE (base_cand->stride) == INTEGER_CST) |
| { |
| /* Y = B + (1 * S), S constant |
| X = Y * c |
| =========================== |
| X = (B + S) * c */ |
| base = base_cand->base_expr; |
| index = wi::to_widest (base_cand->stride); |
| stride = stride_in; |
| ctype = base_cand->cand_type; |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| |
| if (!base) |
| { |
| /* No interpretations had anything useful to propagate, so |
| produce X = (Y + 0) * c. */ |
| base = base_in; |
| index = 0; |
| stride = stride_in; |
| ctype = TREE_TYPE (base_in); |
| } |
| |
| c = alloc_cand_and_find_basis (CAND_MULT, gs, base, index, stride, |
| ctype, sizetype, savings); |
| return c; |
| } |
| |
| /* Given GS which is a multiply of scalar integers, make an appropriate |
| entry in the candidate table. If this is a multiply of two SSA names, |
| create two CAND_MULT interpretations and attempt to find a basis for |
| each of them. Otherwise, create a single CAND_MULT and attempt to |
| find a basis. */ |
| |
| static void |
| slsr_process_mul (gimple *gs, tree rhs1, tree rhs2, bool speed) |
| { |
| slsr_cand_t c, c2; |
| |
| /* If this is a multiply of an SSA name with itself, it is highly |
| unlikely that we will get a strength reduction opportunity, so |
| don't record it as a candidate. This simplifies the logic for |
| finding a basis, so if this is removed that must be considered. */ |
| if (rhs1 == rhs2) |
| return; |
| |
| if (TREE_CODE (rhs2) == SSA_NAME) |
| { |
| /* Record an interpretation of this statement in the candidate table |
| assuming RHS1 is the base expression and RHS2 is the stride. */ |
| c = create_mul_ssa_cand (gs, rhs1, rhs2, speed); |
| |
| /* Add the first interpretation to the statement-candidate mapping. */ |
| add_cand_for_stmt (gs, c); |
| |
| /* Record another interpretation of this statement assuming RHS1 |
| is the stride and RHS2 is the base expression. */ |
| c2 = create_mul_ssa_cand (gs, rhs2, rhs1, speed); |
| c->next_interp = c2->cand_num; |
| c2->first_interp = c->cand_num; |
| } |
| else if (TREE_CODE (rhs2) == INTEGER_CST && !integer_zerop (rhs2)) |
| { |
| /* Record an interpretation for the multiply-immediate. */ |
| c = create_mul_imm_cand (gs, rhs1, rhs2, speed); |
| |
| /* Add the interpretation to the statement-candidate mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| } |
| |
| /* Create a candidate entry for a statement GS, where GS adds two |
| SSA names BASE_IN and ADDEND_IN if SUBTRACT_P is false, and |
| subtracts ADDEND_IN from BASE_IN otherwise. Propagate any known |
| information about the two SSA names into the new candidate. |
| Return the new candidate. */ |
| |
| static slsr_cand_t |
| create_add_ssa_cand (gimple *gs, tree base_in, tree addend_in, |
| bool subtract_p, bool speed) |
| { |
| tree base = NULL_TREE, stride = NULL_TREE, ctype = NULL_TREE; |
| tree stype = NULL_TREE; |
| widest_int index; |
| unsigned savings = 0; |
| slsr_cand_t c; |
| slsr_cand_t base_cand = base_cand_from_table (base_in); |
| slsr_cand_t addend_cand = base_cand_from_table (addend_in); |
| |
| /* The most useful transformation is a multiply-immediate feeding |
| an add or subtract. Look for that first. */ |
| while (addend_cand && !base && addend_cand->kind != CAND_PHI) |
| { |
| if (addend_cand->kind == CAND_MULT |
| && addend_cand->index == 0 |
| && TREE_CODE (addend_cand->stride) == INTEGER_CST) |
| { |
| /* Z = (B + 0) * S, S constant |
| X = Y +/- Z |
| =========================== |
| X = Y + ((+/-1 * S) * B) */ |
| base = base_in; |
| index = wi::to_widest (addend_cand->stride); |
| if (subtract_p) |
| index = -index; |
| stride = addend_cand->base_expr; |
| ctype = TREE_TYPE (base_in); |
| stype = addend_cand->cand_type; |
| if (has_single_use (addend_in)) |
| savings = (addend_cand->dead_savings |
| + stmt_cost (addend_cand->cand_stmt, speed)); |
| } |
| |
| addend_cand = lookup_cand (addend_cand->next_interp); |
| } |
| |
| while (base_cand && !base && base_cand->kind != CAND_PHI) |
| { |
| if (base_cand->kind == CAND_ADD |
| && (base_cand->index == 0 |
| || operand_equal_p (base_cand->stride, |
| integer_zero_node, 0))) |
| { |
| /* Y = B + (i' * S), i' * S = 0 |
| X = Y +/- Z |
| ============================ |
| X = B + (+/-1 * Z) */ |
| base = base_cand->base_expr; |
| index = subtract_p ? -1 : 1; |
| stride = addend_in; |
| ctype = base_cand->cand_type; |
| stype = (TREE_CODE (addend_in) == INTEGER_CST ? sizetype |
| : TREE_TYPE (addend_in)); |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| else if (subtract_p) |
| { |
| slsr_cand_t subtrahend_cand = base_cand_from_table (addend_in); |
| |
| while (subtrahend_cand && !base && subtrahend_cand->kind != CAND_PHI) |
| { |
| if (subtrahend_cand->kind == CAND_MULT |
| && subtrahend_cand->index == 0 |
| && TREE_CODE (subtrahend_cand->stride) == INTEGER_CST) |
| { |
| /* Z = (B + 0) * S, S constant |
| X = Y - Z |
| =========================== |
| Value: X = Y + ((-1 * S) * B) */ |
| base = base_in; |
| index = wi::to_widest (subtrahend_cand->stride); |
| index = -index; |
| stride = subtrahend_cand->base_expr; |
| ctype = TREE_TYPE (base_in); |
| stype = subtrahend_cand->cand_type; |
| if (has_single_use (addend_in)) |
| savings = (subtrahend_cand->dead_savings |
| + stmt_cost (subtrahend_cand->cand_stmt, speed)); |
| } |
| |
| subtrahend_cand = lookup_cand (subtrahend_cand->next_interp); |
| } |
| } |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| |
| if (!base) |
| { |
| /* No interpretations had anything useful to propagate, so |
| produce X = Y + (1 * Z). */ |
| base = base_in; |
| index = subtract_p ? -1 : 1; |
| stride = addend_in; |
| ctype = TREE_TYPE (base_in); |
| stype = (TREE_CODE (addend_in) == INTEGER_CST ? sizetype |
| : TREE_TYPE (addend_in)); |
| } |
| |
| c = alloc_cand_and_find_basis (CAND_ADD, gs, base, index, stride, |
| ctype, stype, savings); |
| return c; |
| } |
| |
| /* Create a candidate entry for a statement GS, where GS adds SSA |
| name BASE_IN to constant INDEX_IN. Propagate any known information |
| about BASE_IN into the new candidate. Return the new candidate. */ |
| |
| static slsr_cand_t |
| create_add_imm_cand (gimple *gs, tree base_in, const widest_int &index_in, |
| bool speed) |
| { |
| enum cand_kind kind = CAND_ADD; |
| tree base = NULL_TREE, stride = NULL_TREE, ctype = NULL_TREE; |
| tree stype = NULL_TREE; |
| widest_int index, multiple; |
| unsigned savings = 0; |
| slsr_cand_t c; |
| slsr_cand_t base_cand = base_cand_from_table (base_in); |
| |
| while (base_cand && !base && base_cand->kind != CAND_PHI) |
| { |
| signop sign = TYPE_SIGN (TREE_TYPE (base_cand->stride)); |
| |
| if (TREE_CODE (base_cand->stride) == INTEGER_CST |
| && wi::multiple_of_p (index_in, wi::to_widest (base_cand->stride), |
| sign, &multiple)) |
| { |
| /* Y = (B + i') * S, S constant, c = kS for some integer k |
| X = Y + c |
| ============================ |
| X = (B + (i'+ k)) * S |
| OR |
| Y = B + (i' * S), S constant, c = kS for some integer k |
| X = Y + c |
| ============================ |
| X = (B + (i'+ k)) * S */ |
| kind = base_cand->kind; |
| base = base_cand->base_expr; |
| index = base_cand->index + multiple; |
| stride = base_cand->stride; |
| ctype = base_cand->cand_type; |
| stype = base_cand->stride_type; |
| if (has_single_use (base_in)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| } |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| |
| if (!base) |
| { |
| /* No interpretations had anything useful to propagate, so |
| produce X = Y + (c * 1). */ |
| kind = CAND_ADD; |
| base = base_in; |
| index = index_in; |
| stride = integer_one_node; |
| ctype = TREE_TYPE (base_in); |
| stype = sizetype; |
| } |
| |
| c = alloc_cand_and_find_basis (kind, gs, base, index, stride, |
| ctype, stype, savings); |
| return c; |
| } |
| |
| /* Given GS which is an add or subtract of scalar integers or pointers, |
| make at least one appropriate entry in the candidate table. */ |
| |
| static void |
| slsr_process_add (gimple *gs, tree rhs1, tree rhs2, bool speed) |
| { |
| bool subtract_p = gimple_assign_rhs_code (gs) == MINUS_EXPR; |
| slsr_cand_t c = NULL, c2; |
| |
| if (TREE_CODE (rhs2) == SSA_NAME) |
| { |
| /* First record an interpretation assuming RHS1 is the base expression |
| and RHS2 is the stride. But it doesn't make sense for the |
| stride to be a pointer, so don't record a candidate in that case. */ |
| if (!POINTER_TYPE_P (TREE_TYPE (rhs2))) |
| { |
| c = create_add_ssa_cand (gs, rhs1, rhs2, subtract_p, speed); |
| |
| /* Add the first interpretation to the statement-candidate |
| mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| |
| /* If the two RHS operands are identical, or this is a subtract, |
| we're done. */ |
| if (operand_equal_p (rhs1, rhs2, 0) || subtract_p) |
| return; |
| |
| /* Otherwise, record another interpretation assuming RHS2 is the |
| base expression and RHS1 is the stride, again provided that the |
| stride is not a pointer. */ |
| if (!POINTER_TYPE_P (TREE_TYPE (rhs1))) |
| { |
| c2 = create_add_ssa_cand (gs, rhs2, rhs1, false, speed); |
| if (c) |
| { |
| c->next_interp = c2->cand_num; |
| c2->first_interp = c->cand_num; |
| } |
| else |
| add_cand_for_stmt (gs, c2); |
| } |
| } |
| else if (TREE_CODE (rhs2) == INTEGER_CST) |
| { |
| /* Record an interpretation for the add-immediate. */ |
| widest_int index = wi::to_widest (rhs2); |
| if (subtract_p) |
| index = -index; |
| |
| c = create_add_imm_cand (gs, rhs1, index, speed); |
| |
| /* Add the interpretation to the statement-candidate mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| } |
| |
| /* Given GS which is a negate of a scalar integer, make an appropriate |
| entry in the candidate table. A negate is equivalent to a multiply |
| by -1. */ |
| |
| static void |
| slsr_process_neg (gimple *gs, tree rhs1, bool speed) |
| { |
| /* Record a CAND_MULT interpretation for the multiply by -1. */ |
| slsr_cand_t c = create_mul_imm_cand (gs, rhs1, integer_minus_one_node, speed); |
| |
| /* Add the interpretation to the statement-candidate mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| |
| /* Help function for legal_cast_p, operating on two trees. Checks |
| whether it's allowable to cast from RHS to LHS. See legal_cast_p |
| for more details. */ |
| |
| static bool |
| legal_cast_p_1 (tree lhs_type, tree rhs_type) |
| { |
| unsigned lhs_size, rhs_size; |
| bool lhs_wraps, rhs_wraps; |
| |
| lhs_size = TYPE_PRECISION (lhs_type); |
| rhs_size = TYPE_PRECISION (rhs_type); |
| lhs_wraps = ANY_INTEGRAL_TYPE_P (lhs_type) && TYPE_OVERFLOW_WRAPS (lhs_type); |
| rhs_wraps = ANY_INTEGRAL_TYPE_P (rhs_type) && TYPE_OVERFLOW_WRAPS (rhs_type); |
| |
| if (lhs_size < rhs_size |
| || (rhs_wraps && !lhs_wraps) |
| || (rhs_wraps && lhs_wraps && rhs_size != lhs_size)) |
| return false; |
| |
| return true; |
| } |
| |
| /* Return TRUE if GS is a statement that defines an SSA name from |
| a conversion and is legal for us to combine with an add and multiply |
| in the candidate table. For example, suppose we have: |
| |
| A = B + i; |
| C = (type) A; |
| D = C * S; |
| |
| Without the type-cast, we would create a CAND_MULT for D with base B, |
| index i, and stride S. We want to record this candidate only if it |
| is equivalent to apply the type cast following the multiply: |
| |
| A = B + i; |
| E = A * S; |
| D = (type) E; |
| |
| We will record the type with the candidate for D. This allows us |
| to use a similar previous candidate as a basis. If we have earlier seen |
| |
| A' = B + i'; |
| C' = (type) A'; |
| D' = C' * S; |
| |
| we can replace D with |
| |
| D = D' + (i - i') * S; |
| |
| But if moving the type-cast would change semantics, we mustn't do this. |
| |
| This is legitimate for casts from a non-wrapping integral type to |
| any integral type of the same or larger size. It is not legitimate |
| to convert a wrapping type to a non-wrapping type, or to a wrapping |
| type of a different size. I.e., with a wrapping type, we must |
| assume that the addition B + i could wrap, in which case performing |
| the multiply before or after one of the "illegal" type casts will |
| have different semantics. */ |
| |
| static bool |
| legal_cast_p (gimple *gs, tree rhs) |
| { |
| if (!is_gimple_assign (gs) |
| || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (gs))) |
| return false; |
| |
| return legal_cast_p_1 (TREE_TYPE (gimple_assign_lhs (gs)), TREE_TYPE (rhs)); |
| } |
| |
| /* Given GS which is a cast to a scalar integer type, determine whether |
| the cast is legal for strength reduction. If so, make at least one |
| appropriate entry in the candidate table. */ |
| |
| static void |
| slsr_process_cast (gimple *gs, tree rhs1, bool speed) |
| { |
| tree lhs, ctype; |
| slsr_cand_t base_cand, c = NULL, c2; |
| unsigned savings = 0; |
| |
| if (!legal_cast_p (gs, rhs1)) |
| return; |
| |
| lhs = gimple_assign_lhs (gs); |
| base_cand = base_cand_from_table (rhs1); |
| ctype = TREE_TYPE (lhs); |
| |
| if (base_cand && base_cand->kind != CAND_PHI) |
| { |
| slsr_cand_t first_cand = NULL; |
| |
| while (base_cand) |
| { |
| /* Propagate all data from the base candidate except the type, |
| which comes from the cast, and the base candidate's cast, |
| which is no longer applicable. */ |
| if (has_single_use (rhs1)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| |
| c = alloc_cand_and_find_basis (base_cand->kind, gs, |
| base_cand->base_expr, |
| base_cand->index, base_cand->stride, |
| ctype, base_cand->stride_type, |
| savings); |
| if (!first_cand) |
| first_cand = c; |
| |
| if (first_cand != c) |
| c->first_interp = first_cand->cand_num; |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| } |
| else |
| { |
| /* If nothing is known about the RHS, create fresh CAND_ADD and |
| CAND_MULT interpretations: |
| |
| X = Y + (0 * 1) |
| X = (Y + 0) * 1 |
| |
| The first of these is somewhat arbitrary, but the choice of |
| 1 for the stride simplifies the logic for propagating casts |
| into their uses. */ |
| c = alloc_cand_and_find_basis (CAND_ADD, gs, rhs1, 0, |
| integer_one_node, ctype, sizetype, 0); |
| c2 = alloc_cand_and_find_basis (CAND_MULT, gs, rhs1, 0, |
| integer_one_node, ctype, sizetype, 0); |
| c->next_interp = c2->cand_num; |
| c2->first_interp = c->cand_num; |
| } |
| |
| /* Add the first (or only) interpretation to the statement-candidate |
| mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| |
| /* Given GS which is a copy of a scalar integer type, make at least one |
| appropriate entry in the candidate table. |
| |
| This interface is included for completeness, but is unnecessary |
| if this pass immediately follows a pass that performs copy |
| propagation, such as DOM. */ |
| |
| static void |
| slsr_process_copy (gimple *gs, tree rhs1, bool speed) |
| { |
| slsr_cand_t base_cand, c = NULL, c2; |
| unsigned savings = 0; |
| |
| base_cand = base_cand_from_table (rhs1); |
| |
| if (base_cand && base_cand->kind != CAND_PHI) |
| { |
| slsr_cand_t first_cand = NULL; |
| |
| while (base_cand) |
| { |
| /* Propagate all data from the base candidate. */ |
| if (has_single_use (rhs1)) |
| savings = (base_cand->dead_savings |
| + stmt_cost (base_cand->cand_stmt, speed)); |
| |
| c = alloc_cand_and_find_basis (base_cand->kind, gs, |
| base_cand->base_expr, |
| base_cand->index, base_cand->stride, |
| base_cand->cand_type, |
| base_cand->stride_type, savings); |
| if (!first_cand) |
| first_cand = c; |
| |
| if (first_cand != c) |
| c->first_interp = first_cand->cand_num; |
| |
| base_cand = lookup_cand (base_cand->next_interp); |
| } |
| } |
| else |
| { |
| /* If nothing is known about the RHS, create fresh CAND_ADD and |
| CAND_MULT interpretations: |
| |
| X = Y + (0 * 1) |
| X = (Y + 0) * 1 |
| |
| The first of these is somewhat arbitrary, but the choice of |
| 1 for the stride simplifies the logic for propagating casts |
| into their uses. */ |
| c = alloc_cand_and_find_basis (CAND_ADD, gs, rhs1, 0, |
| integer_one_node, TREE_TYPE (rhs1), |
| sizetype, 0); |
| c2 = alloc_cand_and_find_basis (CAND_MULT, gs, rhs1, 0, |
| integer_one_node, TREE_TYPE (rhs1), |
| sizetype, 0); |
| c->next_interp = c2->cand_num; |
| c2->first_interp = c->cand_num; |
| } |
| |
| /* Add the first (or only) interpretation to the statement-candidate |
| mapping. */ |
| add_cand_for_stmt (gs, c); |
| } |
| |
| class find_candidates_dom_walker : public dom_walker |
| { |
| public: |
| find_candidates_dom_walker (cdi_direction direction) |
| : dom_walker (direction) {} |
| virtual edge before_dom_children (basic_block); |
| }; |
| |
| /* Find strength-reduction candidates in block BB. */ |
| |
| edge |
| find_candidates_dom_walker::before_dom_children (basic_block bb) |
| { |
| bool speed = optimize_bb_for_speed_p (bb); |
| |
| for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| slsr_process_phi (gsi.phi (), speed); |
| |
| for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *gs = gsi_stmt (gsi); |
| |
| if (stmt_could_throw_p (cfun, gs)) |
| continue; |
| |
| if (gimple_vuse (gs) && gimple_assign_single_p (gs)) |
| slsr_process_ref (gs); |
| |
| else if (is_gimple_assign (gs) |
| && (INTEGRAL_TYPE_P (TREE_TYPE (gimple_assign_lhs (gs))) |
| || POINTER_TYPE_P (TREE_TYPE (gimple_assign_lhs (gs))))) |
| { |
| tree rhs1 = NULL_TREE, rhs2 = NULL_TREE; |
| |
| switch (gimple_assign_rhs_code (gs)) |
| { |
| case MULT_EXPR: |
| case PLUS_EXPR: |
| rhs1 = gimple_assign_rhs1 (gs); |
| rhs2 = gimple_assign_rhs2 (gs); |
| /* Should never happen, but currently some buggy situations |
| in earlier phases put constants in rhs1. */ |
| if (TREE_CODE (rhs1) != SSA_NAME) |
| continue; |
| break; |
| |
| /* Possible future opportunity: rhs1 of a ptr+ can be |
| an ADDR_EXPR. */ |
| case POINTER_PLUS_EXPR: |
| case MINUS_EXPR: |
| rhs2 = gimple_assign_rhs2 (gs); |
| gcc_fallthrough (); |
| |
| CASE_CONVERT: |
| case SSA_NAME: |
| case NEGATE_EXPR: |
| rhs1 = gimple_assign_rhs1 (gs); |
| if (TREE_CODE (rhs1) != SSA_NAME) |
| continue; |
| break; |
| |
| default: |
| ; |
| } |
| |
| switch (gimple_assign_rhs_code (gs)) |
| { |
| case MULT_EXPR: |
| slsr_process_mul (gs, rhs1, rhs2, speed); |
| break; |
| |
| case PLUS_EXPR: |
| case POINTER_PLUS_EXPR: |
| case MINUS_EXPR: |
| slsr_process_add (gs, rhs1, rhs2, speed); |
| break; |
| |
| case NEGATE_EXPR: |
| slsr_process_neg (gs, rhs1, speed); |
| break; |
| |
| CASE_CONVERT: |
| slsr_process_cast (gs, rhs1, speed); |
| break; |
| |
| case SSA_NAME: |
| slsr_process_copy (gs, rhs1, speed); |
| break; |
| |
| default: |
| ; |
| } |
| } |
| } |
| return NULL; |
| } |
| |
| /* Dump a candidate for debug. */ |
| |
| static void |
| dump_candidate (slsr_cand_t c) |
| { |
| fprintf (dump_file, "%3d [%d] ", c->cand_num, |
| gimple_bb (c->cand_stmt)->index); |
| print_gimple_stmt (dump_file, c->cand_stmt, 0); |
| switch (c->kind) |
| { |
| case CAND_MULT: |
| fputs (" MULT : (", dump_file); |
| print_generic_expr (dump_file, c->base_expr); |
| fputs (" + ", dump_file); |
| print_decs (c->index, dump_file); |
| fputs (") * ", dump_file); |
| if (TREE_CODE (c->stride) != INTEGER_CST |
| && c->stride_type != TREE_TYPE (c->stride)) |
| { |
| fputs ("(", dump_file); |
| print_generic_expr (dump_file, c->stride_type); |
| fputs (")", dump_file); |
| } |
| print_generic_expr (dump_file, c->stride); |
| fputs (" : ", dump_file); |
| break; |
| case CAND_ADD: |
| fputs (" ADD : ", dump_file); |
| print_generic_expr (dump_file, c->base_expr); |
| fputs (" + (", dump_file); |
| print_decs (c->index, dump_file); |
| fputs (" * ", dump_file); |
| if (TREE_CODE (c->stride) != INTEGER_CST |
| && c->stride_type != TREE_TYPE (c->stride)) |
| { |
| fputs ("(", dump_file); |
| print_generic_expr (dump_file, c->stride_type); |
| fputs (")", dump_file); |
| } |
| print_generic_expr (dump_file, c->stride); |
| fputs (") : ", dump_file); |
| break; |
| case CAND_REF: |
| fputs (" REF : ", dump_file); |
| print_generic_expr (dump_file, c->base_expr); |
| fputs (" + (", dump_file); |
| print_generic_expr (dump_file, c->stride); |
| fputs (") + ", dump_file); |
| print_decs (c->index, dump_file); |
| fputs (" : ", dump_file); |
| break; |
| case CAND_PHI: |
| fputs (" PHI : ", dump_file); |
| print_generic_expr (dump_file, c->base_expr); |
| fputs (" + (unknown * ", dump_file); |
| print_generic_expr (dump_file, c->stride); |
| fputs (") : ", dump_file); |
| break; |
| default: |
| gcc_unreachable (); |
| } |
| print_generic_expr (dump_file, c->cand_type); |
| fprintf (dump_file, "\n basis: %d dependent: %d sibling: %d\n", |
| c->basis, c->dependent, c->sibling); |
| fprintf (dump_file, |
| " next-interp: %d first-interp: %d dead-savings: %d\n", |
| c->next_interp, c->first_interp, c->dead_savings); |
| if (c->def_phi) |
| fprintf (dump_file, " phi: %d\n", c->def_phi); |
| fputs ("\n", dump_file); |
| } |
| |
| /* Dump the candidate vector for debug. */ |
| |
| static void |
| dump_cand_vec (void) |
| { |
| unsigned i; |
| slsr_cand_t c; |
| |
| fprintf (dump_file, "\nStrength reduction candidate vector:\n\n"); |
| |
| FOR_EACH_VEC_ELT (cand_vec, i, c) |
| if (c != NULL) |
| dump_candidate (c); |
| } |
| |
| /* Callback used to dump the candidate chains hash table. */ |
| |
| int |
| ssa_base_cand_dump_callback (cand_chain **slot, void *ignored ATTRIBUTE_UNUSED) |
| { |
| const_cand_chain_t chain = *slot; |
| cand_chain_t p; |
| |
| print_generic_expr (dump_file, chain->base_expr); |
| fprintf (dump_file, " -> %d", chain->cand->cand_num); |
| |
| for (p = chain->next; p; p = p->next) |
| fprintf (dump_file, " -> %d", p->cand->cand_num); |
| |
| fputs ("\n", dump_file); |
| return 1; |
| } |
| |
| /* Dump the candidate chains. */ |
| |
| static void |
| dump_cand_chains (void) |
| { |
| fprintf (dump_file, "\nStrength reduction candidate chains:\n\n"); |
| base_cand_map->traverse_noresize <void *, ssa_base_cand_dump_callback> |
| (NULL); |
| fputs ("\n", dump_file); |
| } |
| |
| /* Dump the increment vector for debug. */ |
| |
| static void |
| dump_incr_vec (void) |
| { |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| unsigned i; |
| |
| fprintf (dump_file, "\nIncrement vector:\n\n"); |
| |
| for (i = 0; i < incr_vec_len; i++) |
| { |
| fprintf (dump_file, "%3d increment: ", i); |
| print_decs (incr_vec[i].incr, dump_file); |
| fprintf (dump_file, "\n count: %d", incr_vec[i].count); |
| fprintf (dump_file, "\n cost: %d", incr_vec[i].cost); |
| fputs ("\n initializer: ", dump_file); |
| print_generic_expr (dump_file, incr_vec[i].initializer); |
| fputs ("\n\n", dump_file); |
| } |
| } |
| } |
| |
| /* Replace *EXPR in candidate C with an equivalent strength-reduced |
| data reference. */ |
| |
| static void |
| replace_ref (tree *expr, slsr_cand_t c) |
| { |
| tree add_expr, mem_ref, acc_type = TREE_TYPE (*expr); |
| unsigned HOST_WIDE_INT misalign; |
| unsigned align; |
| |
| /* Ensure the memory reference carries the minimum alignment |
| requirement for the data type. See PR58041. */ |
| get_object_alignment_1 (*expr, &align, &misalign); |
| if (misalign != 0) |
| align = least_bit_hwi (misalign); |
| if (align < TYPE_ALIGN (acc_type)) |
| acc_type = build_aligned_type (acc_type, align); |
| |
| add_expr = fold_build2 (POINTER_PLUS_EXPR, c->cand_type, |
| c->base_expr, c->stride); |
| mem_ref = fold_build2 (MEM_REF, acc_type, add_expr, |
| wide_int_to_tree (c->cand_type, c->index)); |
| |
| /* Gimplify the base addressing expression for the new MEM_REF tree. */ |
| gimple_stmt_iterator gsi = gsi_for_stmt (c->cand_stmt); |
| TREE_OPERAND (mem_ref, 0) |
| = force_gimple_operand_gsi (&gsi, TREE_OPERAND (mem_ref, 0), |
| /*simple_p=*/true, NULL, |
| /*before=*/true, GSI_SAME_STMT); |
| copy_ref_info (mem_ref, *expr); |
| *expr = mem_ref; |
| update_stmt (c->cand_stmt); |
| } |
| |
| /* Return true if CAND_REF candidate C is a valid memory reference. */ |
| |
| static bool |
| valid_mem_ref_cand_p (slsr_cand_t c) |
| { |
| if (TREE_CODE (TREE_OPERAND (c->stride, 1)) != INTEGER_CST) |
| return false; |
| |
| struct mem_address addr |
| = { NULL_TREE, c->base_expr, TREE_OPERAND (c->stride, 0), |
| TREE_OPERAND (c->stride, 1), wide_int_to_tree (sizetype, c->index) }; |
| |
| return |
| valid_mem_ref_p (TYPE_MODE (c->cand_type), TYPE_ADDR_SPACE (c->cand_type), |
| &addr); |
| } |
| |
| /* Replace CAND_REF candidate C, each sibling of candidate C, and each |
| dependent of candidate C with an equivalent strength-reduced data |
| reference. */ |
| |
| static void |
| replace_refs (slsr_cand_t c) |
| { |
| /* Replacing a chain of only 2 candidates which are valid memory references |
| is generally counter-productive because you cannot recoup the additional |
| calculation added in front of them. */ |
| if (c->basis == 0 |
| && c->dependent |
| && !lookup_cand (c->dependent)->dependent |
| && valid_mem_ref_cand_p (c) |
| && valid_mem_ref_cand_p (lookup_cand (c->dependent))) |
| return; |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("Replacing reference: ", dump_file); |
| print_gimple_stmt (dump_file, c->cand_stmt, 0); |
| } |
| |
| if (gimple_vdef (c->cand_stmt)) |
| { |
| tree *lhs = gimple_assign_lhs_ptr (c->cand_stmt); |
| replace_ref (lhs, c); |
| } |
| else |
| { |
| tree *rhs = gimple_assign_rhs1_ptr (c->cand_stmt); |
| replace_ref (rhs, c); |
| } |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("With: ", dump_file); |
| print_gimple_stmt (dump_file, c->cand_stmt, 0); |
| fputs ("\n", dump_file); |
| } |
| |
| if (c->sibling) |
| replace_refs (lookup_cand (c->sibling)); |
| |
| if (c->dependent) |
| replace_refs (lookup_cand (c->dependent)); |
| } |
| |
| /* Return TRUE if candidate C is dependent upon a PHI. */ |
| |
| static bool |
| phi_dependent_cand_p (slsr_cand_t c) |
| { |
| /* A candidate is not necessarily dependent upon a PHI just because |
| it has a phi definition for its base name. It may have a basis |
| that relies upon the same phi definition, in which case the PHI |
| is irrelevant to this candidate. */ |
| return (c->def_phi |
| && c->basis |
| && lookup_cand (c->basis)->def_phi != c->def_phi); |
| } |
| |
| /* Calculate the increment required for candidate C relative to |
| its basis. */ |
| |
| static widest_int |
| cand_increment (slsr_cand_t c) |
| { |
| slsr_cand_t basis; |
| |
| /* If the candidate doesn't have a basis, just return its own |
| index. This is useful in record_increments to help us find |
| an existing initializer. Also, if the candidate's basis is |
| hidden by a phi, then its own index will be the increment |
| from the newly introduced phi basis. */ |
| if (!c->basis || phi_dependent_cand_p (c)) |
| return c->index; |
| |
| basis = lookup_cand (c->basis); |
| gcc_assert (operand_equal_p (c->base_expr, basis->base_expr, 0)); |
| return c->index - basis->index; |
| } |
| |
| /* Calculate the increment required for candidate C relative to |
| its basis. If we aren't going to generate pointer arithmetic |
| for this candidate, return the absolute value of that increment |
| instead. */ |
| |
| static inline widest_int |
| cand_abs_increment (slsr_cand_t c) |
| { |
| widest_int increment = cand_increment (c); |
| |
| if (!address_arithmetic_p && wi::neg_p (increment)) |
| increment = -increment; |
| |
| return increment; |
| } |
| |
| /* Return TRUE iff candidate C has already been replaced under |
| another interpretation. */ |
| |
| static inline bool |
| cand_already_replaced (slsr_cand_t c) |
| { |
| return (gimple_bb (c->cand_stmt) == 0); |
| } |
| |
| /* Common logic used by replace_unconditional_candidate and |
| replace_conditional_candidate. */ |
| |
| static void |
| replace_mult_candidate (slsr_cand_t c, tree basis_name, widest_int bump) |
| { |
| tree target_type = TREE_TYPE (gimple_assign_lhs (c->cand_stmt)); |
| enum tree_code cand_code = gimple_assign_rhs_code (c->cand_stmt); |
| |
| /* It is not useful to replace casts, copies, negates, or adds of |
| an SSA name and a constant. */ |
| if (cand_code == SSA_NAME |
| || CONVERT_EXPR_CODE_P (cand_code) |
| || cand_code == PLUS_EXPR |
| || cand_code == POINTER_PLUS_EXPR |
| || cand_code == MINUS_EXPR |
| || cand_code == NEGATE_EXPR) |
| return; |
| |
| enum tree_code code = PLUS_EXPR; |
| tree bump_tree; |
| gimple *stmt_to_print = NULL; |
| |
| if (wi::neg_p (bump)) |
| { |
| code = MINUS_EXPR; |
| bump = -bump; |
| } |
| |
| /* It is possible that the resulting bump doesn't fit in target_type. |
| Abandon the replacement in this case. This does not affect |
| siblings or dependents of C. */ |
| if (bump != wi::ext (bump, TYPE_PRECISION (target_type), |
| TYPE_SIGN (target_type))) |
| return; |
| |
| bump_tree = wide_int_to_tree (target_type, bump); |
| |
| /* If the basis name and the candidate's LHS have incompatible types, |
| introduce a cast. */ |
| if (!useless_type_conversion_p (target_type, TREE_TYPE (basis_name))) |
| basis_name = introduce_cast_before_cand (c, target_type, basis_name); |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("Replacing: ", dump_file); |
| print_gimple_stmt (dump_file, c->cand_stmt, 0); |
| } |
| |
| if (bump == 0) |
| { |
| tree lhs = gimple_assign_lhs (c->cand_stmt); |
| gassign *copy_stmt = gimple_build_assign (lhs, basis_name); |
| gimple_stmt_iterator gsi = gsi_for_stmt (c->cand_stmt); |
| slsr_cand_t cc = lookup_cand (c->first_interp); |
| gimple_set_location (copy_stmt, gimple_location (c->cand_stmt)); |
| gsi_replace (&gsi, copy_stmt, false); |
| while (cc) |
| { |
| cc->cand_stmt = copy_stmt; |
| cc = lookup_cand (cc->next_interp); |
| } |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| stmt_to_print = copy_stmt; |
| } |
| else |
| { |
| tree rhs1, rhs2; |
| if (cand_code != NEGATE_EXPR) { |
| rhs1 = gimple_assign_rhs1 (c->cand_stmt); |
| rhs2 = gimple_assign_rhs2 (c->cand_stmt); |
| } |
| if (cand_code != NEGATE_EXPR |
| && ((operand_equal_p (rhs1, basis_name, 0) |
| && operand_equal_p (rhs2, bump_tree, 0)) |
| || (operand_equal_p (rhs1, bump_tree, 0) |
| && operand_equal_p (rhs2, basis_name, 0)))) |
| { |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("(duplicate, not actually replacing)", dump_file); |
| stmt_to_print = c->cand_stmt; |
| } |
| } |
| else |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (c->cand_stmt); |
| slsr_cand_t cc = lookup_cand (c->first_interp); |
| gimple_assign_set_rhs_with_ops (&gsi, code, basis_name, bump_tree); |
| update_stmt (gsi_stmt (gsi)); |
| while (cc) |
| { |
| cc->cand_stmt = gsi_stmt (gsi); |
| cc = lookup_cand (cc->next_interp); |
| } |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| stmt_to_print = gsi_stmt (gsi); |
| } |
| } |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("With: ", dump_file); |
| print_gimple_stmt (dump_file, stmt_to_print, 0); |
| fputs ("\n", dump_file); |
| } |
| } |
| |
| /* Replace candidate C with an add or subtract. Note that we only |
| operate on CAND_MULTs with known strides, so we will never generate |
| a POINTER_PLUS_EXPR. Each candidate X = (B + i) * S is replaced by |
| X = Y + ((i - i') * S), as described in the module commentary. The |
| folded value ((i - i') * S) is referred to here as the "bump." */ |
| |
| static void |
| replace_unconditional_candidate (slsr_cand_t c) |
| { |
| slsr_cand_t basis; |
| |
| if (cand_already_replaced (c)) |
| return; |
| |
| basis = lookup_cand (c->basis); |
| widest_int bump = cand_increment (c) * wi::to_widest (c->stride); |
| |
| replace_mult_candidate (c, gimple_assign_lhs (basis->cand_stmt), bump); |
| } |
| |
| /* Return the index in the increment vector of the given INCREMENT, |
| or -1 if not found. The latter can occur if more than |
| MAX_INCR_VEC_LEN increments have been found. */ |
| |
| static inline int |
| incr_vec_index (const widest_int &increment) |
| { |
| unsigned i; |
| |
| for (i = 0; i < incr_vec_len && increment != incr_vec[i].incr; i++) |
| ; |
| |
| if (i < incr_vec_len) |
| return i; |
| else |
| return -1; |
| } |
| |
| /* Create a new statement along edge E to add BASIS_NAME to the product |
| of INCREMENT and the stride of candidate C. Create and return a new |
| SSA name from *VAR to be used as the LHS of the new statement. |
| KNOWN_STRIDE is true iff C's stride is a constant. */ |
| |
| static tree |
| create_add_on_incoming_edge (slsr_cand_t c, tree basis_name, |
| widest_int increment, edge e, location_t loc, |
| bool known_stride) |
| { |
| tree lhs, basis_type; |
| gassign *new_stmt, *cast_stmt = NULL; |
| |
| /* If the add candidate along this incoming edge has the same |
| index as C's hidden basis, the hidden basis represents this |
| edge correctly. */ |
| if (increment == 0) |
| return basis_name; |
| |
| basis_type = TREE_TYPE (basis_name); |
| lhs = make_temp_ssa_name (basis_type, NULL, "slsr"); |
| |
| /* Occasionally people convert integers to pointers without a |
| cast, leading us into trouble if we aren't careful. */ |
| enum tree_code plus_code |
| = POINTER_TYPE_P (basis_type) ? POINTER_PLUS_EXPR : PLUS_EXPR; |
| |
| if (known_stride) |
| { |
| tree bump_tree; |
| enum tree_code code = plus_code; |
| widest_int bump = increment * wi::to_widest (c->stride); |
| if (wi::neg_p (bump) && !POINTER_TYPE_P (basis_type)) |
| { |
| code = MINUS_EXPR; |
| bump = -bump; |
| } |
| |
| tree stride_type = POINTER_TYPE_P (basis_type) ? sizetype : basis_type; |
| bump_tree = wide_int_to_tree (stride_type, bump); |
| new_stmt = gimple_build_assign (lhs, code, basis_name, bump_tree); |
| } |
| else |
| { |
| int i; |
| bool negate_incr = !POINTER_TYPE_P (basis_type) && wi::neg_p (increment); |
| i = incr_vec_index (negate_incr ? -increment : increment); |
| gcc_assert (i >= 0); |
| |
| if (incr_vec[i].initializer) |
| { |
| enum tree_code code = negate_incr ? MINUS_EXPR : plus_code; |
| new_stmt = gimple_build_assign (lhs, code, basis_name, |
| incr_vec[i].initializer); |
| } |
| else { |
| tree stride; |
| |
| if (!types_compatible_p (TREE_TYPE (c->stride), c->stride_type)) |
| { |
| tree cast_stride = make_temp_ssa_name (c->stride_type, NULL, |
| "slsr"); |
| cast_stmt = gimple_build_assign (cast_stride, NOP_EXPR, |
| c->stride); |
| stride = cast_stride; |
| } |
| else |
| stride = c->stride; |
| |
| if (increment == 1) |
| new_stmt = gimple_build_assign (lhs, plus_code, basis_name, stride); |
| else if (increment == -1) |
| new_stmt = gimple_build_assign (lhs, MINUS_EXPR, basis_name, stride); |
| else |
| gcc_unreachable (); |
| } |
| } |
| |
| if (cast_stmt) |
| { |
| gimple_set_location (cast_stmt, loc); |
| gsi_insert_on_edge (e, cast_stmt); |
| } |
| |
| gimple_set_location (new_stmt, loc); |
| gsi_insert_on_edge (e, new_stmt); |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| if (cast_stmt) |
| { |
| fprintf (dump_file, "Inserting cast on edge %d->%d: ", |
| e->src->index, e->dest->index); |
| print_gimple_stmt (dump_file, cast_stmt, 0); |
| } |
| fprintf (dump_file, "Inserting on edge %d->%d: ", e->src->index, |
| e->dest->index); |
| print_gimple_stmt (dump_file, new_stmt, 0); |
| } |
| |
| return lhs; |
| } |
| |
| /* Clear the visited field for a tree of PHI candidates. */ |
| |
| static void |
| clear_visited (gphi *phi) |
| { |
| unsigned i; |
| slsr_cand_t phi_cand = *stmt_cand_map->get (phi); |
| |
| if (phi_cand->visited) |
| { |
| phi_cand->visited = 0; |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| tree arg = gimple_phi_arg_def (phi, i); |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| clear_visited (as_a <gphi *> (arg_def)); |
| } |
| } |
| } |
| |
| /* Recursive helper function for create_phi_basis. */ |
| |
| static tree |
| create_phi_basis_1 (slsr_cand_t c, gimple *from_phi, tree basis_name, |
| location_t loc, bool known_stride) |
| { |
| int i; |
| tree name, phi_arg; |
| gphi *phi; |
| slsr_cand_t basis = lookup_cand (c->basis); |
| int nargs = gimple_phi_num_args (from_phi); |
| basic_block phi_bb = gimple_bb (from_phi); |
| slsr_cand_t phi_cand = *stmt_cand_map->get (from_phi); |
| auto_vec<tree> phi_args (nargs); |
| |
| if (phi_cand->visited) |
| return phi_cand->cached_basis; |
| phi_cand->visited = 1; |
| |
| /* Process each argument of the existing phi that represents |
| conditionally-executed add candidates. */ |
| for (i = 0; i < nargs; i++) |
| { |
| edge e = (*phi_bb->preds)[i]; |
| tree arg = gimple_phi_arg_def (from_phi, i); |
| tree feeding_def; |
| |
| /* If the phi argument is the base name of the CAND_PHI, then |
| this incoming arc should use the hidden basis. */ |
| if (operand_equal_p (arg, phi_cand->base_expr, 0)) |
| if (basis->index == 0) |
| feeding_def = gimple_assign_lhs (basis->cand_stmt); |
| else |
| { |
| widest_int incr = -basis->index; |
| feeding_def = create_add_on_incoming_edge (c, basis_name, incr, |
| e, loc, known_stride); |
| } |
| else |
| { |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| |
| /* If there is another phi along this incoming edge, we must |
| process it in the same fashion to ensure that all basis |
| adjustments are made along its incoming edges. */ |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| feeding_def = create_phi_basis_1 (c, arg_def, basis_name, |
| loc, known_stride); |
| else |
| { |
| slsr_cand_t arg_cand = base_cand_from_table (arg); |
| widest_int diff = arg_cand->index - basis->index; |
| feeding_def = create_add_on_incoming_edge (c, basis_name, diff, |
| e, loc, known_stride); |
| } |
| } |
| |
| /* Because of recursion, we need to save the arguments in a vector |
| so we can create the PHI statement all at once. Otherwise the |
| storage for the half-created PHI can be reclaimed. */ |
| phi_args.safe_push (feeding_def); |
| } |
| |
| /* Create the new phi basis. */ |
| name = make_temp_ssa_name (TREE_TYPE (basis_name), NULL, "slsr"); |
| phi = create_phi_node (name, phi_bb); |
| SSA_NAME_DEF_STMT (name) = phi; |
| |
| FOR_EACH_VEC_ELT (phi_args, i, phi_arg) |
| { |
| edge e = (*phi_bb->preds)[i]; |
| add_phi_arg (phi, phi_arg, e, loc); |
| } |
| |
| update_stmt (phi); |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fputs ("Introducing new phi basis: ", dump_file); |
| print_gimple_stmt (dump_file, phi, 0); |
| } |
| |
| phi_cand->cached_basis = name; |
| return name; |
| } |
| |
| /* Given a candidate C with BASIS_NAME being the LHS of C's basis which |
| is hidden by the phi node FROM_PHI, create a new phi node in the same |
| block as FROM_PHI. The new phi is suitable for use as a basis by C, |
| with its phi arguments representing conditional adjustments to the |
| hidden basis along conditional incoming paths. Those adjustments are |
| made by creating add statements (and sometimes recursively creating |
| phis) along those incoming paths. LOC is the location to attach to |
| the introduced statements. KNOWN_STRIDE is true iff C's stride is a |
| constant. */ |
| |
| static tree |
| create_phi_basis (slsr_cand_t c, gimple *from_phi, tree basis_name, |
| location_t loc, bool known_stride) |
| { |
| tree retval = create_phi_basis_1 (c, from_phi, basis_name, loc, |
| known_stride); |
| gcc_assert (retval); |
| clear_visited (as_a <gphi *> (from_phi)); |
| return retval; |
| } |
| |
| /* Given a candidate C whose basis is hidden by at least one intervening |
| phi, introduce a matching number of new phis to represent its basis |
| adjusted by conditional increments along possible incoming paths. Then |
| replace C as though it were an unconditional candidate, using the new |
| basis. */ |
| |
| static void |
| replace_conditional_candidate (slsr_cand_t c) |
| { |
| tree basis_name, name; |
| slsr_cand_t basis; |
| location_t loc; |
| |
| /* Look up the LHS SSA name from C's basis. This will be the |
| RHS1 of the adds we will introduce to create new phi arguments. */ |
| basis = lookup_cand (c->basis); |
| basis_name = gimple_assign_lhs (basis->cand_stmt); |
| |
| /* Create a new phi statement which will represent C's true basis |
| after the transformation is complete. */ |
| loc = gimple_location (c->cand_stmt); |
| name = create_phi_basis (c, lookup_cand (c->def_phi)->cand_stmt, |
| basis_name, loc, KNOWN_STRIDE); |
| |
| /* Replace C with an add of the new basis phi and a constant. */ |
| widest_int bump = c->index * wi::to_widest (c->stride); |
| |
| replace_mult_candidate (c, name, bump); |
| } |
| |
| /* Recursive helper function for phi_add_costs. SPREAD is a measure of |
| how many PHI nodes we have visited at this point in the tree walk. */ |
| |
| static int |
| phi_add_costs_1 (gimple *phi, slsr_cand_t c, int one_add_cost, int *spread) |
| { |
| unsigned i; |
| int cost = 0; |
| slsr_cand_t phi_cand = *stmt_cand_map->get (phi); |
| |
| if (phi_cand->visited) |
| return 0; |
| |
| phi_cand->visited = 1; |
| (*spread)++; |
| |
| /* If we work our way back to a phi that isn't dominated by the hidden |
| basis, this isn't a candidate for replacement. Indicate this by |
| returning an unreasonably high cost. It's not easy to detect |
| these situations when determining the basis, so we defer the |
| decision until now. */ |
| basic_block phi_bb = gimple_bb (phi); |
| slsr_cand_t basis = lookup_cand (c->basis); |
| basic_block basis_bb = gimple_bb (basis->cand_stmt); |
| |
| if (phi_bb == basis_bb || !dominated_by_p (CDI_DOMINATORS, phi_bb, basis_bb)) |
| return COST_INFINITE; |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| tree arg = gimple_phi_arg_def (phi, i); |
| |
| if (arg != phi_cand->base_expr) |
| { |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| { |
| cost += phi_add_costs_1 (arg_def, c, one_add_cost, spread); |
| |
| if (cost >= COST_INFINITE || *spread > MAX_SPREAD) |
| return COST_INFINITE; |
| } |
| else |
| { |
| slsr_cand_t arg_cand = base_cand_from_table (arg); |
| |
| if (arg_cand->index != c->index) |
| cost += one_add_cost; |
| } |
| } |
| } |
| |
| return cost; |
| } |
| |
| /* Compute the expected costs of inserting basis adjustments for |
| candidate C with phi-definition PHI. The cost of inserting |
| one adjustment is given by ONE_ADD_COST. If PHI has arguments |
| which are themselves phi results, recursively calculate costs |
| for those phis as well. */ |
| |
| static int |
| phi_add_costs (gimple *phi, slsr_cand_t c, int one_add_cost) |
| { |
| int spread = 0; |
| int retval = phi_add_costs_1 (phi, c, one_add_cost, &spread); |
| clear_visited (as_a <gphi *> (phi)); |
| return retval; |
| } |
| /* For candidate C, each sibling of candidate C, and each dependent of |
| candidate C, determine whether the candidate is dependent upon a |
| phi that hides its basis. If not, replace the candidate unconditionally. |
| Otherwise, determine whether the cost of introducing compensation code |
| for the candidate is offset by the gains from strength reduction. If |
| so, replace the candidate and introduce the compensation code. */ |
| |
| static void |
| replace_uncond_cands_and_profitable_phis (slsr_cand_t c) |
| { |
| if (phi_dependent_cand_p (c)) |
| { |
| /* A multiply candidate with a stride of 1 is just an artifice |
| of a copy or cast; there is no value in replacing it. */ |
| if (c->kind == CAND_MULT && wi::to_widest (c->stride) != 1) |
| { |
| /* A candidate dependent upon a phi will replace a multiply by |
| a constant with an add, and will insert at most one add for |
| each phi argument. Add these costs with the potential dead-code |
| savings to determine profitability. */ |
| bool speed = optimize_bb_for_speed_p (gimple_bb (c->cand_stmt)); |
| int mult_savings = stmt_cost (c->cand_stmt, speed); |
| gimple *phi = lookup_cand (c->def_phi)->cand_stmt; |
| tree phi_result = gimple_phi_result (phi); |
| int one_add_cost = add_cost (speed, |
| TYPE_MODE (TREE_TYPE (phi_result))); |
| int add_costs = one_add_cost + phi_add_costs (phi, c, one_add_cost); |
| int cost = add_costs - mult_savings - c->dead_savings; |
| |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| { |
| fprintf (dump_file, " Conditional candidate %d:\n", c->cand_num); |
| fprintf (dump_file, " add_costs = %d\n", add_costs); |
| fprintf (dump_file, " mult_savings = %d\n", mult_savings); |
| fprintf (dump_file, " dead_savings = %d\n", c->dead_savings); |
| fprintf (dump_file, " cost = %d\n", cost); |
| if (cost <= COST_NEUTRAL) |
| fputs (" Replacing...\n", dump_file); |
| else |
| fputs (" Not replaced.\n", dump_file); |
| } |
| |
| if (cost <= COST_NEUTRAL) |
| replace_conditional_candidate (c); |
| } |
| } |
| else |
| replace_unconditional_candidate (c); |
| |
| if (c->sibling) |
| replace_uncond_cands_and_profitable_phis (lookup_cand (c->sibling)); |
| |
| if (c->dependent) |
| replace_uncond_cands_and_profitable_phis (lookup_cand (c->dependent)); |
| } |
| |
| /* Count the number of candidates in the tree rooted at C that have |
| not already been replaced under other interpretations. */ |
| |
| static int |
| count_candidates (slsr_cand_t c) |
| { |
| unsigned count = cand_already_replaced (c) ? 0 : 1; |
| |
| if (c->sibling) |
| count += count_candidates (lookup_cand (c->sibling)); |
| |
| if (c->dependent) |
| count += count_candidates (lookup_cand (c->dependent)); |
| |
| return count; |
| } |
| |
| /* Increase the count of INCREMENT by one in the increment vector. |
| INCREMENT is associated with candidate C. If INCREMENT is to be |
| conditionally executed as part of a conditional candidate replacement, |
| IS_PHI_ADJUST is true, otherwise false. If an initializer |
| T_0 = stride * I is provided by a candidate that dominates all |
| candidates with the same increment, also record T_0 for subsequent use. */ |
| |
| static void |
| record_increment (slsr_cand_t c, widest_int increment, bool is_phi_adjust) |
| { |
| bool found = false; |
| unsigned i; |
| |
| /* Treat increments that differ only in sign as identical so as to |
| share initializers, unless we are generating pointer arithmetic. */ |
| if (!address_arithmetic_p && wi::neg_p (increment)) |
| increment = -increment; |
| |
| for (i = 0; i < incr_vec_len; i++) |
| { |
| if (incr_vec[i].incr == increment) |
| { |
| incr_vec[i].count++; |
| found = true; |
| |
| /* If we previously recorded an initializer that doesn't |
| dominate this candidate, it's not going to be useful to |
| us after all. */ |
| if (incr_vec[i].initializer |
| && !dominated_by_p (CDI_DOMINATORS, |
| gimple_bb (c->cand_stmt), |
| incr_vec[i].init_bb)) |
| { |
| incr_vec[i].initializer = NULL_TREE; |
| incr_vec[i].init_bb = NULL; |
| } |
| |
| break; |
| } |
| } |
| |
| if (!found && incr_vec_len < MAX_INCR_VEC_LEN - 1) |
| { |
| /* The first time we see an increment, create the entry for it. |
| If this is the root candidate which doesn't have a basis, set |
| the count to zero. We're only processing it so it can possibly |
| provide an initializer for other candidates. */ |
| incr_vec[incr_vec_len].incr = increment; |
| incr_vec[incr_vec_len].count = c->basis || is_phi_adjust ? 1 : 0; |
| incr_vec[incr_vec_len].cost = COST_INFINITE; |
| |
| /* Optimistically record the first occurrence of this increment |
| as providing an initializer (if it does); we will revise this |
| opinion later if it doesn't dominate all other occurrences. |
| Exception: increments of 0, 1 never need initializers; |
| and phi adjustments don't ever provide initializers. */ |
| if (c->kind == CAND_ADD |
| && !is_phi_adjust |
| && c->index == increment |
| && (increment > 1 || increment < 0) |
| && (gimple_assign_rhs_code (c->cand_stmt) == PLUS_EXPR |
| || gimple_assign_rhs_code (c->cand_stmt) == POINTER_PLUS_EXPR)) |
| { |
| tree t0 = NULL_TREE; |
| tree rhs1 = gimple_assign_rhs1 (c->cand_stmt); |
| tree rhs2 = gimple_assign_rhs2 (c->cand_stmt); |
| if (operand_equal_p (rhs1, c->base_expr, 0)) |
| t0 = rhs2; |
| else if (operand_equal_p (rhs2, c->base_expr, 0)) |
| t0 = rhs1; |
| if (t0 |
| && SSA_NAME_DEF_STMT (t0) |
| && gimple_bb (SSA_NAME_DEF_STMT (t0))) |
| { |
| incr_vec[incr_vec_len].initializer = t0; |
| incr_vec[incr_vec_len++].init_bb |
| = gimple_bb (SSA_NAME_DEF_STMT (t0)); |
| } |
| else |
| { |
| incr_vec[incr_vec_len].initializer = NULL_TREE; |
| incr_vec[incr_vec_len++].init_bb = NULL; |
| } |
| } |
| else |
| { |
| incr_vec[incr_vec_len].initializer = NULL_TREE; |
| incr_vec[incr_vec_len++].init_bb = NULL; |
| } |
| } |
| } |
| |
| /* Recursive helper function for record_phi_increments. */ |
| |
| static void |
| record_phi_increments_1 (slsr_cand_t basis, gimple *phi) |
| { |
| unsigned i; |
| slsr_cand_t phi_cand = *stmt_cand_map->get (phi); |
| |
| if (phi_cand->visited) |
| return; |
| phi_cand->visited = 1; |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| tree arg = gimple_phi_arg_def (phi, i); |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| record_phi_increments_1 (basis, arg_def); |
| else |
| { |
| widest_int diff; |
| |
| if (operand_equal_p (arg, phi_cand->base_expr, 0)) |
| { |
| diff = -basis->index; |
| record_increment (phi_cand, diff, PHI_ADJUST); |
| } |
| else |
| { |
| slsr_cand_t arg_cand = base_cand_from_table (arg); |
| diff = arg_cand->index - basis->index; |
| record_increment (arg_cand, diff, PHI_ADJUST); |
| } |
| } |
| } |
| } |
| |
| /* Given phi statement PHI that hides a candidate from its BASIS, find |
| the increments along each incoming arc (recursively handling additional |
| phis that may be present) and record them. These increments are the |
| difference in index between the index-adjusting statements and the |
| index of the basis. */ |
| |
| static void |
| record_phi_increments (slsr_cand_t basis, gimple *phi) |
| { |
| record_phi_increments_1 (basis, phi); |
| clear_visited (as_a <gphi *> (phi)); |
| } |
| |
| /* Determine how many times each unique increment occurs in the set |
| of candidates rooted at C's parent, recording the data in the |
| increment vector. For each unique increment I, if an initializer |
| T_0 = stride * I is provided by a candidate that dominates all |
| candidates with the same increment, also record T_0 for subsequent |
| use. */ |
| |
| static void |
| record_increments (slsr_cand_t c) |
| { |
| if (!cand_already_replaced (c)) |
| { |
| if (!phi_dependent_cand_p (c)) |
| record_increment (c, cand_increment (c), NOT_PHI_ADJUST); |
| else |
| { |
| /* A candidate with a basis hidden by a phi will have one |
| increment for its relationship to the index represented by |
| the phi, and potentially additional increments along each |
| incoming edge. For the root of the dependency tree (which |
| has no basis), process just the initial index in case it has |
| an initializer that can be used by subsequent candidates. */ |
| record_increment (c, c->index, NOT_PHI_ADJUST); |
| |
| if (c->basis) |
| record_phi_increments (lookup_cand (c->basis), |
| lookup_cand (c->def_phi)->cand_stmt); |
| } |
| } |
| |
| if (c->sibling) |
| record_increments (lookup_cand (c->sibling)); |
| |
| if (c->dependent) |
| record_increments (lookup_cand (c->dependent)); |
| } |
| |
| /* Recursive helper function for phi_incr_cost. */ |
| |
| static int |
| phi_incr_cost_1 (slsr_cand_t c, const widest_int &incr, gimple *phi, |
| int *savings) |
| { |
| unsigned i; |
| int cost = 0; |
| slsr_cand_t basis = lookup_cand (c->basis); |
| slsr_cand_t phi_cand = *stmt_cand_map->get (phi); |
| |
| if (phi_cand->visited) |
| return 0; |
| phi_cand->visited = 1; |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| tree arg = gimple_phi_arg_def (phi, i); |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| { |
| int feeding_savings = 0; |
| tree feeding_var = gimple_phi_result (arg_def); |
| cost += phi_incr_cost_1 (c, incr, arg_def, &feeding_savings); |
| if (uses_consumed_by_stmt (feeding_var, phi)) |
| *savings += feeding_savings; |
| } |
| else |
| { |
| widest_int diff; |
| slsr_cand_t arg_cand; |
| |
| /* When the PHI argument is just a pass-through to the base |
| expression of the hidden basis, the difference is zero minus |
| the index of the basis. There is no potential savings by |
| eliminating a statement in this case. */ |
| if (operand_equal_p (arg, phi_cand->base_expr, 0)) |
| { |
| arg_cand = (slsr_cand_t)NULL; |
| diff = -basis->index; |
| } |
| else |
| { |
| arg_cand = base_cand_from_table (arg); |
| diff = arg_cand->index - basis->index; |
| } |
| |
| if (incr == diff) |
| { |
| tree basis_lhs = gimple_assign_lhs (basis->cand_stmt); |
| cost += add_cost (true, TYPE_MODE (TREE_TYPE (basis_lhs))); |
| if (arg_cand) |
| { |
| tree lhs = gimple_assign_lhs (arg_cand->cand_stmt); |
| if (uses_consumed_by_stmt (lhs, phi)) |
| *savings += stmt_cost (arg_cand->cand_stmt, true); |
| } |
| } |
| } |
| } |
| |
| return cost; |
| } |
| |
| /* Add up and return the costs of introducing add statements that |
| require the increment INCR on behalf of candidate C and phi |
| statement PHI. Accumulate into *SAVINGS the potential savings |
| from removing existing statements that feed PHI and have no other |
| uses. */ |
| |
| static int |
| phi_incr_cost (slsr_cand_t c, const widest_int &incr, gimple *phi, |
| int *savings) |
| { |
| int retval = phi_incr_cost_1 (c, incr, phi, savings); |
| clear_visited (as_a <gphi *> (phi)); |
| return retval; |
| } |
| |
| /* Return the first candidate in the tree rooted at C that has not |
| already been replaced, favoring siblings over dependents. */ |
| |
| static slsr_cand_t |
| unreplaced_cand_in_tree (slsr_cand_t c) |
| { |
| if (!cand_already_replaced (c)) |
| return c; |
| |
| if (c->sibling) |
| { |
| slsr_cand_t sib = unreplaced_cand_in_tree (lookup_cand (c->sibling)); |
| if (sib) |
| return sib; |
| } |
| |
| if (c->dependent) |
| { |
| slsr_cand_t dep = unreplaced_cand_in_tree (lookup_cand (c->dependent)); |
| if (dep) |
| return dep; |
| } |
| |
| return NULL; |
| } |
| |
| /* Return TRUE if the candidates in the tree rooted at C should be |
| optimized for speed, else FALSE. We estimate this based on the block |
| containing the most dominant candidate in the tree that has not yet |
| been replaced. */ |
| |
| static bool |
| optimize_cands_for_speed_p (slsr_cand_t c) |
| { |
| slsr_cand_t c2 = unreplaced_cand_in_tree (c); |
| gcc_assert (c2); |
| return optimize_bb_for_speed_p (gimple_bb (c2->cand_stmt)); |
| } |
| |
| /* Add COST_IN to the lowest cost of any dependent path starting at |
| candidate C or any of its siblings, counting only candidates along |
| such paths with increment INCR. Assume that replacing a candidate |
| reduces cost by REPL_SAVINGS. Also account for savings from any |
| statements that would go dead. If COUNT_PHIS is true, include |
| costs of introducing feeding statements for conditional candidates. */ |
| |
| static int |
| lowest_cost_path (int cost_in, int repl_savings, slsr_cand_t c, |
| const widest_int &incr, bool count_phis) |
| { |
| int local_cost, sib_cost, savings = 0; |
| widest_int cand_incr = cand_abs_increment (c); |
| |
| if (cand_already_replaced (c)) |
| local_cost = cost_in; |
| else if (incr == cand_incr) |
| local_cost = cost_in - repl_savings - c->dead_savings; |
| else |
| local_cost = cost_in - c->dead_savings; |
| |
| if (count_phis |
| && phi_dependent_cand_p (c) |
| && !cand_already_replaced (c)) |
| { |
| gimple *phi = lookup_cand (c->def_phi)->cand_stmt; |
| local_cost += phi_incr_cost (c, incr, phi, &savings); |
| |
| if (uses_consumed_by_stmt (gimple_phi_result (phi), c->cand_stmt)) |
| local_cost -= savings; |
| } |
| |
| if (c->dependent) |
| local_cost = lowest_cost_path (local_cost, repl_savings, |
| lookup_cand (c->dependent), incr, |
| count_phis); |
| |
| if (c->sibling) |
| { |
| sib_cost = lowest_cost_path (cost_in, repl_savings, |
| lookup_cand (c->sibling), incr, |
| count_phis); |
| local_cost = MIN (local_cost, sib_cost); |
| } |
| |
| return local_cost; |
| } |
| |
| /* Compute the total savings that would accrue from all replacements |
| in the candidate tree rooted at C, counting only candidates with |
| increment INCR. Assume that replacing a candidate reduces cost |
| by REPL_SAVINGS. Also account for savings from statements that |
| would go dead. */ |
| |
| static int |
| total_savings (int repl_savings, slsr_cand_t c, const widest_int &incr, |
| bool count_phis) |
| { |
| int savings = 0; |
| widest_int cand_incr = cand_abs_increment (c); |
| |
| if (incr == cand_incr && !cand_already_replaced (c)) |
| savings += repl_savings + c->dead_savings; |
| |
| if (count_phis |
| && phi_dependent_cand_p (c) |
| && !cand_already_replaced (c)) |
| { |
| int phi_savings = 0; |
| gimple *phi = lookup_cand (c->def_phi)->cand_stmt; |
| savings -= phi_incr_cost (c, incr, phi, &phi_savings); |
| |
| if (uses_consumed_by_stmt (gimple_phi_result (phi), c->cand_stmt)) |
| savings += phi_savings; |
| } |
| |
| if (c->dependent) |
| savings += total_savings (repl_savings, lookup_cand (c->dependent), incr, |
| count_phis); |
| |
| if (c->sibling) |
| savings += total_savings (repl_savings, lookup_cand (c->sibling), incr, |
| count_phis); |
| |
| return savings; |
| } |
| |
| /* Use target-specific costs to determine and record which increments |
| in the current candidate tree are profitable to replace, assuming |
| MODE and SPEED. FIRST_DEP is the first dependent of the root of |
| the candidate tree. |
| |
| One slight limitation here is that we don't account for the possible |
| introduction of casts in some cases. See replace_one_candidate for |
| the cases where these are introduced. This should probably be cleaned |
| up sometime. */ |
| |
| static void |
| analyze_increments (slsr_cand_t first_dep, machine_mode mode, bool speed) |
| { |
| unsigned i; |
| |
| for (i = 0; i < incr_vec_len; i++) |
| { |
| HOST_WIDE_INT incr = incr_vec[i].incr.to_shwi (); |
| |
| /* If somehow this increment is bigger than a HWI, we won't |
| be optimizing candidates that use it. And if the increment |
| has a count of zero, nothing will be done with it. */ |
| if (!wi::fits_shwi_p (incr_vec[i].incr) || !incr_vec[i].count) |
| incr_vec[i].cost = COST_INFINITE; |
| |
| /* Increments of 0, 1, and -1 are always profitable to replace, |
| because they always replace a multiply or add with an add or |
| copy, and may cause one or more existing instructions to go |
| dead. Exception: -1 can't be assumed to be profitable for |
| pointer addition. */ |
| else if (incr == 0 |
| || incr == 1 |
| || (incr == -1 |
| && !POINTER_TYPE_P (first_dep->cand_type))) |
| incr_vec[i].cost = COST_NEUTRAL; |
| |
| /* If we need to add an initializer, give up if a cast from the |
| candidate's type to its stride's type can lose precision. |
| Note that this already takes into account that the stride may |
| have been cast to a wider type, in which case this test won't |
| fire. Example: |
| |
| short int _1; |
| _2 = (int) _1; |
| _3 = _2 * 10; |
| _4 = x + _3; ADD: x + (10 * (int)_1) : int |
| _5 = _2 * 15; |
| _6 = x + _5; ADD: x + (15 * (int)_1) : int |
| |
| Although the stride was a short int initially, the stride |
| used in the analysis has been widened to an int, and such |
| widening will be done in the initializer as well. */ |
| else if (!incr_vec[i].initializer |
| && TREE_CODE (first_dep->stride) != INTEGER_CST |
| && !legal_cast_p_1 (first_dep->stride_type, |
| TREE_TYPE (gimple_assign_lhs |
| (first_dep->cand_stmt)))) |
| incr_vec[i].cost = COST_INFINITE; |
| |
| /* If we need to add an initializer, make sure we don't introduce |
| a multiply by a pointer type, which can happen in certain cast |
| scenarios. */ |
| else if (!incr_vec[i].initializer |
| && TREE_CODE (first_dep->stride) != INTEGER_CST |
| && POINTER_TYPE_P (first_dep->stride_type)) |
| incr_vec[i].cost = COST_INFINITE; |
| |
| /* For any other increment, if this is a multiply candidate, we |
| must introduce a temporary T and initialize it with |
| T_0 = stride * increment. When optimizing for speed, walk the |
| candidate tree to calculate the best cost reduction along any |
| path; if it offsets the fixed cost of inserting the initializer, |
| replacing the increment is profitable. When optimizing for |
| size, instead calculate the total cost reduction from replacing |
| all candidates with this increment. */ |
| else if (first_dep->kind == CAND_MULT) |
| { |
| int cost = mult_by_coeff_cost (incr, mode, speed); |
| int repl_savings; |
| |
| if (tree_fits_shwi_p (first_dep->stride)) |
| { |
| HOST_WIDE_INT hwi_stride = tree_to_shwi (first_dep->stride); |
| repl_savings = mult_by_coeff_cost (hwi_stride, mode, speed); |
| } |
| else |
| repl_savings = mul_cost (speed, mode); |
| repl_savings -= add_cost (speed, mode); |
| |
| if (speed) |
| cost = lowest_cost_path (cost, repl_savings, first_dep, |
| incr_vec[i].incr, COUNT_PHIS); |
| else |
| cost -= total_savings (repl_savings, first_dep, incr_vec[i].incr, |
| COUNT_PHIS); |
| |
| incr_vec[i].cost = cost; |
| } |
| |
| /* If this is an add candidate, the initializer may already |
| exist, so only calculate the cost of the initializer if it |
| doesn't. We are replacing one add with another here, so the |
| known replacement savings is zero. We will account for removal |
| of dead instructions in lowest_cost_path or total_savings. */ |
| else |
| { |
| int cost = 0; |
| if (!incr_vec[i].initializer) |
| cost = mult_by_coeff_cost (incr, mode, speed); |
| |
| if (speed) |
| cost = lowest_cost_path (cost, 0, first_dep, incr_vec[i].incr, |
| DONT_COUNT_PHIS); |
| else |
| cost -= total_savings (0, first_dep, incr_vec[i].incr, |
| DONT_COUNT_PHIS); |
| |
| incr_vec[i].cost = cost; |
| } |
| } |
| } |
| |
| /* Return the nearest common dominator of BB1 and BB2. If the blocks |
| are identical, return the earlier of C1 and C2 in *WHERE. Otherwise, |
| if the NCD matches BB1, return C1 in *WHERE; if the NCD matches BB2, |
| return C2 in *WHERE; and if the NCD matches neither, return NULL in |
| *WHERE. Note: It is possible for one of C1 and C2 to be NULL. */ |
| |
| static basic_block |
| ncd_for_two_cands (basic_block bb1, basic_block bb2, |
| slsr_cand_t c1, slsr_cand_t c2, slsr_cand_t *where) |
| { |
| basic_block ncd; |
| |
| if (!bb1) |
| { |
| *where = c2; |
| return bb2; |
| } |
| |
| if (!bb2) |
| { |
| *where = c1; |
| return bb1; |
| } |
| |
| ncd = nearest_common_dominator (CDI_DOMINATORS, bb1, bb2); |
| |
| /* If both candidates are in the same block, the earlier |
| candidate wins. */ |
| if (bb1 == ncd && bb2 == ncd) |
| { |
| if (!c1 || (c2 && c2->cand_num < c1->cand_num)) |
| *where = c2; |
| else |
| *where = c1; |
| } |
| |
| /* Otherwise, if one of them produced a candidate in the |
| dominator, that one wins. */ |
| else if (bb1 == ncd) |
| *where = c1; |
| |
| else if (bb2 == ncd) |
| *where = c2; |
| |
| /* If neither matches the dominator, neither wins. */ |
| else |
| *where = NULL; |
| |
| return ncd; |
| } |
| |
| /* Consider all candidates that feed PHI. Find the nearest common |
| dominator of those candidates requiring the given increment INCR. |
| Further find and return the nearest common dominator of this result |
| with block NCD. If the returned block contains one or more of the |
| candidates, return the earliest candidate in the block in *WHERE. */ |
| |
| static basic_block |
| ncd_with_phi (slsr_cand_t c, const widest_int &incr, gphi *phi, |
| basic_block ncd, slsr_cand_t *where) |
| { |
| unsigned i; |
| slsr_cand_t basis = lookup_cand (c->basis); |
| slsr_cand_t phi_cand = *stmt_cand_map->get (phi); |
| |
| for (i = 0; i < gimple_phi_num_args (phi); i++) |
| { |
| tree arg = gimple_phi_arg_def (phi, i); |
| gimple *arg_def = SSA_NAME_DEF_STMT (arg); |
| |
| if (gimple_code (arg_def) == GIMPLE_PHI) |
| ncd = ncd_with_phi (c, incr, as_a <gphi *> (arg_def), ncd, where); |
| else |
| { |
| widest_int diff; |
| |
| if (operand_equal_p (arg, phi_cand->base_expr, 0)) |
| diff = -basis->index; |
| else |
| { |
| slsr_cand_t arg_cand = base_cand_from_table (arg); |
| diff = arg_cand->index - basis->index; |
| } |
| |
| basic_block pred = gimple_phi_arg_edge (phi, i)->src; |
| |
| if ((incr == diff) || (!address_arithmetic_p && incr == -diff)) |
| ncd = ncd_for_two_cands (ncd, pred, *where, NULL, where); |
| } |
| } |
| |
| return ncd; |
| } |
| |
| /* Consider the candidate C togethe
|