| /* Callgraph clones |
| Copyright (C) 2003-2021 Free Software Foundation, Inc. |
| Contributed by Jan Hubicka |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it under |
| the terms of the GNU General Public License as published by the Free |
| Software Foundation; either version 3, or (at your option) any later |
| version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| /* This module provide facilities for cloning functions. I.e. creating |
| new functions based on existing functions with simple modifications, |
| such as replacement of parameters. |
| |
| To allow whole program optimization without actual presence of function |
| bodies, an additional infrastructure is provided for so-called virtual |
| clones |
| |
| A virtual clone in the callgraph is a function that has no |
| associated body, just a description of how to create its body based |
| on a different function (which itself may be a virtual clone). |
| |
| The description of function modifications includes adjustments to |
| the function's signature (which allows, for example, removing or |
| adding function arguments), substitutions to perform on the |
| function body, and, for inlined functions, a pointer to the |
| function that it will be inlined into. |
| |
| It is also possible to redirect any edge of the callgraph from a |
| function to its virtual clone. This implies updating of the call |
| site to adjust for the new function signature. |
| |
| Most of the transformations performed by inter-procedural |
| optimizations can be represented via virtual clones. For |
| instance, a constant propagation pass can produce a virtual clone |
| of the function which replaces one of its arguments by a |
| constant. The inliner can represent its decisions by producing a |
| clone of a function whose body will be later integrated into |
| a given function. |
| |
| Using virtual clones, the program can be easily updated |
| during the Execute stage, solving most of pass interactions |
| problems that would otherwise occur during Transform. |
| |
| Virtual clones are later materialized in the LTRANS stage and |
| turned into real functions. Passes executed after the virtual |
| clone were introduced also perform their Transform stage |
| on new functions, so for a pass there is no significant |
| difference between operating on a real function or a virtual |
| clone introduced before its Execute stage. |
| |
| Optimization passes then work on virtual clones introduced before |
| their Execute stage as if they were real functions. The |
| only difference is that clones are not visible during the |
| Generate Summary stage. */ |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "target.h" |
| #include "rtl.h" |
| #include "tree.h" |
| #include "gimple.h" |
| #include "stringpool.h" |
| #include "cgraph.h" |
| #include "lto-streamer.h" |
| #include "tree-eh.h" |
| #include "tree-cfg.h" |
| #include "tree-inline.h" |
| #include "dumpfile.h" |
| #include "gimple-pretty-print.h" |
| #include "alloc-pool.h" |
| #include "symbol-summary.h" |
| #include "tree-vrp.h" |
| #include "ipa-prop.h" |
| #include "ipa-fnsummary.h" |
| #include "symtab-thunks.h" |
| #include "symtab-clones.h" |
| |
| /* Create clone of edge in the node N represented by CALL_EXPR |
| the callgraph. */ |
| |
| cgraph_edge * |
| cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, |
| profile_count num, profile_count den, |
| bool update_original) |
| { |
| cgraph_edge *new_edge; |
| profile_count::adjust_for_ipa_scaling (&num, &den); |
| profile_count prof_count = count.apply_scale (num, den); |
| |
| if (indirect_unknown_callee) |
| { |
| tree decl; |
| |
| if (call_stmt && (decl = gimple_call_fndecl (call_stmt)) |
| /* When the call is speculative, we need to resolve it |
| via cgraph_resolve_speculation and not here. */ |
| && !speculative) |
| { |
| cgraph_node *callee = cgraph_node::get (decl); |
| gcc_checking_assert (callee); |
| new_edge = n->create_edge (callee, call_stmt, prof_count, true); |
| } |
| else |
| { |
| new_edge = n->create_indirect_edge (call_stmt, |
| indirect_info->ecf_flags, |
| prof_count, true); |
| *new_edge->indirect_info = *indirect_info; |
| } |
| } |
| else |
| { |
| new_edge = n->create_edge (callee, call_stmt, prof_count, true); |
| if (indirect_info) |
| { |
| new_edge->indirect_info |
| = ggc_cleared_alloc<cgraph_indirect_call_info> (); |
| *new_edge->indirect_info = *indirect_info; |
| } |
| } |
| |
| new_edge->inline_failed = inline_failed; |
| new_edge->indirect_inlining_edge = indirect_inlining_edge; |
| if (!call_stmt) |
| new_edge->lto_stmt_uid = stmt_uid; |
| new_edge->speculative_id = speculative_id; |
| /* Clone flags that depend on call_stmt availability manually. */ |
| new_edge->can_throw_external = can_throw_external; |
| new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p; |
| new_edge->speculative = speculative; |
| new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor; |
| |
| /* Update IPA profile. Local profiles need no updating in original. */ |
| if (update_original) |
| count = count.combine_with_ipa_count_within (count.ipa () |
| - new_edge->count.ipa (), |
| caller->count); |
| symtab->call_edge_duplication_hooks (this, new_edge); |
| return new_edge; |
| } |
| |
| /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private |
| clone or its thunk. */ |
| |
| static void |
| set_new_clone_decl_and_node_flags (cgraph_node *new_node) |
| { |
| DECL_EXTERNAL (new_node->decl) = 0; |
| TREE_PUBLIC (new_node->decl) = 0; |
| DECL_COMDAT (new_node->decl) = 0; |
| DECL_WEAK (new_node->decl) = 0; |
| DECL_VIRTUAL_P (new_node->decl) = 0; |
| DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0; |
| DECL_STATIC_DESTRUCTOR (new_node->decl) = 0; |
| DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0); |
| DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0); |
| DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0; |
| |
| new_node->externally_visible = 0; |
| new_node->local = 1; |
| new_node->lowered = true; |
| } |
| |
| /* Duplicate thunk THUNK if necessary but make it to refer to NODE. |
| ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted. |
| Function can return NODE if no thunk is necessary, which can happen when |
| thunk is this_adjusting but we are removing this parameter. */ |
| |
| static cgraph_node * |
| duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node) |
| { |
| cgraph_node *new_thunk, *thunk_of; |
| thunk_of = thunk->callees->callee->ultimate_alias_target (); |
| |
| if (thunk_of->thunk) |
| node = duplicate_thunk_for_node (thunk_of, node); |
| |
| if (!DECL_ARGUMENTS (thunk->decl)) |
| thunk->get_untransformed_body (); |
| |
| thunk_info *i = thunk_info::get (thunk); |
| cgraph_edge *cs; |
| for (cs = node->callers; cs; cs = cs->next_caller) |
| if (cs->caller->thunk) |
| { |
| thunk_info *i2 = thunk_info::get (cs->caller); |
| if (*i2 == *i) |
| return cs->caller; |
| } |
| |
| tree new_decl; |
| clone_info *info = clone_info::get (node); |
| if (info && info->param_adjustments) |
| { |
| /* We do not need to duplicate this_adjusting thunks if we have removed |
| this. */ |
| if (i->this_adjusting |
| && !info->param_adjustments->first_param_intact_p ()) |
| return node; |
| |
| new_decl = copy_node (thunk->decl); |
| ipa_param_body_adjustments body_adj (info->param_adjustments, |
| new_decl); |
| body_adj.modify_formal_parameters (); |
| } |
| else |
| new_decl = copy_node (thunk->decl); |
| |
| gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl)); |
| gcc_checking_assert (!DECL_INITIAL (new_decl)); |
| gcc_checking_assert (!DECL_RESULT (new_decl)); |
| gcc_checking_assert (!DECL_RTL_SET_P (new_decl)); |
| |
| DECL_NAME (new_decl) = clone_function_name_numbered (thunk->decl, |
| "artificial_thunk"); |
| SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); |
| |
| /* We need to force DECL_IGNORED_P because the new thunk is created after |
| early debug was run. */ |
| DECL_IGNORED_P (new_decl) = 1; |
| |
| new_thunk = cgraph_node::create (new_decl); |
| set_new_clone_decl_and_node_flags (new_thunk); |
| new_thunk->definition = true; |
| new_thunk->can_change_signature = node->can_change_signature; |
| new_thunk->thunk = thunk->thunk; |
| new_thunk->unique_name = in_lto_p; |
| new_thunk->former_clone_of = thunk->decl; |
| if (info && info->param_adjustments) |
| clone_info::get_create (new_thunk)->param_adjustments |
| = info->param_adjustments; |
| new_thunk->unit_id = thunk->unit_id; |
| new_thunk->merged_comdat = thunk->merged_comdat; |
| new_thunk->merged_extern_inline = thunk->merged_extern_inline; |
| |
| cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count); |
| symtab->call_edge_duplication_hooks (thunk->callees, e); |
| symtab->call_cgraph_duplication_hooks (thunk, new_thunk); |
| return new_thunk; |
| } |
| |
| /* If E does not lead to a thunk, simply redirect it to N. Otherwise create |
| one or more equivalent thunks for N and redirect E to the first in the |
| chain. Note that it is then necessary to call |
| n->expand_all_artificial_thunks once all callers are redirected. */ |
| |
| void |
| cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n) |
| { |
| cgraph_node *orig_to = callee->ultimate_alias_target (); |
| if (orig_to->thunk) |
| n = duplicate_thunk_for_node (orig_to, n); |
| |
| redirect_callee (n); |
| } |
| |
| /* Call expand_thunk on all callers that are thunks and if analyze those nodes |
| that were expanded. */ |
| |
| void |
| cgraph_node::expand_all_artificial_thunks () |
| { |
| cgraph_edge *e; |
| for (e = callers; e;) |
| if (e->caller->thunk) |
| { |
| cgraph_node *thunk = e->caller; |
| |
| e = e->next_caller; |
| if (expand_thunk (thunk, false, false)) |
| { |
| thunk->thunk = false; |
| thunk->analyze (); |
| ipa_analyze_node (thunk); |
| inline_analyze_function (thunk); |
| } |
| thunk->expand_all_artificial_thunks (); |
| } |
| else |
| e = e->next_caller; |
| } |
| |
| void |
| dump_callgraph_transformation (const cgraph_node *original, |
| const cgraph_node *clone, |
| const char *suffix) |
| { |
| if (symtab->ipa_clones_dump_file) |
| { |
| fprintf (symtab->ipa_clones_dump_file, |
| "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n", |
| original->asm_name (), original->order, |
| DECL_SOURCE_FILE (original->decl), |
| DECL_SOURCE_LINE (original->decl), |
| DECL_SOURCE_COLUMN (original->decl), clone->asm_name (), |
| clone->order, DECL_SOURCE_FILE (clone->decl), |
| DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl), |
| suffix); |
| |
| symtab->cloned_nodes.add (original); |
| symtab->cloned_nodes.add (clone); |
| } |
| } |
| |
| /* Turn profile of N to local profile. */ |
| |
| static void |
| localize_profile (cgraph_node *n) |
| { |
| n->count = n->count.guessed_local (); |
| for (cgraph_edge *e = n->callees; e; e=e->next_callee) |
| { |
| e->count = e->count.guessed_local (); |
| if (!e->inline_failed) |
| localize_profile (e->callee); |
| } |
| for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee) |
| e->count = e->count.guessed_local (); |
| } |
| |
| /* Create node representing clone of N executed COUNT times. Decrease |
| the execution counts from original node too. |
| The new clone will have decl set to DECL that may or may not be the same |
| as decl of N. |
| |
| When UPDATE_ORIGINAL is true, the counts are subtracted from the original |
| function's profile to reflect the fact that part of execution is handled |
| by node. |
| When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about |
| the new clone. Otherwise the caller is responsible for doing so later. |
| |
| If the new node is being inlined into another one, NEW_INLINED_TO should be |
| the outline function the new one is (even indirectly) inlined to. All hooks |
| will see this in node's inlined_to, when invoked. Can be NULL if the |
| node is not inlined. |
| |
| If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information |
| will be overwritten by the new structure. Otherwise the new node will |
| share parameter manipulation information with the original node. */ |
| |
| cgraph_node * |
| cgraph_node::create_clone (tree new_decl, profile_count prof_count, |
| bool update_original, |
| vec<cgraph_edge *> redirect_callers, |
| bool call_duplication_hook, |
| cgraph_node *new_inlined_to, |
| ipa_param_adjustments *param_adjustments, |
| const char *suffix) |
| { |
| cgraph_node *new_node = symtab->create_empty (); |
| cgraph_edge *e; |
| unsigned i; |
| profile_count old_count = count; |
| bool nonzero = count.ipa ().nonzero_p (); |
| |
| if (new_inlined_to) |
| dump_callgraph_transformation (this, new_inlined_to, "inlining to"); |
| |
| /* When inlining we scale precisely to prof_count, when cloning we can |
| preserve local profile. */ |
| if (!new_inlined_to) |
| prof_count = count.combine_with_ipa_count (prof_count); |
| new_node->count = prof_count; |
| |
| /* Update IPA profile. Local profiles need no updating in original. */ |
| if (update_original) |
| { |
| if (inlined_to) |
| count = count.combine_with_ipa_count_within (count.ipa () |
| - prof_count.ipa (), |
| inlined_to->count); |
| else |
| count = count.combine_with_ipa_count (count.ipa () - prof_count.ipa ()); |
| } |
| new_node->decl = new_decl; |
| new_node->register_symbol (); |
| new_node->lto_file_data = lto_file_data; |
| new_node->analyzed = analyzed; |
| new_node->definition = definition; |
| new_node->versionable = versionable; |
| new_node->can_change_signature = can_change_signature; |
| new_node->redefined_extern_inline = redefined_extern_inline; |
| new_node->tm_may_enter_irr = tm_may_enter_irr; |
| new_node->externally_visible = false; |
| new_node->no_reorder = no_reorder; |
| new_node->local = true; |
| new_node->inlined_to = new_inlined_to; |
| new_node->rtl = rtl; |
| new_node->frequency = frequency; |
| new_node->tp_first_run = tp_first_run; |
| new_node->tm_clone = tm_clone; |
| new_node->icf_merged = icf_merged; |
| new_node->thunk = thunk; |
| new_node->unit_id = unit_id; |
| new_node->merged_comdat = merged_comdat; |
| new_node->merged_extern_inline = merged_extern_inline; |
| clone_info *info = clone_info::get (this); |
| |
| if (param_adjustments) |
| clone_info::get_create (new_node)->param_adjustments = param_adjustments; |
| else if (info && info->param_adjustments) |
| clone_info::get_create (new_node)->param_adjustments |
| = info->param_adjustments; |
| new_node->split_part = split_part; |
| |
| FOR_EACH_VEC_ELT (redirect_callers, i, e) |
| { |
| /* Redirect calls to the old version node to point to its new |
| version. The only exception is when the edge was proved to |
| be unreachable during the cloning procedure. */ |
| if (!e->callee |
| || !fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE)) |
| e->redirect_callee_duplicating_thunks (new_node); |
| } |
| new_node->expand_all_artificial_thunks (); |
| |
| for (e = callees;e; e=e->next_callee) |
| e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count, |
| update_original); |
| |
| for (e = indirect_calls; e; e = e->next_callee) |
| e->clone (new_node, e->call_stmt, e->lto_stmt_uid, |
| new_node->count, old_count, update_original); |
| new_node->clone_references (this); |
| |
| new_node->next_sibling_clone = clones; |
| if (clones) |
| clones->prev_sibling_clone = new_node; |
| clones = new_node; |
| new_node->clone_of = this; |
| |
| if (call_duplication_hook) |
| symtab->call_cgraph_duplication_hooks (this, new_node); |
| /* With partial train run we do not want to assume that original's |
| count is zero whenever we redurect all executed edges to clone. |
| Simply drop profile to local one in this case. */ |
| if (update_original |
| && opt_for_fn (decl, flag_profile_partial_training) |
| && nonzero |
| && count.ipa_p () |
| && !count.ipa ().nonzero_p () |
| && !inlined_to) |
| localize_profile (this); |
| |
| if (!new_inlined_to) |
| dump_callgraph_transformation (this, new_node, suffix); |
| |
| return new_node; |
| } |
| |
| static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids; |
| |
| /* Return a new assembler name for a clone of decl named NAME. Apart |
| from the string SUFFIX, the new name will end with a unique (for |
| each NAME) unspecified number. If clone numbering is not needed |
| then the two argument clone_function_name should be used instead. |
| Should not be called directly except for by |
| lto-partition.c:privatize_symbol_name_1. */ |
| |
| tree |
| clone_function_name_numbered (const char *name, const char *suffix) |
| { |
| /* Initialize the function->counter mapping the first time it's |
| needed. */ |
| if (!clone_fn_ids) |
| clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (64); |
| unsigned int &suffix_counter = clone_fn_ids->get_or_insert ( |
| IDENTIFIER_POINTER (get_identifier (name))); |
| return clone_function_name (name, suffix, suffix_counter++); |
| } |
| |
| /* Return a new assembler name for a clone of DECL. Apart from string |
| SUFFIX, the new name will end with a unique (for each DECL |
| assembler name) unspecified number. If clone numbering is not |
| needed then the two argument clone_function_name should be used |
| instead. */ |
| |
| tree |
| clone_function_name_numbered (tree decl, const char *suffix) |
| { |
| tree name = DECL_ASSEMBLER_NAME (decl); |
| return clone_function_name_numbered (IDENTIFIER_POINTER (name), |
| suffix); |
| } |
| |
| /* Return a new assembler name for a clone of decl named NAME. Apart |
| from the string SUFFIX, the new name will end with the specified |
| NUMBER. If clone numbering is not needed then the two argument |
| clone_function_name should be used instead. */ |
| |
| tree |
| clone_function_name (const char *name, const char *suffix, |
| unsigned long number) |
| { |
| size_t len = strlen (name); |
| char *tmp_name, *prefix; |
| |
| prefix = XALLOCAVEC (char, len + strlen (suffix) + 2); |
| memcpy (prefix, name, len); |
| strcpy (prefix + len + 1, suffix); |
| prefix[len] = symbol_table::symbol_suffix_separator (); |
| ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number); |
| return get_identifier (tmp_name); |
| } |
| |
| /* Return a new assembler name for a clone of DECL. Apart from the |
| string SUFFIX, the new name will end with the specified NUMBER. If |
| clone numbering is not needed then the two argument |
| clone_function_name should be used instead. */ |
| |
| tree |
| clone_function_name (tree decl, const char *suffix, |
| unsigned long number) |
| { |
| return clone_function_name ( |
| IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number); |
| } |
| |
| /* Return a new assembler name ending with the string SUFFIX for a |
| clone of DECL. */ |
| |
| tree |
| clone_function_name (tree decl, const char *suffix) |
| { |
| tree identifier = DECL_ASSEMBLER_NAME (decl); |
| /* For consistency this needs to behave the same way as |
| ASM_FORMAT_PRIVATE_NAME does, but without the final number |
| suffix. */ |
| char *separator = XALLOCAVEC (char, 2); |
| separator[0] = symbol_table::symbol_suffix_separator (); |
| separator[1] = 0; |
| #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL) |
| const char *prefix = "__"; |
| #else |
| const char *prefix = ""; |
| #endif |
| char *result = ACONCAT ((prefix, |
| IDENTIFIER_POINTER (identifier), |
| separator, |
| suffix, |
| (char*)0)); |
| return get_identifier (result); |
| } |
| |
| |
| /* Create callgraph node clone with new declaration. The actual body will be |
| copied later at compilation stage. The name of the new clone will be |
| constructed from the name of the original node, SUFFIX and NUM_SUFFIX. |
| |
| TODO: after merging in ipa-sra use function call notes instead of args_to_skip |
| bitmap interface. |
| */ |
| cgraph_node * |
| cgraph_node::create_virtual_clone (const vec<cgraph_edge *> &redirect_callers, |
| vec<ipa_replace_map *, va_gc> *tree_map, |
| ipa_param_adjustments *param_adjustments, |
| const char * suffix, unsigned num_suffix) |
| { |
| tree old_decl = decl; |
| cgraph_node *new_node = NULL; |
| tree new_decl; |
| size_t len, i; |
| ipa_replace_map *map; |
| char *name; |
| |
| gcc_checking_assert (versionable); |
| /* TODO: It would be nice if we could recognize that param_adjustments do not |
| actually perform any changes, but at the moment let's require it simply |
| does not exist. */ |
| gcc_assert (can_change_signature || !param_adjustments); |
| |
| /* Make a new FUNCTION_DECL tree node */ |
| if (!param_adjustments) |
| new_decl = copy_node (old_decl); |
| else |
| new_decl = param_adjustments->adjust_decl (old_decl); |
| |
| /* These pointers represent function body and will be populated only when clone |
| is materialized. */ |
| gcc_assert (new_decl != old_decl); |
| DECL_STRUCT_FUNCTION (new_decl) = NULL; |
| DECL_ARGUMENTS (new_decl) = NULL; |
| DECL_INITIAL (new_decl) = NULL; |
| DECL_RESULT (new_decl) = NULL; |
| /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning |
| sometimes storing only clone decl instead of original. */ |
| |
| /* Generate a new name for the new version. */ |
| len = IDENTIFIER_LENGTH (DECL_NAME (old_decl)); |
| name = XALLOCAVEC (char, len + strlen (suffix) + 2); |
| memcpy (name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), len); |
| strcpy (name + len + 1, suffix); |
| name[len] = '.'; |
| DECL_NAME (new_decl) = get_identifier (name); |
| SET_DECL_ASSEMBLER_NAME (new_decl, |
| clone_function_name (old_decl, suffix, num_suffix)); |
| SET_DECL_RTL (new_decl, NULL); |
| |
| new_node = create_clone (new_decl, count, false, |
| redirect_callers, false, NULL, param_adjustments, |
| suffix); |
| |
| /* Update the properties. |
| Make clone visible only within this translation unit. Make sure |
| that is not weak also. |
| ??? We cannot use COMDAT linkage because there is no |
| ABI support for this. */ |
| set_new_clone_decl_and_node_flags (new_node); |
| new_node->ipcp_clone = ipcp_clone; |
| if (tree_map) |
| clone_info::get_create (new_node)->tree_map = tree_map; |
| if (!implicit_section) |
| new_node->set_section (*this); |
| |
| /* Clones of global symbols or symbols with unique names are unique. */ |
| if ((TREE_PUBLIC (old_decl) |
| && !DECL_EXTERNAL (old_decl) |
| && !DECL_WEAK (old_decl) |
| && !DECL_COMDAT (old_decl)) |
| || in_lto_p) |
| new_node->unique_name = true; |
| FOR_EACH_VEC_SAFE_ELT (tree_map, i, map) |
| { |
| tree repl = map->new_tree; |
| if (map->force_load_ref) |
| { |
| gcc_assert (TREE_CODE (repl) == ADDR_EXPR); |
| repl = get_base_address (TREE_OPERAND (repl, 0)); |
| } |
| new_node->maybe_create_reference (repl, NULL); |
| } |
| |
| if (ipa_transforms_to_apply.exists ()) |
| new_node->ipa_transforms_to_apply |
| = ipa_transforms_to_apply.copy (); |
| |
| symtab->call_cgraph_duplication_hooks (this, new_node); |
| |
| return new_node; |
| } |
| |
| /* callgraph node being removed from symbol table; see if its entry can be |
| replaced by other inline clone. |
| INFO is clone info to attach to the new root. */ |
| cgraph_node * |
| cgraph_node::find_replacement (clone_info *info) |
| { |
| cgraph_node *next_inline_clone, *replacement; |
| |
| for (next_inline_clone = clones; |
| next_inline_clone |
| && next_inline_clone->decl != decl; |
| next_inline_clone = next_inline_clone->next_sibling_clone) |
| ; |
| |
| /* If there is inline clone of the node being removed, we need |
| to put it into the position of removed node and reorganize all |
| other clones to be based on it. */ |
| if (next_inline_clone) |
| { |
| cgraph_node *n; |
| cgraph_node *new_clones; |
| |
| replacement = next_inline_clone; |
| |
| /* Unlink inline clone from the list of clones of removed node. */ |
| if (next_inline_clone->next_sibling_clone) |
| next_inline_clone->next_sibling_clone->prev_sibling_clone |
| = next_inline_clone->prev_sibling_clone; |
| if (next_inline_clone->prev_sibling_clone) |
| { |
| gcc_assert (clones != next_inline_clone); |
| next_inline_clone->prev_sibling_clone->next_sibling_clone |
| = next_inline_clone->next_sibling_clone; |
| } |
| else |
| { |
| gcc_assert (clones == next_inline_clone); |
| clones = next_inline_clone->next_sibling_clone; |
| } |
| |
| new_clones = clones; |
| clones = NULL; |
| |
| /* Copy clone info. */ |
| if (info) |
| *clone_info::get_create (next_inline_clone) = *info; |
| |
| /* Now place it into clone tree at same level at NODE. */ |
| next_inline_clone->clone_of = clone_of; |
| next_inline_clone->prev_sibling_clone = NULL; |
| next_inline_clone->next_sibling_clone = NULL; |
| if (clone_of) |
| { |
| if (clone_of->clones) |
| clone_of->clones->prev_sibling_clone = next_inline_clone; |
| next_inline_clone->next_sibling_clone = clone_of->clones; |
| clone_of->clones = next_inline_clone; |
| } |
| |
| /* Merge the clone list. */ |
| if (new_clones) |
| { |
| if (!next_inline_clone->clones) |
| next_inline_clone->clones = new_clones; |
| else |
| { |
| n = next_inline_clone->clones; |
| while (n->next_sibling_clone) |
| n = n->next_sibling_clone; |
| n->next_sibling_clone = new_clones; |
| new_clones->prev_sibling_clone = n; |
| } |
| } |
| |
| /* Update clone_of pointers. */ |
| n = new_clones; |
| while (n) |
| { |
| n->clone_of = next_inline_clone; |
| n = n->next_sibling_clone; |
| } |
| |
| /* Update order in order to be able to find a LTO section |
| with function body. */ |
| replacement->order = order; |
| |
| return replacement; |
| } |
| else |
| return NULL; |
| } |
| |
| /* Like cgraph_set_call_stmt but walk the clone tree and update all |
| clones sharing the same function body. |
| When WHOLE_SPECULATIVE_EDGES is true, all three components of |
| speculative edge gets updated. Otherwise we update only direct |
| call. */ |
| |
| void |
| cgraph_node::set_call_stmt_including_clones (gimple *old_stmt, |
| gcall *new_stmt, |
| bool update_speculative) |
| { |
| cgraph_node *node; |
| cgraph_edge *master_edge = get_edge (old_stmt); |
| |
| if (master_edge) |
| cgraph_edge::set_call_stmt (master_edge, new_stmt, update_speculative); |
| |
| node = clones; |
| if (node) |
| while (node != this) |
| { |
| cgraph_edge *edge = node->get_edge (old_stmt); |
| if (edge) |
| { |
| edge = cgraph_edge::set_call_stmt (edge, new_stmt, |
| update_speculative); |
| /* If UPDATE_SPECULATIVE is false, it means that we are turning |
| speculative call into a real code sequence. Update the |
| callgraph edges. */ |
| if (edge->speculative && !update_speculative) |
| { |
| cgraph_edge *indirect = edge->speculative_call_indirect_edge (); |
| |
| for (cgraph_edge *next, *direct |
| = edge->first_speculative_call_target (); |
| direct; |
| direct = next) |
| { |
| next = direct->next_speculative_call_target (); |
| direct->speculative_call_target_ref ()->speculative = false; |
| direct->speculative = false; |
| } |
| indirect->speculative = false; |
| } |
| } |
| if (node->clones) |
| node = node->clones; |
| else if (node->next_sibling_clone) |
| node = node->next_sibling_clone; |
| else |
| { |
| while (node != this && !node->next_sibling_clone) |
| node = node->clone_of; |
| if (node != this) |
| node = node->next_sibling_clone; |
| } |
| } |
| } |
| |
| /* Like cgraph_create_edge walk the clone tree and update all clones sharing |
| same function body. If clones already have edge for OLD_STMT; only |
| update the edge same way as cgraph_set_call_stmt_including_clones does. |
| |
| TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative |
| frequencies of the clones. */ |
| |
| void |
| cgraph_node::create_edge_including_clones (cgraph_node *callee, |
| gimple *old_stmt, gcall *stmt, |
| profile_count count, |
| cgraph_inline_failed_t reason) |
| { |
| cgraph_node *node; |
| |
| if (!get_edge (stmt)) |
| { |
| cgraph_edge *edge = create_edge (callee, stmt, count); |
| edge->inline_failed = reason; |
| } |
| |
| node = clones; |
| if (node) |
| while (node != this) |
| /* Thunk clones do not get updated while copying inline function body. */ |
| if (!node->thunk) |
| { |
| cgraph_edge *edge = node->get_edge (old_stmt); |
| |
| /* It is possible that clones already contain the edge while |
| master didn't. Either we promoted indirect call into direct |
| call in the clone or we are processing clones of unreachable |
| master where edges has been removed. */ |
| if (edge) |
| edge = cgraph_edge::set_call_stmt (edge, stmt); |
| else if (! node->get_edge (stmt)) |
| { |
| edge = node->create_edge (callee, stmt, count); |
| edge->inline_failed = reason; |
| } |
| |
| if (node->clones) |
| node = node->clones; |
| else if (node->next_sibling_clone) |
| node = node->next_sibling_clone; |
| else |
| { |
| while (node != this && !node->next_sibling_clone) |
| node = node->clone_of; |
| if (node != this) |
| node = node->next_sibling_clone; |
| } |
| } |
| } |
| |
| /* Remove the node from cgraph and all inline clones inlined into it. |
| Skip however removal of FORBIDDEN_NODE and return true if it needs to be |
| removed. This allows to call the function from outer loop walking clone |
| tree. */ |
| |
| bool |
| cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node) |
| { |
| cgraph_edge *e, *next; |
| bool found = false; |
| |
| if (this == forbidden_node) |
| { |
| cgraph_edge::remove (callers); |
| return true; |
| } |
| for (e = callees; e; e = next) |
| { |
| next = e->next_callee; |
| if (!e->inline_failed) |
| found |= e->callee->remove_symbol_and_inline_clones (forbidden_node); |
| } |
| remove (); |
| return found; |
| } |
| |
| /* The edges representing the callers of the NEW_VERSION node were |
| fixed by cgraph_function_versioning (), now the call_expr in their |
| respective tree code should be updated to call the NEW_VERSION. */ |
| |
| static void |
| update_call_expr (cgraph_node *new_version) |
| { |
| cgraph_edge *e; |
| |
| gcc_assert (new_version); |
| |
| /* Update the call expr on the edges to call the new version. */ |
| for (e = new_version->callers; e; e = e->next_caller) |
| { |
| function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl); |
| gimple_call_set_fndecl (e->call_stmt, new_version->decl); |
| maybe_clean_eh_stmt_fn (inner_function, e->call_stmt); |
| } |
| } |
| |
| |
| /* Create a new cgraph node which is the new version of |
| callgraph node. REDIRECT_CALLERS holds the callers |
| edges which should be redirected to point to |
| NEW_VERSION. ALL the callees edges of the node |
| are cloned to the new version node. Return the new |
| version node. |
| |
| If non-NULL BLOCK_TO_COPY determine what basic blocks |
| was copied to prevent duplications of calls that are dead |
| in the clone. */ |
| |
| cgraph_node * |
| cgraph_node::create_version_clone (tree new_decl, |
| vec<cgraph_edge *> redirect_callers, |
| bitmap bbs_to_copy, |
| const char *suffix) |
| { |
| cgraph_node *new_version; |
| cgraph_edge *e; |
| unsigned i; |
| |
| new_version = cgraph_node::create (new_decl); |
| |
| new_version->analyzed = analyzed; |
| new_version->definition = definition; |
| new_version->local = local; |
| new_version->externally_visible = false; |
| new_version->no_reorder = no_reorder; |
| new_version->local = new_version->definition; |
| new_version->inlined_to = inlined_to; |
| new_version->rtl = rtl; |
| new_version->count = count; |
| new_version->unit_id = unit_id; |
| new_version->merged_comdat = merged_comdat; |
| new_version->merged_extern_inline = merged_extern_inline; |
| |
| for (e = callees; e; e=e->next_callee) |
| if (!bbs_to_copy |
| || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) |
| e->clone (new_version, e->call_stmt, |
| e->lto_stmt_uid, count, count, |
| true); |
| for (e = indirect_calls; e; e=e->next_callee) |
| if (!bbs_to_copy |
| || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) |
| e->clone (new_version, e->call_stmt, |
| e->lto_stmt_uid, count, count, |
| true); |
| FOR_EACH_VEC_ELT (redirect_callers, i, e) |
| { |
| /* Redirect calls to the old version node to point to its new |
| version. */ |
| e->redirect_callee (new_version); |
| } |
| |
| dump_callgraph_transformation (this, new_version, suffix); |
| |
| return new_version; |
| } |
| |
| /* Perform function versioning. |
| Function versioning includes copying of the tree and |
| a callgraph update (creating a new cgraph node and updating |
| its callees and callers). |
| |
| REDIRECT_CALLERS varray includes the edges to be redirected |
| to the new version. |
| |
| TREE_MAP is a mapping of tree nodes we want to replace with |
| new ones (according to results of prior analysis). |
| |
| If non-NULL ARGS_TO_SKIP determine function parameters to remove |
| from new version. |
| If SKIP_RETURN is true, the new version will return void. |
| If non-NULL BLOCK_TO_COPY determine what basic blocks to copy. |
| If non_NULL NEW_ENTRY determine new entry BB of the clone. |
| |
| If TARGET_ATTRIBUTES is non-null, when creating a new declaration, |
| add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p |
| that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET |
| of the declaration. |
| |
| If VERSION_DECL is set true, use clone_function_name_numbered for the |
| function clone. Otherwise, use clone_function_name. |
| |
| Return the new version's cgraph node. */ |
| |
| cgraph_node * |
| cgraph_node::create_version_clone_with_body |
| (vec<cgraph_edge *> redirect_callers, |
| vec<ipa_replace_map *, va_gc> *tree_map, |
| ipa_param_adjustments *param_adjustments, |
| bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix, |
| tree target_attributes, bool version_decl) |
| { |
| tree old_decl = decl; |
| cgraph_node *new_version_node = NULL; |
| tree new_decl; |
| |
| if (!tree_versionable_function_p (old_decl)) |
| return NULL; |
| |
| /* TODO: Restore an assert that we do not change signature if |
| can_change_signature is false. We cannot just check that |
| param_adjustments is NULL because unfortunately ipa-split removes return |
| values from such functions. */ |
| |
| /* Make a new FUNCTION_DECL tree node for the new version. */ |
| if (param_adjustments) |
| new_decl = param_adjustments->adjust_decl (old_decl); |
| else |
| new_decl = copy_node (old_decl); |
| |
| /* Generate a new name for the new version. */ |
| tree fnname = (version_decl ? clone_function_name_numbered (old_decl, suffix) |
| : clone_function_name (old_decl, suffix)); |
| DECL_NAME (new_decl) = fnname; |
| SET_DECL_ASSEMBLER_NAME (new_decl, fnname); |
| SET_DECL_RTL (new_decl, NULL); |
| |
| DECL_VIRTUAL_P (new_decl) = 0; |
| |
| if (target_attributes) |
| { |
| DECL_ATTRIBUTES (new_decl) = target_attributes; |
| |
| location_t saved_loc = input_location; |
| tree v = TREE_VALUE (target_attributes); |
| input_location = DECL_SOURCE_LOCATION (new_decl); |
| bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1); |
| input_location = saved_loc; |
| if (!r) |
| return NULL; |
| } |
| |
| /* When the old decl was a con-/destructor make sure the clone isn't. */ |
| DECL_STATIC_CONSTRUCTOR (new_decl) = 0; |
| DECL_STATIC_DESTRUCTOR (new_decl) = 0; |
| DECL_SET_IS_OPERATOR_NEW (new_decl, 0); |
| DECL_SET_IS_OPERATOR_DELETE (new_decl, 0); |
| DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0; |
| |
| /* Create the new version's call-graph node. |
| and update the edges of the new node. */ |
| new_version_node = create_version_clone (new_decl, redirect_callers, |
| bbs_to_copy, suffix); |
| |
| if (ipa_transforms_to_apply.exists ()) |
| new_version_node->ipa_transforms_to_apply |
| = ipa_transforms_to_apply.copy (); |
| /* Copy the OLD_VERSION_NODE function tree to the new version. */ |
| tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments, |
| false, bbs_to_copy, new_entry_block); |
| |
| /* Update the new version's properties. |
| Make The new version visible only within this translation unit. Make sure |
| that is not weak also. |
| ??? We cannot use COMDAT linkage because there is no |
| ABI support for this. */ |
| new_version_node->make_decl_local (); |
| DECL_VIRTUAL_P (new_version_node->decl) = 0; |
| new_version_node->externally_visible = 0; |
| new_version_node->local = 1; |
| new_version_node->lowered = true; |
| if (!implicit_section) |
| new_version_node->set_section (*this); |
| /* Clones of global symbols or symbols with unique names are unique. */ |
| if ((TREE_PUBLIC (old_decl) |
| && !DECL_EXTERNAL (old_decl) |
| && !DECL_WEAK (old_decl) |
| && !DECL_COMDAT (old_decl)) |
| || in_lto_p) |
| new_version_node->unique_name = true; |
| |
| /* Update the call_expr on the edges to call the new version node. */ |
| update_call_expr (new_version_node); |
| |
| symtab->call_cgraph_insertion_hooks (new_version_node); |
| return new_version_node; |
| } |
| |
| /* Remove the node from the tree of virtual and inline clones and make it a |
| standalone node - not a clone any more. */ |
| |
| void cgraph_node::remove_from_clone_tree () |
| { |
| if (next_sibling_clone) |
| next_sibling_clone->prev_sibling_clone = prev_sibling_clone; |
| if (prev_sibling_clone) |
| prev_sibling_clone->next_sibling_clone = next_sibling_clone; |
| else |
| clone_of->clones = next_sibling_clone; |
| next_sibling_clone = NULL; |
| prev_sibling_clone = NULL; |
| clone_of = NULL; |
| } |
| |
| /* Given virtual clone, turn it into actual clone. */ |
| |
| void |
| cgraph_node::materialize_clone () |
| { |
| clone_info *info = clone_info::get (this); |
| clone_of->get_untransformed_body (); |
| former_clone_of = clone_of->decl; |
| if (clone_of->former_clone_of) |
| former_clone_of = clone_of->former_clone_of; |
| if (symtab->dump_file) |
| { |
| fprintf (symtab->dump_file, "cloning %s to %s\n", |
| clone_of->dump_name (), |
| dump_name ()); |
| if (info && info->tree_map) |
| { |
| fprintf (symtab->dump_file, " replace map:"); |
| for (unsigned int i = 0; |
| i < vec_safe_length (info->tree_map); |
| i++) |
| { |
| ipa_replace_map *replace_info; |
| replace_info = (*info->tree_map)[i]; |
| fprintf (symtab->dump_file, "%s %i -> ", |
| i ? "," : "", replace_info->parm_num); |
| print_generic_expr (symtab->dump_file, |
| replace_info->new_tree); |
| } |
| fprintf (symtab->dump_file, "\n"); |
| } |
| if (info && info->param_adjustments) |
| info->param_adjustments->dump (symtab->dump_file); |
| } |
| clear_stmts_in_references (); |
| /* Copy the OLD_VERSION_NODE function tree to the new version. */ |
| tree_function_versioning (clone_of->decl, decl, |
| info ? info->tree_map : NULL, |
| info ? info->param_adjustments : NULL, |
| true, NULL, NULL); |
| if (symtab->dump_file) |
| { |
| dump_function_to_file (clone_of->decl, symtab->dump_file, |
| dump_flags); |
| dump_function_to_file (decl, symtab->dump_file, dump_flags); |
| } |
| |
| cgraph_node *this_clone_of = clone_of; |
| /* Function is no longer clone. */ |
| remove_from_clone_tree (); |
| if (!this_clone_of->analyzed && !this_clone_of->clones) |
| this_clone_of->release_body (); |
| } |
| |
| #include "gt-cgraphclones.h" |