| /* OpenACC worker partitioning via middle end neutering/broadcasting scheme |
| |
| Copyright (C) 2015-2021 Free Software Foundation, Inc. |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it |
| under the terms of the GNU General Public License as published |
| by the Free Software Foundation; either version 3, or (at your |
| option) any later version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT |
| ANY WARRANTY; without even the implied warranty of MERCHANTABILITY |
| or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public |
| License for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "rtl.h" |
| #include "tree.h" |
| #include "gimple.h" |
| #include "tree-pass.h" |
| #include "ssa.h" |
| #include "cgraph.h" |
| #include "pretty-print.h" |
| #include "fold-const.h" |
| #include "gimplify.h" |
| #include "gimple-iterator.h" |
| #include "gimple-walk.h" |
| #include "tree-inline.h" |
| #include "langhooks.h" |
| #include "omp-general.h" |
| #include "omp-low.h" |
| #include "gimple-pretty-print.h" |
| #include "cfghooks.h" |
| #include "insn-config.h" |
| #include "recog.h" |
| #include "internal-fn.h" |
| #include "bitmap.h" |
| #include "tree-nested.h" |
| #include "stor-layout.h" |
| #include "tree-ssa-threadupdate.h" |
| #include "tree-into-ssa.h" |
| #include "splay-tree.h" |
| #include "target.h" |
| #include "cfgloop.h" |
| #include "tree-cfg.h" |
| #include "omp-offload.h" |
| #include "attribs.h" |
| #include "targhooks.h" |
| #include "diagnostic-core.h" |
| |
| /* Loop structure of the function. The entire function is described as |
| a NULL loop. */ |
| /* Adapted from 'gcc/config/nvptx/nvptx.c:struct parallel'. */ |
| |
| struct parallel_g |
| { |
| /* Parent parallel. */ |
| parallel_g *parent; |
| |
| /* Next sibling parallel. */ |
| parallel_g *next; |
| |
| /* First child parallel. */ |
| parallel_g *inner; |
| |
| /* Partitioning mask of the parallel. */ |
| unsigned mask; |
| |
| /* Partitioning used within inner parallels. */ |
| unsigned inner_mask; |
| |
| /* Location of parallel forked and join. The forked is the first |
| block in the parallel and the join is the first block after of |
| the partition. */ |
| basic_block forked_block; |
| basic_block join_block; |
| |
| gimple *forked_stmt; |
| gimple *join_stmt; |
| |
| gimple *fork_stmt; |
| gimple *joining_stmt; |
| |
| /* Basic blocks in this parallel, but not in child parallels. The |
| FORKED and JOINING blocks are in the partition. The FORK and JOIN |
| blocks are not. */ |
| auto_vec<basic_block> blocks; |
| |
| tree record_type; |
| tree sender_decl; |
| tree receiver_decl; |
| |
| public: |
| parallel_g (parallel_g *parent, unsigned mode); |
| ~parallel_g (); |
| }; |
| |
| /* Constructor links the new parallel into it's parent's chain of |
| children. */ |
| |
| parallel_g::parallel_g (parallel_g *parent_, unsigned mask_) |
| :parent (parent_), next (0), inner (0), mask (mask_), inner_mask (0) |
| { |
| forked_block = join_block = 0; |
| forked_stmt = join_stmt = NULL; |
| fork_stmt = joining_stmt = NULL; |
| |
| record_type = NULL_TREE; |
| sender_decl = NULL_TREE; |
| receiver_decl = NULL_TREE; |
| |
| if (parent) |
| { |
| next = parent->inner; |
| parent->inner = this; |
| } |
| } |
| |
| parallel_g::~parallel_g () |
| { |
| delete inner; |
| delete next; |
| } |
| |
| static bool |
| local_var_based_p (tree decl) |
| { |
| switch (TREE_CODE (decl)) |
| { |
| case VAR_DECL: |
| return !is_global_var (decl); |
| |
| case COMPONENT_REF: |
| case BIT_FIELD_REF: |
| case ARRAY_REF: |
| return local_var_based_p (TREE_OPERAND (decl, 0)); |
| |
| default: |
| return false; |
| } |
| } |
| |
| /* Map of basic blocks to gimple stmts. */ |
| typedef hash_map<basic_block, gimple *> bb_stmt_map_t; |
| |
| /* Calls to OpenACC routines are made by all workers/wavefronts/warps, since |
| the routine likely contains partitioned loops (else will do its own |
| neutering and variable propagation). Return TRUE if a function call CALL |
| should be made in (worker) single mode instead, rather than redundant |
| mode. */ |
| |
| static bool |
| omp_sese_active_worker_call (gcall *call) |
| { |
| #define GOMP_DIM_SEQ GOMP_DIM_MAX |
| tree fndecl = gimple_call_fndecl (call); |
| |
| if (!fndecl) |
| return true; |
| |
| tree attrs = oacc_get_fn_attrib (fndecl); |
| |
| if (!attrs) |
| return true; |
| |
| int level = oacc_fn_attrib_level (attrs); |
| |
| /* Neither regular functions nor "seq" routines should be run by all threads |
| in worker-single mode. */ |
| return level == -1 || level == GOMP_DIM_SEQ; |
| #undef GOMP_DIM_SEQ |
| } |
| |
| /* Split basic blocks such that each forked and join unspecs are at |
| the start of their basic blocks. Thus afterwards each block will |
| have a single partitioning mode. We also do the same for return |
| insns, as they are executed by every thread. Return the |
| partitioning mode of the function as a whole. Populate MAP with |
| head and tail blocks. We also clear the BB visited flag, which is |
| used when finding partitions. */ |
| /* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_split_blocks'. */ |
| |
| static void |
| omp_sese_split_blocks (bb_stmt_map_t *map) |
| { |
| auto_vec<gimple *> worklist; |
| basic_block block; |
| |
| /* Locate all the reorg instructions of interest. */ |
| FOR_ALL_BB_FN (block, cfun) |
| { |
| /* Clear visited flag, for use by parallel locator */ |
| block->flags &= ~BB_VISITED; |
| |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| |
| if (gimple_call_internal_p (stmt, IFN_UNIQUE)) |
| { |
| enum ifn_unique_kind k = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (stmt, 0))); |
| |
| if (k == IFN_UNIQUE_OACC_JOIN) |
| worklist.safe_push (stmt); |
| else if (k == IFN_UNIQUE_OACC_FORK) |
| { |
| gcc_assert (gsi_one_before_end_p (gsi)); |
| basic_block forked_block = single_succ (block); |
| gimple_stmt_iterator gsi2 = gsi_start_bb (forked_block); |
| |
| /* We push a NOP as a placeholder for the "forked" stmt. |
| This is then recognized in omp_sese_find_par. */ |
| gimple *nop = gimple_build_nop (); |
| gsi_insert_before (&gsi2, nop, GSI_SAME_STMT); |
| |
| worklist.safe_push (nop); |
| } |
| } |
| else if (gimple_code (stmt) == GIMPLE_RETURN |
| || gimple_code (stmt) == GIMPLE_COND |
| || gimple_code (stmt) == GIMPLE_SWITCH |
| || (gimple_code (stmt) == GIMPLE_CALL |
| && !gimple_call_internal_p (stmt) |
| && !omp_sese_active_worker_call (as_a <gcall *> (stmt)))) |
| worklist.safe_push (stmt); |
| else if (is_gimple_assign (stmt)) |
| { |
| tree lhs = gimple_assign_lhs (stmt); |
| |
| /* Force assignments to components/fields/elements of local |
| aggregates into fully-partitioned (redundant) mode. This |
| avoids having to broadcast the whole aggregate. The RHS of |
| the assignment will be propagated using the normal |
| mechanism. */ |
| |
| switch (TREE_CODE (lhs)) |
| { |
| case COMPONENT_REF: |
| case BIT_FIELD_REF: |
| case ARRAY_REF: |
| { |
| tree aggr = TREE_OPERAND (lhs, 0); |
| |
| if (local_var_based_p (aggr)) |
| worklist.safe_push (stmt); |
| } |
| break; |
| |
| default: |
| ; |
| } |
| } |
| } |
| } |
| |
| /* Split blocks on the worklist. */ |
| unsigned ix; |
| gimple *stmt; |
| |
| for (ix = 0; worklist.iterate (ix, &stmt); ix++) |
| { |
| basic_block block = gimple_bb (stmt); |
| |
| if (gimple_code (stmt) == GIMPLE_COND) |
| { |
| gcond *orig_cond = as_a <gcond *> (stmt); |
| tree_code code = gimple_expr_code (orig_cond); |
| tree pred = make_ssa_name (boolean_type_node); |
| gimple *asgn = gimple_build_assign (pred, code, |
| gimple_cond_lhs (orig_cond), |
| gimple_cond_rhs (orig_cond)); |
| gcond *new_cond |
| = gimple_build_cond (NE_EXPR, pred, boolean_false_node, |
| gimple_cond_true_label (orig_cond), |
| gimple_cond_false_label (orig_cond)); |
| |
| gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
| gsi_insert_before (&gsi, asgn, GSI_SAME_STMT); |
| gsi_replace (&gsi, new_cond, true); |
| |
| edge e = split_block (block, asgn); |
| block = e->dest; |
| map->get_or_insert (block) = new_cond; |
| } |
| else if ((gimple_code (stmt) == GIMPLE_CALL |
| && !gimple_call_internal_p (stmt)) |
| || is_gimple_assign (stmt)) |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
| gsi_prev (&gsi); |
| |
| edge call = split_block (block, gsi_stmt (gsi)); |
| |
| gimple *call_stmt = gsi_stmt (gsi_start_bb (call->dest)); |
| |
| edge call_to_ret = split_block (call->dest, call_stmt); |
| |
| map->get_or_insert (call_to_ret->src) = call_stmt; |
| } |
| else |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
| gsi_prev (&gsi); |
| |
| if (gsi_end_p (gsi)) |
| map->get_or_insert (block) = stmt; |
| else |
| { |
| /* Split block before insn. The insn is in the new block. */ |
| edge e = split_block (block, gsi_stmt (gsi)); |
| |
| block = e->dest; |
| map->get_or_insert (block) = stmt; |
| } |
| } |
| } |
| } |
| |
| static const char * |
| mask_name (unsigned mask) |
| { |
| switch (mask) |
| { |
| case 0: return "gang redundant"; |
| case 1: return "gang partitioned"; |
| case 2: return "worker partitioned"; |
| case 3: return "gang+worker partitioned"; |
| case 4: return "vector partitioned"; |
| case 5: return "gang+vector partitioned"; |
| case 6: return "worker+vector partitioned"; |
| case 7: return "fully partitioned"; |
| default: return "<illegal>"; |
| } |
| } |
| |
| /* Dump this parallel and all its inner parallels. */ |
| /* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_dump_pars'. */ |
| |
| static void |
| omp_sese_dump_pars (parallel_g *par, unsigned depth) |
| { |
| fprintf (dump_file, "%u: mask %d (%s) head=%d, tail=%d\n", |
| depth, par->mask, mask_name (par->mask), |
| par->forked_block ? par->forked_block->index : -1, |
| par->join_block ? par->join_block->index : -1); |
| |
| fprintf (dump_file, " blocks:"); |
| |
| basic_block block; |
| for (unsigned ix = 0; par->blocks.iterate (ix, &block); ix++) |
| fprintf (dump_file, " %d", block->index); |
| fprintf (dump_file, "\n"); |
| if (par->inner) |
| omp_sese_dump_pars (par->inner, depth + 1); |
| |
| if (par->next) |
| omp_sese_dump_pars (par->next, depth); |
| } |
| |
| /* If BLOCK contains a fork/join marker, process it to create or |
| terminate a loop structure. Add this block to the current loop, |
| and then walk successor blocks. */ |
| /* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_find_par'. */ |
| |
| static parallel_g * |
| omp_sese_find_par (bb_stmt_map_t *map, parallel_g *par, basic_block block) |
| { |
| if (block->flags & BB_VISITED) |
| return par; |
| block->flags |= BB_VISITED; |
| |
| if (gimple **stmtp = map->get (block)) |
| { |
| gimple *stmt = *stmtp; |
| |
| if (gimple_code (stmt) == GIMPLE_COND |
| || gimple_code (stmt) == GIMPLE_SWITCH |
| || gimple_code (stmt) == GIMPLE_RETURN |
| || (gimple_code (stmt) == GIMPLE_CALL |
| && !gimple_call_internal_p (stmt)) |
| || is_gimple_assign (stmt)) |
| { |
| /* A single block that is forced to be at the maximum partition |
| level. Make a singleton par for it. */ |
| par = new parallel_g (par, GOMP_DIM_MASK (GOMP_DIM_GANG) |
| | GOMP_DIM_MASK (GOMP_DIM_WORKER) |
| | GOMP_DIM_MASK (GOMP_DIM_VECTOR)); |
| par->forked_block = block; |
| par->forked_stmt = stmt; |
| par->blocks.safe_push (block); |
| par = par->parent; |
| goto walk_successors; |
| } |
| else if (gimple_nop_p (stmt)) |
| { |
| basic_block pred = single_pred (block); |
| gcc_assert (pred); |
| gimple_stmt_iterator gsi = gsi_last_bb (pred); |
| gimple *final_stmt = gsi_stmt (gsi); |
| |
| if (gimple_call_internal_p (final_stmt, IFN_UNIQUE)) |
| { |
| gcall *call = as_a <gcall *> (final_stmt); |
| enum ifn_unique_kind k = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (call, 0))); |
| |
| if (k == IFN_UNIQUE_OACC_FORK) |
| { |
| HOST_WIDE_INT dim |
| = TREE_INT_CST_LOW (gimple_call_arg (call, 2)); |
| unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0; |
| |
| par = new parallel_g (par, mask); |
| par->forked_block = block; |
| par->forked_stmt = final_stmt; |
| par->fork_stmt = stmt; |
| } |
| else |
| gcc_unreachable (); |
| } |
| else |
| gcc_unreachable (); |
| } |
| else if (gimple_call_internal_p (stmt, IFN_UNIQUE)) |
| { |
| gcall *call = as_a <gcall *> (stmt); |
| enum ifn_unique_kind k = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (call, 0))); |
| if (k == IFN_UNIQUE_OACC_JOIN) |
| { |
| HOST_WIDE_INT dim = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2)); |
| unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0; |
| |
| gcc_assert (par->mask == mask); |
| par->join_block = block; |
| par->join_stmt = stmt; |
| par = par->parent; |
| } |
| else |
| gcc_unreachable (); |
| } |
| else |
| gcc_unreachable (); |
| } |
| |
| if (par) |
| /* Add this block onto the current loop's list of blocks. */ |
| par->blocks.safe_push (block); |
| else |
| /* This must be the entry block. Create a NULL parallel. */ |
| par = new parallel_g (0, 0); |
| |
| walk_successors: |
| /* Walk successor blocks. */ |
| edge e; |
| edge_iterator ei; |
| |
| FOR_EACH_EDGE (e, ei, block->succs) |
| omp_sese_find_par (map, par, e->dest); |
| |
| return par; |
| } |
| |
| /* DFS walk the CFG looking for fork & join markers. Construct |
| loop structures as we go. MAP is a mapping of basic blocks |
| to head & tail markers, discovered when splitting blocks. This |
| speeds up the discovery. We rely on the BB visited flag having |
| been cleared when splitting blocks. */ |
| /* Adapted from 'gcc/config/nvptx/nvptx.c:nvptx_discover_pars'. */ |
| |
| static parallel_g * |
| omp_sese_discover_pars (bb_stmt_map_t *map) |
| { |
| basic_block block; |
| |
| /* Mark exit blocks as visited. */ |
| block = EXIT_BLOCK_PTR_FOR_FN (cfun); |
| block->flags |= BB_VISITED; |
| |
| /* And entry block as not. */ |
| block = ENTRY_BLOCK_PTR_FOR_FN (cfun); |
| block->flags &= ~BB_VISITED; |
| |
| parallel_g *par = omp_sese_find_par (map, 0, block); |
| |
| if (dump_file) |
| { |
| fprintf (dump_file, "\nLoops\n"); |
| omp_sese_dump_pars (par, 0); |
| fprintf (dump_file, "\n"); |
| } |
| |
| return par; |
| } |
| |
| static void |
| populate_single_mode_bitmaps (parallel_g *par, bitmap worker_single, |
| bitmap vector_single, unsigned outer_mask, |
| int depth) |
| { |
| unsigned mask = outer_mask | par->mask; |
| |
| basic_block block; |
| |
| for (unsigned i = 0; par->blocks.iterate (i, &block); i++) |
| { |
| if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0) |
| bitmap_set_bit (worker_single, block->index); |
| |
| if ((mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR)) == 0) |
| bitmap_set_bit (vector_single, block->index); |
| } |
| |
| if (par->inner) |
| populate_single_mode_bitmaps (par->inner, worker_single, vector_single, |
| mask, depth + 1); |
| if (par->next) |
| populate_single_mode_bitmaps (par->next, worker_single, vector_single, |
| outer_mask, depth); |
| } |
| |
| /* A map from SSA names or var decls to record fields. */ |
| |
| typedef hash_map<tree, tree> field_map_t; |
| |
| /* For each propagation record type, this is a map from SSA names or var decls |
| to propagate, to the field in the record type that should be used for |
| transmission and reception. */ |
| |
| typedef hash_map<tree, field_map_t *> record_field_map_t; |
| |
| static void |
| install_var_field (tree var, tree record_type, field_map_t *fields) |
| { |
| tree name; |
| char tmp[20]; |
| |
| if (TREE_CODE (var) == SSA_NAME) |
| { |
| name = SSA_NAME_IDENTIFIER (var); |
| if (!name) |
| { |
| sprintf (tmp, "_%u", (unsigned) SSA_NAME_VERSION (var)); |
| name = get_identifier (tmp); |
| } |
| } |
| else if (TREE_CODE (var) == VAR_DECL) |
| { |
| name = DECL_NAME (var); |
| if (!name) |
| { |
| sprintf (tmp, "D_%u", (unsigned) DECL_UID (var)); |
| name = get_identifier (tmp); |
| } |
| } |
| else |
| gcc_unreachable (); |
| |
| gcc_assert (!fields->get (var)); |
| |
| tree type = TREE_TYPE (var); |
| |
| if (POINTER_TYPE_P (type) |
| && TYPE_RESTRICT (type)) |
| type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT); |
| |
| tree field = build_decl (BUILTINS_LOCATION, FIELD_DECL, name, type); |
| |
| if (TREE_CODE (var) == VAR_DECL && type == TREE_TYPE (var)) |
| { |
| SET_DECL_ALIGN (field, DECL_ALIGN (var)); |
| DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var); |
| TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var); |
| } |
| else |
| SET_DECL_ALIGN (field, TYPE_ALIGN (type)); |
| |
| fields->put (var, field); |
| |
| insert_field_into_struct (record_type, field); |
| } |
| |
| /* Sets of SSA_NAMES or VAR_DECLs to propagate. */ |
| typedef hash_set<tree> propagation_set; |
| |
| static void |
| find_ssa_names_to_propagate (parallel_g *par, unsigned outer_mask, |
| bitmap worker_single, bitmap vector_single, |
| vec<propagation_set *> *prop_set) |
| { |
| unsigned mask = outer_mask | par->mask; |
| |
| if (par->inner) |
| find_ssa_names_to_propagate (par->inner, mask, worker_single, |
| vector_single, prop_set); |
| if (par->next) |
| find_ssa_names_to_propagate (par->next, outer_mask, worker_single, |
| vector_single, prop_set); |
| |
| if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) |
| { |
| basic_block block; |
| int ix; |
| |
| for (ix = 0; par->blocks.iterate (ix, &block); ix++) |
| { |
| for (gphi_iterator psi = gsi_start_phis (block); |
| !gsi_end_p (psi); gsi_next (&psi)) |
| { |
| gphi *phi = psi.phi (); |
| use_operand_p use; |
| ssa_op_iter iter; |
| |
| FOR_EACH_PHI_ARG (use, phi, iter, SSA_OP_USE) |
| { |
| tree var = USE_FROM_PTR (use); |
| |
| if (TREE_CODE (var) != SSA_NAME) |
| continue; |
| |
| gimple *def_stmt = SSA_NAME_DEF_STMT (var); |
| |
| if (gimple_nop_p (def_stmt)) |
| continue; |
| |
| basic_block def_bb = gimple_bb (def_stmt); |
| |
| if (bitmap_bit_p (worker_single, def_bb->index)) |
| { |
| if (!(*prop_set)[def_bb->index]) |
| (*prop_set)[def_bb->index] = new propagation_set; |
| |
| propagation_set *ws_prop = (*prop_set)[def_bb->index]; |
| |
| ws_prop->add (var); |
| } |
| } |
| } |
| |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); gsi_next (&gsi)) |
| { |
| use_operand_p use; |
| ssa_op_iter iter; |
| gimple *stmt = gsi_stmt (gsi); |
| |
| FOR_EACH_SSA_USE_OPERAND (use, stmt, iter, SSA_OP_USE) |
| { |
| tree var = USE_FROM_PTR (use); |
| |
| gimple *def_stmt = SSA_NAME_DEF_STMT (var); |
| |
| if (gimple_nop_p (def_stmt)) |
| continue; |
| |
| basic_block def_bb = gimple_bb (def_stmt); |
| |
| if (bitmap_bit_p (worker_single, def_bb->index)) |
| { |
| if (!(*prop_set)[def_bb->index]) |
| (*prop_set)[def_bb->index] = new propagation_set; |
| |
| propagation_set *ws_prop = (*prop_set)[def_bb->index]; |
| |
| ws_prop->add (var); |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| /* Callback for walk_gimple_stmt to find RHS VAR_DECLs (uses) in a |
| statement. */ |
| |
| static tree |
| find_partitioned_var_uses_1 (tree *node, int *, void *data) |
| { |
| walk_stmt_info *wi = (walk_stmt_info *) data; |
| hash_set<tree> *partitioned_var_uses = (hash_set<tree> *) wi->info; |
| |
| if (!wi->is_lhs && VAR_P (*node)) |
| partitioned_var_uses->add (*node); |
| |
| return NULL_TREE; |
| } |
| |
| static void |
| find_partitioned_var_uses (parallel_g *par, unsigned outer_mask, |
| hash_set<tree> *partitioned_var_uses) |
| { |
| unsigned mask = outer_mask | par->mask; |
| |
| if (par->inner) |
| find_partitioned_var_uses (par->inner, mask, partitioned_var_uses); |
| if (par->next) |
| find_partitioned_var_uses (par->next, outer_mask, partitioned_var_uses); |
| |
| if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) |
| { |
| basic_block block; |
| int ix; |
| |
| for (ix = 0; par->blocks.iterate (ix, &block); ix++) |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); gsi_next (&gsi)) |
| { |
| walk_stmt_info wi; |
| memset (&wi, 0, sizeof (wi)); |
| wi.info = (void *) partitioned_var_uses; |
| walk_gimple_stmt (&gsi, NULL, find_partitioned_var_uses_1, &wi); |
| } |
| } |
| } |
| |
| /* Gang-private variables (typically placed in a GPU's shared memory) do not |
| need to be processed by the worker-propagation mechanism. Populate the |
| GANG_PRIVATE_VARS set with any such variables found in the current |
| function. */ |
| |
| static void |
| find_gang_private_vars (hash_set<tree> *gang_private_vars) |
| { |
| basic_block block; |
| |
| FOR_EACH_BB_FN (block, cfun) |
| { |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| |
| if (gimple_call_internal_p (stmt, IFN_UNIQUE)) |
| { |
| enum ifn_unique_kind k = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (stmt, 0))); |
| if (k == IFN_UNIQUE_OACC_PRIVATE) |
| { |
| HOST_WIDE_INT level |
| = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2)); |
| if (level != GOMP_DIM_GANG) |
| continue; |
| for (unsigned i = 3; i < gimple_call_num_args (stmt); i++) |
| { |
| tree arg = gimple_call_arg (stmt, i); |
| gcc_assert (TREE_CODE (arg) == ADDR_EXPR); |
| tree decl = TREE_OPERAND (arg, 0); |
| gang_private_vars->add (decl); |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| static void |
| find_local_vars_to_propagate (parallel_g *par, unsigned outer_mask, |
| hash_set<tree> *partitioned_var_uses, |
| hash_set<tree> *gang_private_vars, |
| bitmap writes_gang_private, |
| vec<propagation_set *> *prop_set) |
| { |
| unsigned mask = outer_mask | par->mask; |
| |
| if (par->inner) |
| find_local_vars_to_propagate (par->inner, mask, partitioned_var_uses, |
| gang_private_vars, writes_gang_private, |
| prop_set); |
| if (par->next) |
| find_local_vars_to_propagate (par->next, outer_mask, partitioned_var_uses, |
| gang_private_vars, writes_gang_private, |
| prop_set); |
| |
| if (!(mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))) |
| { |
| basic_block block; |
| int ix; |
| |
| for (ix = 0; par->blocks.iterate (ix, &block); ix++) |
| { |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| tree var; |
| unsigned i; |
| |
| FOR_EACH_LOCAL_DECL (cfun, i, var) |
| { |
| if (!VAR_P (var) |
| || is_global_var (var) |
| || AGGREGATE_TYPE_P (TREE_TYPE (var)) |
| || !partitioned_var_uses->contains (var)) |
| continue; |
| |
| if (stmt_may_clobber_ref_p (stmt, var)) |
| { |
| if (dump_file) |
| { |
| fprintf (dump_file, "bb %u: local variable may be " |
| "clobbered in %s mode: ", block->index, |
| mask_name (mask)); |
| print_generic_expr (dump_file, var, TDF_SLIM); |
| fprintf (dump_file, "\n"); |
| } |
| |
| if (gang_private_vars->contains (var)) |
| { |
| /* If we write a gang-private variable, we want a |
| barrier at the end of the block. */ |
| bitmap_set_bit (writes_gang_private, block->index); |
| continue; |
| } |
| |
| if (!(*prop_set)[block->index]) |
| (*prop_set)[block->index] = new propagation_set; |
| |
| propagation_set *ws_prop |
| = (*prop_set)[block->index]; |
| |
| ws_prop->add (var); |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| /* Transform basic blocks FROM, TO (which may be the same block) into: |
| if (GOACC_single_start ()) |
| BLOCK; |
| GOACC_barrier (); |
| \ | / |
| +----+ |
| | | (new) predicate block |
| +----+-- |
| \ | / \ | / |t \ |
| +----+ +----+ +----+ | |
| | | | | ===> | | | f (old) from block |
| +----+ +----+ +----+ | |
| | t/ \f | / |
| +----+/ |
| (split (split before | | skip block |
| at end) condition) +----+ |
| t/ \f |
| */ |
| |
| static void |
| worker_single_simple (basic_block from, basic_block to, |
| hash_set<tree> *def_escapes_block) |
| { |
| gimple *call, *cond; |
| tree lhs, decl; |
| basic_block skip_block; |
| |
| gimple_stmt_iterator gsi = gsi_last_bb (to); |
| if (EDGE_COUNT (to->succs) > 1) |
| { |
| gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_COND); |
| gsi_prev (&gsi); |
| } |
| edge e = split_block (to, gsi_stmt (gsi)); |
| skip_block = e->dest; |
| |
| gimple_stmt_iterator start = gsi_after_labels (from); |
| |
| decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_START); |
| lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); |
| call = gimple_build_call (decl, 0); |
| gimple_call_set_lhs (call, lhs); |
| gsi_insert_before (&start, call, GSI_NEW_STMT); |
| update_stmt (call); |
| |
| cond = gimple_build_cond (EQ_EXPR, lhs, |
| fold_convert_loc (UNKNOWN_LOCATION, |
| TREE_TYPE (lhs), |
| boolean_true_node), |
| NULL_TREE, NULL_TREE); |
| gsi_insert_after (&start, cond, GSI_NEW_STMT); |
| update_stmt (cond); |
| |
| edge et = split_block (from, cond); |
| et->flags &= ~EDGE_FALLTHRU; |
| et->flags |= EDGE_TRUE_VALUE; |
| /* Make the active worker the more probable path so we prefer fallthrough |
| (letting the idle workers jump around more). */ |
| et->probability = profile_probability::likely (); |
| |
| edge ef = make_edge (from, skip_block, EDGE_FALSE_VALUE); |
| ef->probability = et->probability.invert (); |
| |
| basic_block neutered = split_edge (ef); |
| gimple_stmt_iterator neut_gsi = gsi_last_bb (neutered); |
| |
| for (gsi = gsi_start_bb (et->dest); !gsi_end_p (gsi); gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| ssa_op_iter iter; |
| tree var; |
| |
| FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF) |
| { |
| if (def_escapes_block->contains (var)) |
| { |
| gphi *join_phi = create_phi_node (NULL_TREE, skip_block); |
| create_new_def_for (var, join_phi, |
| gimple_phi_result_ptr (join_phi)); |
| add_phi_arg (join_phi, var, e, UNKNOWN_LOCATION); |
| |
| tree neutered_def = copy_ssa_name (var, NULL); |
| /* We really want "don't care" or some value representing |
| undefined here, but optimizers will probably get rid of the |
| zero-assignments anyway. */ |
| gassign *zero = gimple_build_assign (neutered_def, |
| build_zero_cst (TREE_TYPE (neutered_def))); |
| |
| gsi_insert_after (&neut_gsi, zero, GSI_CONTINUE_LINKING); |
| update_stmt (zero); |
| |
| add_phi_arg (join_phi, neutered_def, single_succ_edge (neutered), |
| UNKNOWN_LOCATION); |
| update_stmt (join_phi); |
| } |
| } |
| } |
| } |
| |
| /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it |
| as appropriate. */ |
| /* Adapted from 'gcc/omp-low.c:omp_build_component_ref'. */ |
| |
| static tree |
| oacc_build_component_ref (tree obj, tree field) |
| { |
| tree field_type = TREE_TYPE (field); |
| tree obj_type = TREE_TYPE (obj); |
| if (!ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (obj_type))) |
| field_type = build_qualified_type |
| (field_type, |
| KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (obj_type))); |
| |
| tree ret = build3 (COMPONENT_REF, field_type, obj, field, NULL); |
| if (TREE_THIS_VOLATILE (field)) |
| TREE_THIS_VOLATILE (ret) |= 1; |
| if (TREE_READONLY (field)) |
| TREE_READONLY (ret) |= 1; |
| return ret; |
| } |
| |
| static tree |
| build_receiver_ref (tree var, tree receiver_decl, field_map_t *fields) |
| { |
| tree x = build_simple_mem_ref (receiver_decl); |
| tree field = *fields->get (var); |
| TREE_THIS_NOTRAP (x) = 1; |
| x = oacc_build_component_ref (x, field); |
| return x; |
| } |
| |
| static tree |
| build_sender_ref (tree var, tree sender_decl, field_map_t *fields) |
| { |
| if (POINTER_TYPE_P (TREE_TYPE (sender_decl))) |
| sender_decl = build_simple_mem_ref (sender_decl); |
| tree field = *fields->get (var); |
| return oacc_build_component_ref (sender_decl, field); |
| } |
| |
| static int |
| sort_by_ssa_version_or_uid (const void *p1, const void *p2) |
| { |
| const tree t1 = *(const tree *)p1; |
| const tree t2 = *(const tree *)p2; |
| |
| if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) == SSA_NAME) |
| return SSA_NAME_VERSION (t1) - SSA_NAME_VERSION (t2); |
| else if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) != SSA_NAME) |
| return -1; |
| else if (TREE_CODE (t1) != SSA_NAME && TREE_CODE (t2) == SSA_NAME) |
| return 1; |
| else |
| return DECL_UID (t1) - DECL_UID (t2); |
| } |
| |
| static int |
| sort_by_size_then_ssa_version_or_uid (const void *p1, const void *p2) |
| { |
| const tree t1 = *(const tree *)p1; |
| const tree t2 = *(const tree *)p2; |
| unsigned HOST_WIDE_INT s1 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t1))); |
| unsigned HOST_WIDE_INT s2 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t2))); |
| if (s1 != s2) |
| return s2 - s1; |
| else |
| return sort_by_ssa_version_or_uid (p1, p2); |
| } |
| |
| static void |
| worker_single_copy (basic_block from, basic_block to, |
| hash_set<tree> *def_escapes_block, |
| hash_set<tree> *worker_partitioned_uses, |
| tree record_type, record_field_map_t *record_field_map, |
| unsigned HOST_WIDE_INT placement, |
| bool isolate_broadcasts, bool has_gang_private_write) |
| { |
| /* If we only have virtual defs, we'll have no record type, but we still want |
| to emit single_copy_start and (particularly) single_copy_end to act as |
| a vdef source on the neutered edge representing memory writes on the |
| non-neutered edge. */ |
| if (!record_type) |
| record_type = char_type_node; |
| |
| tree sender_decl |
| = targetm.goacc.create_worker_broadcast_record (record_type, true, |
| ".oacc_worker_o", |
| placement); |
| tree receiver_decl |
| = targetm.goacc.create_worker_broadcast_record (record_type, false, |
| ".oacc_worker_i", |
| placement); |
| |
| gimple_stmt_iterator gsi = gsi_last_bb (to); |
| if (EDGE_COUNT (to->succs) > 1) |
| gsi_prev (&gsi); |
| edge e = split_block (to, gsi_stmt (gsi)); |
| basic_block barrier_block = e->dest; |
| |
| gimple_stmt_iterator start = gsi_after_labels (from); |
| |
| tree decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_START); |
| |
| tree lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); |
| |
| gimple *call |
| = gimple_build_call (decl, 1, |
| POINTER_TYPE_P (TREE_TYPE (sender_decl)) |
| ? sender_decl : build_fold_addr_expr (sender_decl)); |
| gimple_call_set_lhs (call, lhs); |
| gsi_insert_before (&start, call, GSI_NEW_STMT); |
| update_stmt (call); |
| |
| /* The shared-memory range for this block overflowed. Add a barrier before |
| the GOACC_single_copy_start call. */ |
| if (isolate_broadcasts) |
| { |
| decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER); |
| gimple *acc_bar = gimple_build_call (decl, 0); |
| gsi_insert_before (&start, acc_bar, GSI_SAME_STMT); |
| } |
| |
| tree conv_tmp = make_ssa_name (TREE_TYPE (receiver_decl)); |
| |
| gimple *conv = gimple_build_assign (conv_tmp, |
| fold_convert (TREE_TYPE (receiver_decl), |
| lhs)); |
| update_stmt (conv); |
| gsi_insert_after (&start, conv, GSI_NEW_STMT); |
| gimple *asgn = gimple_build_assign (receiver_decl, conv_tmp); |
| gsi_insert_after (&start, asgn, GSI_NEW_STMT); |
| update_stmt (asgn); |
| |
| tree zero_ptr = build_int_cst (TREE_TYPE (receiver_decl), 0); |
| |
| tree recv_tmp = make_ssa_name (TREE_TYPE (receiver_decl)); |
| asgn = gimple_build_assign (recv_tmp, receiver_decl); |
| gsi_insert_after (&start, asgn, GSI_NEW_STMT); |
| update_stmt (asgn); |
| |
| gimple *cond = gimple_build_cond (EQ_EXPR, recv_tmp, zero_ptr, NULL_TREE, |
| NULL_TREE); |
| update_stmt (cond); |
| |
| gsi_insert_after (&start, cond, GSI_NEW_STMT); |
| |
| edge et = split_block (from, cond); |
| et->flags &= ~EDGE_FALLTHRU; |
| et->flags |= EDGE_TRUE_VALUE; |
| /* Make the active worker the more probable path so we prefer fallthrough |
| (letting the idle workers jump around more). */ |
| et->probability = profile_probability::likely (); |
| |
| basic_block body = et->dest; |
| |
| edge ef = make_edge (from, barrier_block, EDGE_FALSE_VALUE); |
| ef->probability = et->probability.invert (); |
| |
| gimple_stmt_iterator bar_gsi = gsi_start_bb (barrier_block); |
| cond = gimple_build_cond (NE_EXPR, recv_tmp, zero_ptr, NULL_TREE, NULL_TREE); |
| |
| if (record_type != char_type_node || has_gang_private_write) |
| { |
| decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER); |
| gimple *acc_bar = gimple_build_call (decl, 0); |
| |
| gsi_insert_before (&bar_gsi, acc_bar, GSI_NEW_STMT); |
| gsi_insert_after (&bar_gsi, cond, GSI_NEW_STMT); |
| } |
| else |
| gsi_insert_before (&bar_gsi, cond, GSI_NEW_STMT); |
| |
| edge et2 = split_block (barrier_block, cond); |
| et2->flags &= ~EDGE_FALLTHRU; |
| et2->flags |= EDGE_TRUE_VALUE; |
| et2->probability = profile_probability::unlikely (); |
| |
| basic_block exit_block = et2->dest; |
| |
| basic_block copyout_block = split_edge (et2); |
| edge ef2 = make_edge (barrier_block, exit_block, EDGE_FALSE_VALUE); |
| ef2->probability = et2->probability.invert (); |
| |
| gimple_stmt_iterator copyout_gsi = gsi_start_bb (copyout_block); |
| |
| edge copyout_to_exit = single_succ_edge (copyout_block); |
| |
| gimple_seq sender_seq = NULL; |
| |
| /* Make sure we iterate over definitions in a stable order. */ |
| auto_vec<tree> escape_vec (def_escapes_block->elements ()); |
| for (hash_set<tree>::iterator it = def_escapes_block->begin (); |
| it != def_escapes_block->end (); ++it) |
| escape_vec.quick_push (*it); |
| escape_vec.qsort (sort_by_ssa_version_or_uid); |
| |
| for (unsigned i = 0; i < escape_vec.length (); i++) |
| { |
| tree var = escape_vec[i]; |
| |
| if (TREE_CODE (var) == SSA_NAME && SSA_NAME_IS_VIRTUAL_OPERAND (var)) |
| continue; |
| |
| tree barrier_def = 0; |
| |
| if (TREE_CODE (var) == SSA_NAME) |
| { |
| gimple *def_stmt = SSA_NAME_DEF_STMT (var); |
| |
| if (gimple_nop_p (def_stmt)) |
| continue; |
| |
| /* The barrier phi takes one result from the actual work of the |
| block we're neutering, and the other result is constant zero of |
| the same type. */ |
| |
| gphi *barrier_phi = create_phi_node (NULL_TREE, barrier_block); |
| barrier_def = create_new_def_for (var, barrier_phi, |
| gimple_phi_result_ptr (barrier_phi)); |
| |
| add_phi_arg (barrier_phi, var, e, UNKNOWN_LOCATION); |
| add_phi_arg (barrier_phi, build_zero_cst (TREE_TYPE (var)), ef, |
| UNKNOWN_LOCATION); |
| |
| update_stmt (barrier_phi); |
| } |
| else |
| gcc_assert (TREE_CODE (var) == VAR_DECL); |
| |
| /* If we had no record type, we will have no fields map. */ |
| field_map_t **fields_p = record_field_map->get (record_type); |
| field_map_t *fields = fields_p ? *fields_p : NULL; |
| |
| if (worker_partitioned_uses->contains (var) |
| && fields |
| && fields->get (var)) |
| { |
| tree neutered_def = make_ssa_name (TREE_TYPE (var)); |
| |
| /* Receive definition from shared memory block. */ |
| |
| tree receiver_ref = build_receiver_ref (var, receiver_decl, fields); |
| gassign *recv = gimple_build_assign (neutered_def, |
| receiver_ref); |
| gsi_insert_after (©out_gsi, recv, GSI_CONTINUE_LINKING); |
| update_stmt (recv); |
| |
| if (TREE_CODE (var) == VAR_DECL) |
| { |
| /* If it's a VAR_DECL, we only copied to an SSA temporary. Copy |
| to the final location now. */ |
| gassign *asgn = gimple_build_assign (var, neutered_def); |
| gsi_insert_after (©out_gsi, asgn, GSI_CONTINUE_LINKING); |
| update_stmt (asgn); |
| } |
| else |
| { |
| /* If it's an SSA name, create a new phi at the join node to |
| represent either the output from the active worker (the |
| barrier) or the inactive workers (the copyout block). */ |
| gphi *join_phi = create_phi_node (NULL_TREE, exit_block); |
| create_new_def_for (barrier_def, join_phi, |
| gimple_phi_result_ptr (join_phi)); |
| add_phi_arg (join_phi, barrier_def, ef2, UNKNOWN_LOCATION); |
| add_phi_arg (join_phi, neutered_def, copyout_to_exit, |
| UNKNOWN_LOCATION); |
| update_stmt (join_phi); |
| } |
| |
| /* Send definition to shared memory block. */ |
| |
| tree sender_ref = build_sender_ref (var, sender_decl, fields); |
| |
| if (TREE_CODE (var) == SSA_NAME) |
| { |
| gassign *send = gimple_build_assign (sender_ref, var); |
| gimple_seq_add_stmt (&sender_seq, send); |
| update_stmt (send); |
| } |
| else if (TREE_CODE (var) == VAR_DECL) |
| { |
| tree tmp = make_ssa_name (TREE_TYPE (var)); |
| gassign *send = gimple_build_assign (tmp, var); |
| gimple_seq_add_stmt (&sender_seq, send); |
| update_stmt (send); |
| send = gimple_build_assign (sender_ref, tmp); |
| gimple_seq_add_stmt (&sender_seq, send); |
| update_stmt (send); |
| } |
| else |
| gcc_unreachable (); |
| } |
| } |
| |
| /* The shared-memory range for this block overflowed. Add a barrier at the |
| end. */ |
| if (isolate_broadcasts) |
| { |
| gsi = gsi_start_bb (exit_block); |
| decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER); |
| gimple *acc_bar = gimple_build_call (decl, 0); |
| gsi_insert_before (&gsi, acc_bar, GSI_SAME_STMT); |
| } |
| |
| /* It's possible for the ET->DEST block (the work done by the active thread) |
| to finish with a control-flow insn, e.g. a UNIQUE function call. Split |
| the block and add SENDER_SEQ in the latter part to avoid having control |
| flow in the middle of a BB. */ |
| |
| decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_END); |
| call = gimple_build_call (decl, 1, |
| POINTER_TYPE_P (TREE_TYPE (sender_decl)) |
| ? sender_decl |
| : build_fold_addr_expr (sender_decl)); |
| gimple_seq_add_stmt (&sender_seq, call); |
| |
| gsi = gsi_last_bb (body); |
| gimple *last = gsi_stmt (gsi); |
| basic_block sender_block = split_block (body, last)->dest; |
| gsi = gsi_last_bb (sender_block); |
| gsi_insert_seq_after (&gsi, sender_seq, GSI_CONTINUE_LINKING); |
| } |
| |
| typedef hash_map<basic_block, std::pair<unsigned HOST_WIDE_INT, bool> > |
| blk_offset_map_t; |
| |
| static void |
| neuter_worker_single (parallel_g *par, unsigned outer_mask, |
| bitmap worker_single, bitmap vector_single, |
| vec<propagation_set *> *prop_set, |
| hash_set<tree> *partitioned_var_uses, |
| record_field_map_t *record_field_map, |
| blk_offset_map_t *blk_offset_map, |
| bitmap writes_gang_private) |
| { |
| unsigned mask = outer_mask | par->mask; |
| |
| if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0) |
| { |
| basic_block block; |
| |
| for (unsigned i = 0; par->blocks.iterate (i, &block); i++) |
| { |
| bool has_defs = false; |
| hash_set<tree> def_escapes_block; |
| hash_set<tree> worker_partitioned_uses; |
| unsigned j; |
| tree var; |
| |
| FOR_EACH_SSA_NAME (j, var, cfun) |
| { |
| if (SSA_NAME_IS_VIRTUAL_OPERAND (var)) |
| { |
| has_defs = true; |
| continue; |
| } |
| |
| gimple *def_stmt = SSA_NAME_DEF_STMT (var); |
| |
| if (gimple_nop_p (def_stmt)) |
| continue; |
| |
| if (gimple_bb (def_stmt)->index != block->index) |
| continue; |
| |
| gimple *use_stmt; |
| imm_use_iterator use_iter; |
| bool uses_outside_block = false; |
| bool worker_partitioned_use = false; |
| |
| FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, var) |
| { |
| int blocknum = gimple_bb (use_stmt)->index; |
| |
| /* Don't propagate SSA names that are only used in the |
| current block, unless the usage is in a phi node: that |
| means the name left the block, then came back in at the |
| top. */ |
| if (blocknum != block->index |
| || gimple_code (use_stmt) == GIMPLE_PHI) |
| uses_outside_block = true; |
| if (!bitmap_bit_p (worker_single, blocknum)) |
| worker_partitioned_use = true; |
| } |
| |
| if (uses_outside_block) |
| def_escapes_block.add (var); |
| |
| if (worker_partitioned_use) |
| { |
| worker_partitioned_uses.add (var); |
| has_defs = true; |
| } |
| } |
| |
| propagation_set *ws_prop = (*prop_set)[block->index]; |
| |
| if (ws_prop) |
| { |
| for (propagation_set::iterator it = ws_prop->begin (); |
| it != ws_prop->end (); |
| ++it) |
| { |
| tree var = *it; |
| if (TREE_CODE (var) == VAR_DECL) |
| { |
| def_escapes_block.add (var); |
| if (partitioned_var_uses->contains (var)) |
| { |
| worker_partitioned_uses.add (var); |
| has_defs = true; |
| } |
| } |
| } |
| |
| delete ws_prop; |
| (*prop_set)[block->index] = 0; |
| } |
| |
| bool only_marker_fns = true; |
| bool join_block = false; |
| |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| if (gimple_code (stmt) == GIMPLE_CALL |
| && gimple_call_internal_p (stmt, IFN_UNIQUE)) |
| { |
| enum ifn_unique_kind k = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (stmt, 0))); |
| if (k != IFN_UNIQUE_OACC_PRIVATE |
| && k != IFN_UNIQUE_OACC_JOIN |
| && k != IFN_UNIQUE_OACC_FORK |
| && k != IFN_UNIQUE_OACC_HEAD_MARK |
| && k != IFN_UNIQUE_OACC_TAIL_MARK) |
| only_marker_fns = false; |
| else if (k == IFN_UNIQUE_OACC_JOIN) |
| /* The JOIN marker is special in that it *cannot* be |
| predicated for worker zero, because it may be lowered |
| to a barrier instruction and all workers must typically |
| execute that barrier. We shouldn't be doing any |
| broadcasts from the join block anyway. */ |
| join_block = true; |
| } |
| else if (gimple_code (stmt) == GIMPLE_CALL |
| && gimple_call_internal_p (stmt, IFN_GOACC_LOOP)) |
| /* Empty. */; |
| else if (gimple_nop_p (stmt)) |
| /* Empty. */; |
| else |
| only_marker_fns = false; |
| } |
| |
| /* We can skip predicating this block for worker zero if the only |
| thing it contains is marker functions that will be removed in the |
| oaccdevlow pass anyway. |
| Don't do this if the block has (any) phi nodes, because those |
| might define SSA names that need broadcasting. |
| TODO: We might be able to skip transforming blocks that only |
| contain some other trivial statements too. */ |
| if (only_marker_fns && !phi_nodes (block)) |
| continue; |
| |
| gcc_assert (!join_block); |
| |
| if (has_defs) |
| { |
| tree record_type = (tree) block->aux; |
| std::pair<unsigned HOST_WIDE_INT, bool> *off_rngalloc |
| = blk_offset_map->get (block); |
| gcc_assert (!record_type || off_rngalloc); |
| unsigned HOST_WIDE_INT offset |
| = off_rngalloc ? off_rngalloc->first : 0; |
| bool range_allocated |
| = off_rngalloc ? off_rngalloc->second : true; |
| bool has_gang_private_write |
| = bitmap_bit_p (writes_gang_private, block->index); |
| worker_single_copy (block, block, &def_escapes_block, |
| &worker_partitioned_uses, record_type, |
| record_field_map, |
| offset, !range_allocated, |
| has_gang_private_write); |
| } |
| else |
| worker_single_simple (block, block, &def_escapes_block); |
| } |
| } |
| |
| if ((outer_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0) |
| { |
| basic_block block; |
| |
| for (unsigned i = 0; par->blocks.iterate (i, &block); i++) |
| for (gimple_stmt_iterator gsi = gsi_start_bb (block); |
| !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| |
| if (gimple_code (stmt) == GIMPLE_CALL |
| && !gimple_call_internal_p (stmt) |
| && !omp_sese_active_worker_call (as_a <gcall *> (stmt))) |
| { |
| /* If we have an OpenACC routine call in worker-single mode, |
| place barriers before and afterwards to prevent |
| clobbering re-used shared memory regions (as are used |
| for AMDGCN at present, for example). */ |
| tree decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER); |
| gsi_insert_before (&gsi, gimple_build_call (decl, 0), |
| GSI_SAME_STMT); |
| gsi_insert_after (&gsi, gimple_build_call (decl, 0), |
| GSI_NEW_STMT); |
| } |
| } |
| } |
| |
| if (par->inner) |
| neuter_worker_single (par->inner, mask, worker_single, vector_single, |
| prop_set, partitioned_var_uses, record_field_map, |
| blk_offset_map, writes_gang_private); |
| if (par->next) |
| neuter_worker_single (par->next, outer_mask, worker_single, vector_single, |
| prop_set, partitioned_var_uses, record_field_map, |
| blk_offset_map, writes_gang_private); |
| } |
| |
| static void |
| dfs_broadcast_reachable_1 (basic_block bb, sbitmap reachable) |
| { |
| if (bb->flags & BB_VISITED) |
| return; |
| |
| bb->flags |= BB_VISITED; |
| |
| if (bb->succs) |
| { |
| edge e; |
| edge_iterator ei; |
| FOR_EACH_EDGE (e, ei, bb->succs) |
| { |
| basic_block dest = e->dest; |
| if (dest->aux) |
| bitmap_set_bit (reachable, dest->index); |
| else |
| dfs_broadcast_reachable_1 (dest, reachable); |
| } |
| } |
| } |
| |
| typedef std::pair<int, tree> idx_decl_pair_t; |
| |
| typedef auto_vec<splay_tree> used_range_vec_t; |
| |
| static int |
| sort_size_descending (const void *a, const void *b) |
| { |
| const idx_decl_pair_t *pa = (const idx_decl_pair_t *) a; |
| const idx_decl_pair_t *pb = (const idx_decl_pair_t *) b; |
| unsigned HOST_WIDE_INT asize = tree_to_uhwi (TYPE_SIZE_UNIT (pa->second)); |
| unsigned HOST_WIDE_INT bsize = tree_to_uhwi (TYPE_SIZE_UNIT (pb->second)); |
| return bsize - asize; |
| } |
| |
| class addr_range |
| { |
| public: |
| addr_range (unsigned HOST_WIDE_INT addr_lo, unsigned HOST_WIDE_INT addr_hi) |
| : lo (addr_lo), hi (addr_hi) |
| { } |
| addr_range (const addr_range &ar) : lo (ar.lo), hi (ar.hi) |
| { } |
| addr_range () : lo (0), hi (0) |
| { } |
| |
| bool invalid () { return lo == 0 && hi == 0; } |
| |
| unsigned HOST_WIDE_INT lo; |
| unsigned HOST_WIDE_INT hi; |
| }; |
| |
| static int |
| splay_tree_compare_addr_range (splay_tree_key a, splay_tree_key b) |
| { |
| addr_range *ar = (addr_range *) a; |
| addr_range *br = (addr_range *) b; |
| if (ar->lo == br->lo && ar->hi == br->hi) |
| return 0; |
| if (ar->hi <= br->lo) |
| return -1; |
| else if (ar->lo >= br->hi) |
| return 1; |
| return 0; |
| } |
| |
| static void |
| splay_tree_free_key (splay_tree_key k) |
| { |
| addr_range *ar = (addr_range *) k; |
| delete ar; |
| } |
| |
| static addr_range |
| first_fit_range (splay_tree s, unsigned HOST_WIDE_INT size, |
| unsigned HOST_WIDE_INT align, addr_range *bounds) |
| { |
| splay_tree_node min = splay_tree_min (s); |
| if (min) |
| { |
| splay_tree_node next; |
| while ((next = splay_tree_successor (s, min->key))) |
| { |
| unsigned HOST_WIDE_INT lo = ((addr_range *) min->key)->hi; |
| unsigned HOST_WIDE_INT hi = ((addr_range *) next->key)->lo; |
| unsigned HOST_WIDE_INT base = (lo + align - 1) & ~(align - 1); |
| if (base + size <= hi) |
| return addr_range (base, base + size); |
| min = next; |
| } |
| |
| unsigned HOST_WIDE_INT base = ((addr_range *)min->key)->hi; |
| base = (base + align - 1) & ~(align - 1); |
| if (base + size <= bounds->hi) |
| return addr_range (base, base + size); |
| else |
| return addr_range (); |
| } |
| else |
| { |
| unsigned HOST_WIDE_INT lo = bounds->lo; |
| lo = (lo + align - 1) & ~(align - 1); |
| if (lo + size <= bounds->hi) |
| return addr_range (lo, lo + size); |
| else |
| return addr_range (); |
| } |
| } |
| |
| static int |
| merge_ranges_1 (splay_tree_node n, void *ptr) |
| { |
| splay_tree accum = (splay_tree) ptr; |
| addr_range ar = *(addr_range *) n->key; |
| |
| splay_tree_node old = splay_tree_lookup (accum, n->key); |
| |
| /* We might have an overlap. Create a new range covering the |
| overlapping parts. */ |
| if (old) |
| { |
| addr_range *old_ar = (addr_range *) old->key; |
| ar.lo = MIN (old_ar->lo, ar.lo); |
| ar.hi = MAX (old_ar->hi, ar.hi); |
| splay_tree_remove (accum, old->key); |
| } |
| |
| addr_range *new_ar = new addr_range (ar); |
| |
| splay_tree_insert (accum, (splay_tree_key) new_ar, n->value); |
| |
| return 0; |
| } |
| |
| static void |
| merge_ranges (splay_tree accum, splay_tree sp) |
| { |
| splay_tree_foreach (sp, merge_ranges_1, (void *) accum); |
| } |
| |
| static void |
| oacc_do_neutering (unsigned HOST_WIDE_INT bounds_lo, |
| unsigned HOST_WIDE_INT bounds_hi) |
| { |
| bb_stmt_map_t bb_stmt_map; |
| auto_bitmap worker_single, vector_single; |
| |
| omp_sese_split_blocks (&bb_stmt_map); |
| |
| if (dump_file) |
| { |
| fprintf (dump_file, "\n\nAfter splitting:\n\n"); |
| dump_function_to_file (current_function_decl, dump_file, dump_flags); |
| } |
| |
| unsigned mask = 0; |
| |
| /* If this is a routine, calculate MASK as if the outer levels are already |
| partitioned. */ |
| { |
| tree attr = oacc_get_fn_attrib (current_function_decl); |
| tree dims = TREE_VALUE (attr); |
| unsigned ix; |
| for (ix = 0; ix != GOMP_DIM_MAX; ix++, dims = TREE_CHAIN (dims)) |
| { |
| tree allowed = TREE_PURPOSE (dims); |
| if (allowed && integer_zerop (allowed)) |
| mask |= GOMP_DIM_MASK (ix); |
| } |
| } |
| |
| parallel_g *par = omp_sese_discover_pars (&bb_stmt_map); |
| populate_single_mode_bitmaps (par, worker_single, vector_single, mask, 0); |
| |
| basic_block bb; |
| FOR_ALL_BB_FN (bb, cfun) |
| bb->aux = NULL; |
| |
| vec<propagation_set *> prop_set (vNULL); |
| prop_set.safe_grow_cleared (last_basic_block_for_fn (cfun), true); |
| |
| find_ssa_names_to_propagate (par, mask, worker_single, vector_single, |
| &prop_set); |
| |
| hash_set<tree> partitioned_var_uses; |
| hash_set<tree> gang_private_vars; |
| auto_bitmap writes_gang_private; |
| |
| find_gang_private_vars (&gang_private_vars); |
| find_partitioned_var_uses (par, mask, &partitioned_var_uses); |
| find_local_vars_to_propagate (par, mask, &partitioned_var_uses, |
| &gang_private_vars, writes_gang_private, |
| &prop_set); |
| |
| record_field_map_t record_field_map; |
| |
| FOR_ALL_BB_FN (bb, cfun) |
| { |
| propagation_set *ws_prop = prop_set[bb->index]; |
| if (ws_prop) |
| { |
| tree record_type = lang_hooks.types.make_type (RECORD_TYPE); |
| tree name = create_tmp_var_name (".oacc_ws_data_s"); |
| name = build_decl (UNKNOWN_LOCATION, TYPE_DECL, name, record_type); |
| DECL_ARTIFICIAL (name) = 1; |
| DECL_NAMELESS (name) = 1; |
| TYPE_NAME (record_type) = name; |
| TYPE_ARTIFICIAL (record_type) = 1; |
| |
| auto_vec<tree> field_vec (ws_prop->elements ()); |
| for (hash_set<tree>::iterator it = ws_prop->begin (); |
| it != ws_prop->end (); ++it) |
| field_vec.quick_push (*it); |
| |
| field_vec.qsort (sort_by_size_then_ssa_version_or_uid); |
| |
| field_map_t *fields = new field_map_t; |
| |
| bool existed; |
| existed = record_field_map.put (record_type, fields); |
| gcc_checking_assert (!existed); |
| |
| /* Insert var fields in reverse order, so the last inserted element |
| is the first in the structure. */ |
| for (int i = field_vec.length () - 1; i >= 0; i--) |
| install_var_field (field_vec[i], record_type, fields); |
| |
| layout_type (record_type); |
| |
| bb->aux = (tree) record_type; |
| } |
| } |
| |
| sbitmap *reachable |
| = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), |
| last_basic_block_for_fn (cfun)); |
| |
| bitmap_vector_clear (reachable, last_basic_block_for_fn (cfun)); |
| |
| auto_vec<std::pair<int, tree> > priority; |
| |
| FOR_ALL_BB_FN (bb, cfun) |
| { |
| if (bb->aux) |
| { |
| tree record_type = (tree) bb->aux; |
| |
| basic_block bb2; |
| FOR_ALL_BB_FN (bb2, cfun) |
| bb2->flags &= ~BB_VISITED; |
| |
| priority.safe_push (std::make_pair (bb->index, record_type)); |
| dfs_broadcast_reachable_1 (bb, reachable[bb->index]); |
| } |
| } |
| |
| sbitmap *inverted |
| = sbitmap_vector_alloc (last_basic_block_for_fn (cfun), |
| last_basic_block_for_fn (cfun)); |
| |
| bitmap_vector_clear (inverted, last_basic_block_for_fn (cfun)); |
| |
| for (int i = 0; i < last_basic_block_for_fn (cfun); i++) |
| { |
| sbitmap_iterator bi; |
| unsigned int j; |
| EXECUTE_IF_SET_IN_BITMAP (reachable[i], 0, j, bi) |
| bitmap_set_bit (inverted[j], i); |
| } |
| |
| for (int i = 0; i < last_basic_block_for_fn (cfun); i++) |
| bitmap_ior (reachable[i], reachable[i], inverted[i]); |
| |
| sbitmap_vector_free (inverted); |
| |
| used_range_vec_t used_ranges; |
| |
| used_ranges.safe_grow_cleared (last_basic_block_for_fn (cfun)); |
| |
| blk_offset_map_t blk_offset_map; |
| |
| addr_range worker_shm_bounds (bounds_lo, bounds_hi); |
| |
| priority.qsort (sort_size_descending); |
| for (unsigned int i = 0; i < priority.length (); i++) |
| { |
| idx_decl_pair_t p = priority[i]; |
| int blkno = p.first; |
| tree record_type = p.second; |
| HOST_WIDE_INT size = tree_to_uhwi (TYPE_SIZE_UNIT (record_type)); |
| HOST_WIDE_INT align = TYPE_ALIGN_UNIT (record_type); |
| |
| splay_tree conflicts = splay_tree_new (splay_tree_compare_addr_range, |
| splay_tree_free_key, NULL); |
| |
| if (!used_ranges[blkno]) |
| used_ranges[blkno] = splay_tree_new (splay_tree_compare_addr_range, |
| splay_tree_free_key, NULL); |
| else |
| merge_ranges (conflicts, used_ranges[blkno]); |
| |
| sbitmap_iterator bi; |
| unsigned int j; |
| EXECUTE_IF_SET_IN_BITMAP (reachable[blkno], 0, j, bi) |
| if (used_ranges[j]) |
| merge_ranges (conflicts, used_ranges[j]); |
| |
| addr_range ar |
| = first_fit_range (conflicts, size, align, &worker_shm_bounds); |
| |
| splay_tree_delete (conflicts); |
| |
| if (ar.invalid ()) |
| { |
| unsigned HOST_WIDE_INT base |
| = (bounds_lo + align - 1) & ~(align - 1); |
| if (base + size > bounds_hi) |
| error_at (UNKNOWN_LOCATION, "shared-memory region overflow"); |
| std::pair<unsigned HOST_WIDE_INT, bool> base_inrng |
| = std::make_pair (base, false); |
| blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng); |
| } |
| else |
| { |
| splay_tree_node old = splay_tree_lookup (used_ranges[blkno], |
| (splay_tree_key) &ar); |
| if (old) |
| { |
| fprintf (stderr, "trying to map [%d..%d] but [%d..%d] is " |
| "already mapped in block %d\n", (int) ar.lo, |
| (int) ar.hi, (int) ((addr_range *) old->key)->lo, |
| (int) ((addr_range *) old->key)->hi, blkno); |
| abort (); |
| } |
| |
| addr_range *arp = new addr_range (ar); |
| splay_tree_insert (used_ranges[blkno], (splay_tree_key) arp, |
| (splay_tree_value) blkno); |
| std::pair<unsigned HOST_WIDE_INT, bool> base_inrng |
| = std::make_pair (ar.lo, true); |
| blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng); |
| } |
| } |
| |
| sbitmap_vector_free (reachable); |
| |
| neuter_worker_single (par, mask, worker_single, vector_single, &prop_set, |
| &partitioned_var_uses, &record_field_map, |
| &blk_offset_map, writes_gang_private); |
| |
| for (auto it : record_field_map) |
| delete it.second; |
| record_field_map.empty (); |
| |
| /* These are supposed to have been 'delete'd by 'neuter_worker_single'. */ |
| for (auto it : prop_set) |
| gcc_checking_assert (!it); |
| prop_set.release (); |
| |
| delete par; |
| |
| /* This doesn't seem to make a difference. */ |
| loops_state_clear (LOOP_CLOSED_SSA); |
| |
| /* Neutering worker-single neutered blocks will invalidate dominance info. |
| It may be possible to incrementally update just the affected blocks, but |
| obliterate everything for now. */ |
| free_dominance_info (CDI_DOMINATORS); |
| free_dominance_info (CDI_POST_DOMINATORS); |
| |
| if (dump_file) |
| { |
| fprintf (dump_file, "\n\nAfter neutering:\n\n"); |
| dump_function_to_file (current_function_decl, dump_file, dump_flags); |
| } |
| } |
| |
| static int |
| execute_omp_oacc_neuter_broadcast () |
| { |
| unsigned HOST_WIDE_INT reduction_size[GOMP_DIM_MAX]; |
| unsigned HOST_WIDE_INT private_size[GOMP_DIM_MAX]; |
| |
| for (unsigned i = 0; i < GOMP_DIM_MAX; i++) |
| { |
| reduction_size[i] = 0; |
| private_size[i] = 0; |
| } |
| |
| /* Calculate shared memory size required for reduction variables and |
| gang-private memory for this offloaded function. */ |
| basic_block bb; |
| FOR_ALL_BB_FN (bb, cfun) |
| { |
| for (gimple_stmt_iterator gsi = gsi_start_bb (bb); |
| !gsi_end_p (gsi); |
| gsi_next (&gsi)) |
| { |
| gimple *stmt = gsi_stmt (gsi); |
| if (!is_gimple_call (stmt)) |
| continue; |
| gcall *call = as_a <gcall *> (stmt); |
| if (!gimple_call_internal_p (call)) |
| continue; |
| enum internal_fn ifn_code = gimple_call_internal_fn (call); |
| switch (ifn_code) |
| { |
| default: break; |
| case IFN_GOACC_REDUCTION: |
| if (integer_minus_onep (gimple_call_arg (call, 3))) |
| continue; |
| else |
| { |
| unsigned code = TREE_INT_CST_LOW (gimple_call_arg (call, 0)); |
| /* Only count reduction variables once: the choice to pick |
| the setup call is fairly arbitrary. */ |
| if (code == IFN_GOACC_REDUCTION_SETUP) |
| { |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| tree var = gimple_call_arg (call, 2); |
| tree offset = gimple_call_arg (call, 5); |
| tree var_type = TREE_TYPE (var); |
| unsigned HOST_WIDE_INT limit |
| = (tree_to_uhwi (offset) |
| + tree_to_uhwi (TYPE_SIZE_UNIT (var_type))); |
| reduction_size[level] |
| = MAX (reduction_size[level], limit); |
| } |
| } |
| break; |
| case IFN_UNIQUE: |
| { |
| enum ifn_unique_kind kind |
| = ((enum ifn_unique_kind) |
| TREE_INT_CST_LOW (gimple_call_arg (call, 0))); |
| |
| if (kind == IFN_UNIQUE_OACC_PRIVATE) |
| { |
| HOST_WIDE_INT level |
| = TREE_INT_CST_LOW (gimple_call_arg (call, 2)); |
| if (level == -1) |
| break; |
| for (unsigned i = 3; |
| i < gimple_call_num_args (call); |
| i++) |
| { |
| tree arg = gimple_call_arg (call, i); |
| gcc_assert (TREE_CODE (arg) == ADDR_EXPR); |
| tree decl = TREE_OPERAND (arg, 0); |
| unsigned HOST_WIDE_INT align = DECL_ALIGN_UNIT (decl); |
| private_size[level] = ((private_size[level] + align - 1) |
| & ~(align - 1)); |
| unsigned HOST_WIDE_INT decl_size |
| = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (decl))); |
| private_size[level] += decl_size; |
| } |
| } |
| } |
| break; |
| } |
| } |
| } |
| |
| int dims[GOMP_DIM_MAX]; |
| for (unsigned i = 0; i < GOMP_DIM_MAX; i++) |
| dims[i] = oacc_get_fn_dim_size (current_function_decl, i); |
| |
| /* Find bounds of shared-memory buffer space we can use. */ |
| unsigned HOST_WIDE_INT bounds_lo = 0, bounds_hi = 0; |
| if (targetm.goacc.shared_mem_layout) |
| targetm.goacc.shared_mem_layout (&bounds_lo, &bounds_hi, dims, |
| private_size, reduction_size); |
| |
| /* Perform worker partitioning unless we know 'num_workers(1)'. */ |
| if (dims[GOMP_DIM_WORKER] != 1) |
| oacc_do_neutering (bounds_lo, bounds_hi); |
| |
| return 0; |
| } |
| |
| namespace { |
| |
| const pass_data pass_data_omp_oacc_neuter_broadcast = |
| { |
| GIMPLE_PASS, /* type */ |
| "omp_oacc_neuter_broadcast", /* name */ |
| OPTGROUP_OMP, /* optinfo_flags */ |
| TV_NONE, /* tv_id */ |
| PROP_cfg, /* properties_required */ |
| 0, /* properties_provided */ |
| 0, /* properties_destroyed */ |
| 0, /* todo_flags_start */ |
| TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */ |
| }; |
| |
| class pass_omp_oacc_neuter_broadcast : public gimple_opt_pass |
| { |
| public: |
| pass_omp_oacc_neuter_broadcast (gcc::context *ctxt) |
| : gimple_opt_pass (pass_data_omp_oacc_neuter_broadcast, ctxt) |
| {} |
| |
| /* opt_pass methods: */ |
| virtual bool gate (function *fun) |
| { |
| if (!flag_openacc) |
| return false; |
| |
| if (!targetm.goacc.create_worker_broadcast_record) |
| return false; |
| |
| /* Only relevant for OpenACC offloaded functions. */ |
| tree attr = oacc_get_fn_attrib (fun->decl); |
| if (!attr) |
| return false; |
| |
| return true; |
| } |
| |
| virtual unsigned int execute (function *) |
| { |
| return execute_omp_oacc_neuter_broadcast (); |
| } |
| |
| }; // class pass_omp_oacc_neuter_broadcast |
| |
| } // anon namespace |
| |
| gimple_opt_pass * |
| make_pass_omp_oacc_neuter_broadcast (gcc::context *ctxt) |
| { |
| return new pass_omp_oacc_neuter_broadcast (ctxt); |
| } |