| /* Copyright (C) 2017-2020 Free Software Foundation, Inc. |
| |
| This file is part of GCC. |
| |
| GCC is free software; you can redistribute it and/or modify it under |
| the terms of the GNU General Public License as published by the Free |
| Software Foundation; either version 3, or (at your option) any later |
| version. |
| |
| GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| for more details. |
| |
| You should have received a copy of the GNU General Public License |
| along with GCC; see the file COPYING3. If not see |
| <http://www.gnu.org/licenses/>. */ |
| |
| /* {{{ Includes. */ |
| |
| #include "config.h" |
| #include "system.h" |
| #include "coretypes.h" |
| #include "backend.h" |
| #include "target.h" |
| #include "tree.h" |
| #include "gimple.h" |
| #include "tree-pass.h" |
| #include "gimple-iterator.h" |
| #include "cfghooks.h" |
| #include "cfgloop.h" |
| #include "tm_p.h" |
| #include "stringpool.h" |
| #include "fold-const.h" |
| #include "varasm.h" |
| #include "omp-low.h" |
| #include "omp-general.h" |
| #include "internal-fn.h" |
| #include "tree-vrp.h" |
| #include "tree-ssanames.h" |
| #include "tree-ssa-operands.h" |
| #include "gimplify.h" |
| #include "tree-phinodes.h" |
| #include "cgraph.h" |
| #include "targhooks.h" |
| #include "langhooks-def.h" |
| |
| /* }}} */ |
| /* {{{ OMP GCN pass. |
| |
| This pass is intended to make any GCN-specfic transformations to OpenMP |
| target regions. |
| |
| At present, its only purpose is to convert some "omp" built-in functions |
| to use closer-to-the-metal "gcn" built-in functions. */ |
| |
| unsigned int |
| execute_omp_gcn (void) |
| { |
| tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| tree thr_num_id = DECL_NAME (thr_num_tree); |
| tree team_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); |
| tree team_num_id = DECL_NAME (team_num_tree); |
| basic_block bb; |
| gimple_stmt_iterator gsi; |
| unsigned int todo = 0; |
| |
| FOR_EACH_BB_FN (bb, cfun) |
| for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) |
| { |
| gimple *call = gsi_stmt (gsi); |
| tree decl; |
| |
| if (is_gimple_call (call) && (decl = gimple_call_fndecl (call))) |
| { |
| tree decl_id = DECL_NAME (decl); |
| tree lhs = gimple_get_lhs (call); |
| |
| if (decl_id == thr_num_id) |
| { |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| fprintf (dump_file, |
| "Replace '%s' with __builtin_gcn_dim_pos.\n", |
| IDENTIFIER_POINTER (decl_id)); |
| |
| /* Transform this: |
| lhs = __builtin_omp_get_thread_num () |
| to this: |
| lhs = __builtin_gcn_dim_pos (1) */ |
| tree fn = targetm.builtin_decl (GCN_BUILTIN_OMP_DIM_POS, 0); |
| tree fnarg = build_int_cst (unsigned_type_node, 1); |
| gimple *stmt = gimple_build_call (fn, 1, fnarg); |
| gimple_call_set_lhs (stmt, lhs); |
| gsi_replace (&gsi, stmt, true); |
| |
| todo |= TODO_update_ssa; |
| } |
| else if (decl_id == team_num_id) |
| { |
| if (dump_file && (dump_flags & TDF_DETAILS)) |
| fprintf (dump_file, |
| "Replace '%s' with __builtin_gcn_dim_pos.\n", |
| IDENTIFIER_POINTER (decl_id)); |
| |
| /* Transform this: |
| lhs = __builtin_omp_get_team_num () |
| to this: |
| lhs = __builtin_gcn_dim_pos (0) */ |
| tree fn = targetm.builtin_decl (GCN_BUILTIN_OMP_DIM_POS, 0); |
| tree fnarg = build_zero_cst (unsigned_type_node); |
| gimple *stmt = gimple_build_call (fn, 1, fnarg); |
| gimple_call_set_lhs (stmt, lhs); |
| gsi_replace (&gsi, stmt, true); |
| |
| todo |= TODO_update_ssa; |
| } |
| } |
| } |
| |
| return todo; |
| } |
| |
| namespace |
| { |
| |
| const pass_data pass_data_omp_gcn = { |
| GIMPLE_PASS, |
| "omp_gcn", /* name */ |
| OPTGROUP_NONE, /* optinfo_flags */ |
| TV_NONE, /* tv_id */ |
| 0, /* properties_required */ |
| 0, /* properties_provided */ |
| 0, /* properties_destroyed */ |
| 0, /* todo_flags_start */ |
| TODO_df_finish, /* todo_flags_finish */ |
| }; |
| |
| class pass_omp_gcn : public gimple_opt_pass |
| { |
| public: |
| pass_omp_gcn (gcc::context *ctxt) |
| : gimple_opt_pass (pass_data_omp_gcn, ctxt) |
| { |
| } |
| |
| /* opt_pass methods: */ |
| virtual bool gate (function *) |
| { |
| return flag_openmp; |
| } |
| |
| virtual unsigned int execute (function *) |
| { |
| return execute_omp_gcn (); |
| } |
| |
| }; /* class pass_omp_gcn. */ |
| |
| } /* anon namespace. */ |
| |
| gimple_opt_pass * |
| make_pass_omp_gcn (gcc::context *ctxt) |
| { |
| return new pass_omp_gcn (ctxt); |
| } |
| |
| /* }}} */ |
| /* {{{ OpenACC reductions. */ |
| |
| /* Global lock variable, needed for 128bit worker & gang reductions. */ |
| |
| static GTY(()) tree global_lock_var; |
| |
| /* Lazily generate the global_lock_var decl and return its address. */ |
| |
| static tree |
| gcn_global_lock_addr () |
| { |
| tree v = global_lock_var; |
| |
| if (!v) |
| { |
| tree name = get_identifier ("__reduction_lock"); |
| tree type = build_qualified_type (unsigned_type_node, |
| TYPE_QUAL_VOLATILE); |
| v = build_decl (BUILTINS_LOCATION, VAR_DECL, name, type); |
| global_lock_var = v; |
| DECL_ARTIFICIAL (v) = 1; |
| DECL_EXTERNAL (v) = 1; |
| TREE_STATIC (v) = 1; |
| TREE_PUBLIC (v) = 1; |
| TREE_USED (v) = 1; |
| mark_addressable (v); |
| mark_decl_referenced (v); |
| } |
| |
| return build_fold_addr_expr (v); |
| } |
| |
| /* Helper function for gcn_reduction_update. |
| |
| Insert code to locklessly update *PTR with *PTR OP VAR just before |
| GSI. We use a lockless scheme for nearly all case, which looks |
| like: |
| actual = initval (OP); |
| do { |
| guess = actual; |
| write = guess OP myval; |
| actual = cmp&swap (ptr, guess, write) |
| } while (actual bit-different-to guess); |
| return write; |
| |
| This relies on a cmp&swap instruction, which is available for 32- and |
| 64-bit types. Larger types must use a locking scheme. */ |
| |
| static tree |
| gcn_lockless_update (location_t loc, gimple_stmt_iterator *gsi, |
| tree ptr, tree var, tree_code op) |
| { |
| unsigned fn = GCN_BUILTIN_CMP_SWAP; |
| tree_code code = NOP_EXPR; |
| tree arg_type = unsigned_type_node; |
| tree var_type = TREE_TYPE (var); |
| |
| if (TREE_CODE (var_type) == COMPLEX_TYPE |
| || TREE_CODE (var_type) == REAL_TYPE) |
| code = VIEW_CONVERT_EXPR; |
| |
| if (TYPE_SIZE (var_type) == TYPE_SIZE (long_long_unsigned_type_node)) |
| { |
| arg_type = long_long_unsigned_type_node; |
| fn = GCN_BUILTIN_CMP_SWAPLL; |
| } |
| |
| tree swap_fn = gcn_builtin_decl (fn, true); |
| |
| gimple_seq init_seq = NULL; |
| tree init_var = make_ssa_name (arg_type); |
| tree init_expr = omp_reduction_init_op (loc, op, var_type); |
| init_expr = fold_build1 (code, arg_type, init_expr); |
| gimplify_assign (init_var, init_expr, &init_seq); |
| gimple *init_end = gimple_seq_last (init_seq); |
| |
| gsi_insert_seq_before (gsi, init_seq, GSI_SAME_STMT); |
| |
| /* Split the block just after the init stmts. */ |
| basic_block pre_bb = gsi_bb (*gsi); |
| edge pre_edge = split_block (pre_bb, init_end); |
| basic_block loop_bb = pre_edge->dest; |
| pre_bb = pre_edge->src; |
| /* Reset the iterator. */ |
| *gsi = gsi_for_stmt (gsi_stmt (*gsi)); |
| |
| tree expect_var = make_ssa_name (arg_type); |
| tree actual_var = make_ssa_name (arg_type); |
| tree write_var = make_ssa_name (arg_type); |
| |
| /* Build and insert the reduction calculation. */ |
| gimple_seq red_seq = NULL; |
| tree write_expr = fold_build1 (code, var_type, expect_var); |
| write_expr = fold_build2 (op, var_type, write_expr, var); |
| write_expr = fold_build1 (code, arg_type, write_expr); |
| gimplify_assign (write_var, write_expr, &red_seq); |
| |
| gsi_insert_seq_before (gsi, red_seq, GSI_SAME_STMT); |
| |
| /* Build & insert the cmp&swap sequence. */ |
| gimple_seq latch_seq = NULL; |
| tree swap_expr = build_call_expr_loc (loc, swap_fn, 3, |
| ptr, expect_var, write_var); |
| gimplify_assign (actual_var, swap_expr, &latch_seq); |
| |
| gcond *cond = gimple_build_cond (EQ_EXPR, actual_var, expect_var, |
| NULL_TREE, NULL_TREE); |
| gimple_seq_add_stmt (&latch_seq, cond); |
| |
| gimple *latch_end = gimple_seq_last (latch_seq); |
| gsi_insert_seq_before (gsi, latch_seq, GSI_SAME_STMT); |
| |
| /* Split the block just after the latch stmts. */ |
| edge post_edge = split_block (loop_bb, latch_end); |
| basic_block post_bb = post_edge->dest; |
| loop_bb = post_edge->src; |
| *gsi = gsi_for_stmt (gsi_stmt (*gsi)); |
| |
| post_edge->flags ^= EDGE_TRUE_VALUE | EDGE_FALLTHRU; |
| /* post_edge->probability = profile_probability::even (); */ |
| edge loop_edge = make_edge (loop_bb, loop_bb, EDGE_FALSE_VALUE); |
| /* loop_edge->probability = profile_probability::even (); */ |
| set_immediate_dominator (CDI_DOMINATORS, loop_bb, pre_bb); |
| set_immediate_dominator (CDI_DOMINATORS, post_bb, loop_bb); |
| |
| gphi *phi = create_phi_node (expect_var, loop_bb); |
| add_phi_arg (phi, init_var, pre_edge, loc); |
| add_phi_arg (phi, actual_var, loop_edge, loc); |
| |
| loop *loop = alloc_loop (); |
| loop->header = loop_bb; |
| loop->latch = loop_bb; |
| add_loop (loop, loop_bb->loop_father); |
| |
| return fold_build1 (code, var_type, write_var); |
| } |
| |
| /* Helper function for gcn_reduction_update. |
| |
| Insert code to lockfully update *PTR with *PTR OP VAR just before |
| GSI. This is necessary for types larger than 64 bits, where there |
| is no cmp&swap instruction to implement a lockless scheme. We use |
| a lock variable in global memory. |
| |
| while (cmp&swap (&lock_var, 0, 1)) |
| continue; |
| T accum = *ptr; |
| accum = accum OP var; |
| *ptr = accum; |
| cmp&swap (&lock_var, 1, 0); |
| return accum; |
| |
| A lock in global memory is necessary to force execution engine |
| descheduling and avoid resource starvation that can occur if the |
| lock is in shared memory. */ |
| |
| static tree |
| gcn_lockfull_update (location_t loc, gimple_stmt_iterator *gsi, |
| tree ptr, tree var, tree_code op) |
| { |
| tree var_type = TREE_TYPE (var); |
| tree swap_fn = gcn_builtin_decl (GCN_BUILTIN_CMP_SWAP, true); |
| tree uns_unlocked = build_int_cst (unsigned_type_node, 0); |
| tree uns_locked = build_int_cst (unsigned_type_node, 1); |
| |
| /* Split the block just before the gsi. Insert a gimple nop to make |
| this easier. */ |
| gimple *nop = gimple_build_nop (); |
| gsi_insert_before (gsi, nop, GSI_SAME_STMT); |
| basic_block entry_bb = gsi_bb (*gsi); |
| edge entry_edge = split_block (entry_bb, nop); |
| basic_block lock_bb = entry_edge->dest; |
| /* Reset the iterator. */ |
| *gsi = gsi_for_stmt (gsi_stmt (*gsi)); |
| |
| /* Build and insert the locking sequence. */ |
| gimple_seq lock_seq = NULL; |
| tree lock_var = make_ssa_name (unsigned_type_node); |
| tree lock_expr = gcn_global_lock_addr (); |
| lock_expr = build_call_expr_loc (loc, swap_fn, 3, lock_expr, |
| uns_unlocked, uns_locked); |
| gimplify_assign (lock_var, lock_expr, &lock_seq); |
| gcond *cond = gimple_build_cond (EQ_EXPR, lock_var, uns_unlocked, |
| NULL_TREE, NULL_TREE); |
| gimple_seq_add_stmt (&lock_seq, cond); |
| gimple *lock_end = gimple_seq_last (lock_seq); |
| gsi_insert_seq_before (gsi, lock_seq, GSI_SAME_STMT); |
| |
| /* Split the block just after the lock sequence. */ |
| edge locked_edge = split_block (lock_bb, lock_end); |
| basic_block update_bb = locked_edge->dest; |
| lock_bb = locked_edge->src; |
| *gsi = gsi_for_stmt (gsi_stmt (*gsi)); |
| |
| /* Create the lock loop. */ |
| locked_edge->flags ^= EDGE_TRUE_VALUE | EDGE_FALLTHRU; |
| locked_edge->probability = profile_probability::even (); |
| edge loop_edge = make_edge (lock_bb, lock_bb, EDGE_FALSE_VALUE); |
| loop_edge->probability = profile_probability::even (); |
| set_immediate_dominator (CDI_DOMINATORS, lock_bb, entry_bb); |
| set_immediate_dominator (CDI_DOMINATORS, update_bb, lock_bb); |
| |
| /* Create the loop structure. */ |
| loop *lock_loop = alloc_loop (); |
| lock_loop->header = lock_bb; |
| lock_loop->latch = lock_bb; |
| lock_loop->nb_iterations_estimate = 1; |
| lock_loop->any_estimate = true; |
| add_loop (lock_loop, entry_bb->loop_father); |
| |
| /* Build and insert the reduction calculation. */ |
| gimple_seq red_seq = NULL; |
| tree acc_in = make_ssa_name (var_type); |
| tree ref_in = build_simple_mem_ref (ptr); |
| TREE_THIS_VOLATILE (ref_in) = 1; |
| gimplify_assign (acc_in, ref_in, &red_seq); |
| |
| tree acc_out = make_ssa_name (var_type); |
| tree update_expr = fold_build2 (op, var_type, ref_in, var); |
| gimplify_assign (acc_out, update_expr, &red_seq); |
| |
| tree ref_out = build_simple_mem_ref (ptr); |
| TREE_THIS_VOLATILE (ref_out) = 1; |
| gimplify_assign (ref_out, acc_out, &red_seq); |
| |
| gsi_insert_seq_before (gsi, red_seq, GSI_SAME_STMT); |
| |
| /* Build & insert the unlock sequence. */ |
| gimple_seq unlock_seq = NULL; |
| tree unlock_expr = gcn_global_lock_addr (); |
| unlock_expr = build_call_expr_loc (loc, swap_fn, 3, unlock_expr, |
| uns_locked, uns_unlocked); |
| gimplify_and_add (unlock_expr, &unlock_seq); |
| gsi_insert_seq_before (gsi, unlock_seq, GSI_SAME_STMT); |
| |
| return acc_out; |
| } |
| |
| /* Emit a sequence to update a reduction accumulator at *PTR with the |
| value held in VAR using operator OP. Return the updated value. |
| |
| TODO: optimize for atomic ops and independent complex ops. */ |
| |
| static tree |
| gcn_reduction_update (location_t loc, gimple_stmt_iterator *gsi, |
| tree ptr, tree var, tree_code op) |
| { |
| tree type = TREE_TYPE (var); |
| tree size = TYPE_SIZE (type); |
| |
| if (size == TYPE_SIZE (unsigned_type_node) |
| || size == TYPE_SIZE (long_long_unsigned_type_node)) |
| return gcn_lockless_update (loc, gsi, ptr, var, op); |
| else |
| return gcn_lockfull_update (loc, gsi, ptr, var, op); |
| } |
| |
| /* Return a temporary variable decl to use for an OpenACC worker reduction. */ |
| |
| static tree |
| gcn_goacc_get_worker_red_decl (tree type, unsigned offset) |
| { |
| machine_function *machfun = cfun->machine; |
| tree existing_decl; |
| |
| if (TREE_CODE (type) == REFERENCE_TYPE) |
| type = TREE_TYPE (type); |
| |
| tree var_type |
| = build_qualified_type (type, |
| (TYPE_QUALS (type) |
| | ENCODE_QUAL_ADDR_SPACE (ADDR_SPACE_LDS))); |
| |
| if (machfun->reduc_decls |
| && offset < machfun->reduc_decls->length () |
| && (existing_decl = (*machfun->reduc_decls)[offset])) |
| { |
| gcc_assert (TREE_TYPE (existing_decl) == var_type); |
| return existing_decl; |
| } |
| else |
| { |
| char name[50]; |
| sprintf (name, ".oacc_reduction_%u", offset); |
| tree decl = create_tmp_var_raw (var_type, name); |
| |
| DECL_CONTEXT (decl) = NULL_TREE; |
| TREE_STATIC (decl) = 1; |
| |
| varpool_node::finalize_decl (decl); |
| |
| vec_safe_grow_cleared (machfun->reduc_decls, offset + 1); |
| (*machfun->reduc_decls)[offset] = decl; |
| |
| return decl; |
| } |
| |
| return NULL_TREE; |
| } |
| |
| /* Expand IFN_GOACC_REDUCTION_SETUP. */ |
| |
| static void |
| gcn_goacc_reduction_setup (gcall *call) |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (call); |
| tree lhs = gimple_call_lhs (call); |
| tree var = gimple_call_arg (call, 2); |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| gimple_seq seq = NULL; |
| |
| push_gimplify_context (true); |
| |
| if (level != GOMP_DIM_GANG) |
| { |
| /* Copy the receiver object. */ |
| tree ref_to_res = gimple_call_arg (call, 1); |
| |
| if (!integer_zerop (ref_to_res)) |
| var = build_simple_mem_ref (ref_to_res); |
| } |
| |
| if (level == GOMP_DIM_WORKER) |
| { |
| tree var_type = TREE_TYPE (var); |
| /* Store incoming value to worker reduction buffer. */ |
| tree offset = gimple_call_arg (call, 5); |
| tree decl |
| = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset)); |
| |
| gimplify_assign (decl, var, &seq); |
| } |
| |
| if (lhs) |
| gimplify_assign (lhs, var, &seq); |
| |
| pop_gimplify_context (NULL); |
| gsi_replace_with_seq (&gsi, seq, true); |
| } |
| |
| /* Expand IFN_GOACC_REDUCTION_INIT. */ |
| |
| static void |
| gcn_goacc_reduction_init (gcall *call) |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (call); |
| tree lhs = gimple_call_lhs (call); |
| tree var = gimple_call_arg (call, 2); |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| enum tree_code rcode |
| = (enum tree_code) TREE_INT_CST_LOW (gimple_call_arg (call, 4)); |
| tree init = omp_reduction_init_op (gimple_location (call), rcode, |
| TREE_TYPE (var)); |
| gimple_seq seq = NULL; |
| |
| push_gimplify_context (true); |
| |
| if (level == GOMP_DIM_GANG) |
| { |
| /* If there's no receiver object, propagate the incoming VAR. */ |
| tree ref_to_res = gimple_call_arg (call, 1); |
| if (integer_zerop (ref_to_res)) |
| init = var; |
| } |
| |
| if (lhs) |
| gimplify_assign (lhs, init, &seq); |
| |
| pop_gimplify_context (NULL); |
| gsi_replace_with_seq (&gsi, seq, true); |
| } |
| |
| /* Expand IFN_GOACC_REDUCTION_FINI. */ |
| |
| static void |
| gcn_goacc_reduction_fini (gcall *call) |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (call); |
| tree lhs = gimple_call_lhs (call); |
| tree ref_to_res = gimple_call_arg (call, 1); |
| tree var = gimple_call_arg (call, 2); |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| enum tree_code op |
| = (enum tree_code) TREE_INT_CST_LOW (gimple_call_arg (call, 4)); |
| gimple_seq seq = NULL; |
| tree r = NULL_TREE;; |
| |
| push_gimplify_context (true); |
| |
| tree accum = NULL_TREE; |
| |
| if (level == GOMP_DIM_WORKER) |
| { |
| tree var_type = TREE_TYPE (var); |
| tree offset = gimple_call_arg (call, 5); |
| tree decl |
| = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset)); |
| |
| accum = build_fold_addr_expr (decl); |
| } |
| else if (integer_zerop (ref_to_res)) |
| r = var; |
| else |
| accum = ref_to_res; |
| |
| if (accum) |
| { |
| /* UPDATE the accumulator. */ |
| gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT); |
| seq = NULL; |
| r = gcn_reduction_update (gimple_location (call), &gsi, accum, var, op); |
| } |
| |
| if (lhs) |
| gimplify_assign (lhs, r, &seq); |
| pop_gimplify_context (NULL); |
| |
| gsi_replace_with_seq (&gsi, seq, true); |
| } |
| |
| /* Expand IFN_GOACC_REDUCTION_TEARDOWN. */ |
| |
| static void |
| gcn_goacc_reduction_teardown (gcall *call) |
| { |
| gimple_stmt_iterator gsi = gsi_for_stmt (call); |
| tree lhs = gimple_call_lhs (call); |
| tree var = gimple_call_arg (call, 2); |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| gimple_seq seq = NULL; |
| |
| push_gimplify_context (true); |
| |
| if (level == GOMP_DIM_WORKER) |
| { |
| tree var_type = TREE_TYPE (var); |
| |
| /* Read the worker reduction buffer. */ |
| tree offset = gimple_call_arg (call, 5); |
| tree decl |
| = gcn_goacc_get_worker_red_decl (var_type, TREE_INT_CST_LOW (offset)); |
| var = decl; |
| } |
| |
| if (level != GOMP_DIM_GANG) |
| { |
| /* Write to the receiver object. */ |
| tree ref_to_res = gimple_call_arg (call, 1); |
| |
| if (!integer_zerop (ref_to_res)) |
| gimplify_assign (build_simple_mem_ref (ref_to_res), var, &seq); |
| } |
| |
| if (lhs) |
| gimplify_assign (lhs, var, &seq); |
| |
| pop_gimplify_context (NULL); |
| |
| gsi_replace_with_seq (&gsi, seq, true); |
| } |
| |
| /* Implement TARGET_GOACC_REDUCTION. |
| |
| Expand calls to the GOACC REDUCTION internal function, into a sequence of |
| gimple instructions. */ |
| |
| void |
| gcn_goacc_reduction (gcall *call) |
| { |
| int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3)); |
| |
| if (level == GOMP_DIM_VECTOR) |
| { |
| default_goacc_reduction (call); |
| return; |
| } |
| |
| unsigned code = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 0)); |
| |
| switch (code) |
| { |
| case IFN_GOACC_REDUCTION_SETUP: |
| gcn_goacc_reduction_setup (call); |
| break; |
| |
| case IFN_GOACC_REDUCTION_INIT: |
| gcn_goacc_reduction_init (call); |
| break; |
| |
| case IFN_GOACC_REDUCTION_FINI: |
| gcn_goacc_reduction_fini (call); |
| break; |
| |
| case IFN_GOACC_REDUCTION_TEARDOWN: |
| gcn_goacc_reduction_teardown (call); |
| break; |
| |
| default: |
| gcc_unreachable (); |
| } |
| } |
| |
| /* Implement TARGET_GOACC_ADJUST_PROPAGATION_RECORD. |
| |
| Tweak (worker) propagation record, e.g. to put it in shared memory. */ |
| |
| tree |
| gcn_goacc_adjust_propagation_record (tree record_type, bool sender, |
| const char *name) |
| { |
| tree type = record_type; |
| |
| TYPE_ADDR_SPACE (type) = ADDR_SPACE_LDS; |
| |
| if (!sender) |
| type = build_pointer_type (type); |
| |
| tree decl = create_tmp_var_raw (type, name); |
| |
| if (sender) |
| { |
| DECL_CONTEXT (decl) = NULL_TREE; |
| TREE_STATIC (decl) = 1; |
| } |
| |
| if (sender) |
| varpool_node::finalize_decl (decl); |
| |
| return decl; |
| } |
| |
| void |
| gcn_goacc_adjust_gangprivate_decl (tree var) |
| { |
| tree type = TREE_TYPE (var); |
| tree lds_type = build_qualified_type (type, |
| TYPE_QUALS_NO_ADDR_SPACE (type) |
| | ENCODE_QUAL_ADDR_SPACE (ADDR_SPACE_LDS)); |
| machine_function *machfun = cfun->machine; |
| |
| TREE_TYPE (var) = lds_type; |
| TREE_STATIC (var) = 1; |
| |
| /* We're making VAR static. We have to mangle the name to avoid collisions |
| between different local variables that share the same names. */ |
| lhd_set_decl_assembler_name (var); |
| |
| varpool_node::finalize_decl (var); |
| |
| if (machfun) |
| machfun->use_flat_addressing = true; |
| } |
| |
| /* }}} */ |