blob: 3d38b7f958732b7f8f58e78a9867fc143f75c40e [file]
/* Copyright (C) 2005-2026 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Offloading and Multi Processing Library
(libgomp).
Libgomp is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
/* This file handles the maintenance of threads in response to team
creation and termination. */
#include "libgomp.h"
#include "pool.h"
#include <stdlib.h>
#include <string.h>
static void
gomp_release_held_threads (struct gomp_thread_pool *pool,
struct gomp_team *team, unsigned team_id)
{
gomp_assert (team_id == 0, "releasing threads from non-primary thread %u",
team_id);
if (pool->prev_barrier)
{
struct gomp_team *saved_team __attribute__ ((unused))
= pool->last_team ? pool->last_team : team;
gomp_assert (pool->prev_barrier == &saved_team->barrier,
"prev_barrier not within cached team: %p != %p",
pool->prev_barrier, &saved_team->barrier);
gomp_team_barrier_done_final (pool->prev_barrier, team_id);
pool->prev_barrier = NULL;
}
}
#ifdef LIBGOMP_USE_PTHREADS
pthread_attr_t gomp_thread_attr;
/* This key is for the thread destructor. */
pthread_key_t gomp_thread_destructor;
/* This is the libgomp per-thread data structure. */
#if defined HAVE_TLS || defined USE_EMUTLS
__thread struct gomp_thread gomp_tls_data;
#else
pthread_key_t gomp_tls_key;
#endif
/* This structure is used to communicate across pthread_create. */
struct gomp_thread_start_data
{
struct gomp_team *team;
struct gomp_thread_pool *thread_pool;
unsigned int place;
unsigned team_id;
unsigned active_level;
unsigned place_partition_off;
unsigned place_partition_len;
bool nested;
pthread_t handle;
};
static inline void
gomp_init_team_state (struct gomp_team_state *ts, struct gomp_team *team,
unsigned team_id, unsigned active_level,
unsigned place_partition_off,
unsigned place_partition_len)
{
ts->team = team;
ts->work_share = &team->work_shares[0];
ts->last_work_share = NULL;
ts->team_id = team_id;
ts->level = team->prev_ts.level + 1;
ts->active_level = active_level;
ts->place_partition_off = place_partition_off;
ts->place_partition_len = place_partition_len;
ts->def_allocator = team->ti.def_allocator;
#ifdef HAVE_SYNC_BUILTINS
ts->single_count = 0;
#endif
ts->static_trip = 0;
}
/* This function is a pthread_create entry point. This contains the idle
loop in which a thread waits to be called up to become part of a team. */
static void *
gomp_thread_start (void *xdata)
{
struct gomp_thread_start_data *data = xdata;
struct gomp_thread *thr;
struct gomp_thread_pool *pool;
void (*local_fn) (void *);
void *local_data;
#if defined HAVE_TLS || defined USE_EMUTLS
thr = &gomp_tls_data;
#else
struct gomp_thread local_thr;
thr = &local_thr;
#endif
gomp_sem_init (&thr->release, 0);
/* Extract what we need from data. */
struct gomp_team *team = data->team;
gomp_init_team_state (&thr->ts, team, data->team_id, data->active_level,
data->place_partition_off, data->place_partition_len);
local_fn = team->ti.fn;
local_data = team->ti.data;
thr->num_teams = team->ti.num_teams;
thr->team_num = team->ti.team_num;
thr->thread_pool = data->thread_pool;
thr->place = data->place;
thr->exit = false;
#ifdef GOMP_NEEDS_THREAD_HANDLE
thr->handle = data->handle;
#endif
#if !(defined HAVE_TLS || defined USE_EMUTLS)
pthread_setspecific (gomp_tls_key, thr);
#endif
team->ordered_release[thr->ts.team_id] = &thr->release;
/* Make thread pool local. */
pool = thr->thread_pool;
if (data->nested)
{
struct gomp_team *team = thr->ts.team;
thr->task = &team->implicit_task[thr->ts.team_id];
gomp_init_task (thr->task, team->ti.task_parent_init,
&team->ti.task_icv_init);
thr->task->taskgroup = team->ti.task_taskgroup;
struct gomp_task *task = thr->task;
gomp_barrier_wait (&team->barrier, thr->ts.team_id);
local_fn (local_data);
gomp_team_barrier_wait_final (&team->barrier, thr->ts.team_id);
gomp_finish_task (task);
gomp_barrier_wait_last (&team->barrier, thr->ts.team_id);
}
else
{
pool->threads[thr->ts.team_id] = thr;
gomp_simple_barrier_wait (&pool->threads_dock);
/* Ensure we initialise the task *after* the wait on threads_dock just in
case affinity changes have meant that this new thread has the same
team_id as a thread used to have. In this case we want to ensure the
`gomp_finish_task` below is certain to happen before the
`gomp_init_task` here. */
thr->task = &team->implicit_task[thr->ts.team_id];
gomp_init_task (thr->task, team->ti.task_parent_init,
&team->ti.task_icv_init);
thr->task->taskgroup = team->ti.task_taskgroup;
do
{
struct gomp_team *team = thr->ts.team;
unsigned team_id = thr->ts.team_id;
struct gomp_task *task = thr->task;
thr->fn = NULL;
local_fn (local_data);
gomp_team_barrier_wait_final (&team->barrier, thr->ts.team_id);
gomp_finish_task (task);
/* Hold at the simple barrier if we're exiting.
Do this in order to synchronise with the primary thread that could
(after the next piece of work has been performed) free our team.
Need to ensure that the barrier waited on above is not freed until
we've completed it. With any threads that will be used in the
next iteration we know that it will synchronise with the primary
thread before it frees this team because it will synchronise with
the primary thread in the next iteration of this loop (in
`gomp_team_barrier_wait_final`).
Also have to hold here if we adjusted our team_id since the above
`task` location is possibly going to be used by a newly started
thread so we have to synchronise with those. Use boolean
short-circuit to avoid data-race on thr->ts.team in case where the
final barrier can't hold threads. */
if (!gomp_barrier_can_hold (&team->barrier)
|| thr->exit
|| (thr->ts.team == team && thr->ts.team_id != team_id))
gomp_simple_barrier_wait (&pool->threads_dock);
/* Early exit because if `thr->fn == NULL` then we can't guarantee
the `thr->ts.team` hasn't been freed by the primary thread racing
ahead. Hence we don't want to write to it. */
if (thr->exit)
break;
/* Usually `thr->fn` is not used or set. It is used when secondary
threads are "set going" without being assigned to any team.
See `gomp_pause_host` and `gomp_free_thread`.
In this case we don't want to bother reading the team
initialisation things because they've not been set up for us. */
if (__builtin_expect (thr->fn != NULL, 0))
{
local_fn = thr->fn;
local_data = thr->data;
continue;
}
team = thr->ts.team;
thr->ts.work_share = &team->work_shares[0];
thr->ts.last_work_share = NULL;
thr->ts.def_allocator = team->ti.def_allocator;
#ifdef HAVE_SYNC_BUILTINS
thr->ts.single_count = 0;
#endif
thr->ts.static_trip = 0;
local_fn = team->ti.fn;
local_data = team->ti.data;
thr->num_teams = team->ti.num_teams;
thr->team_num = team->ti.team_num;
thr->task = &team->implicit_task[thr->ts.team_id];
gomp_init_task (thr->task, team->ti.task_parent_init,
&team->ti.task_icv_init);
thr->task->taskgroup = team->ti.task_taskgroup;
}
while (true);
}
gomp_sem_destroy (&thr->release);
pthread_detach (pthread_self ());
thr->thread_pool = NULL;
thr->task = NULL;
return NULL;
}
#endif
static inline struct gomp_team *
get_last_team (unsigned nthreads)
{
struct gomp_thread *thr = gomp_thread ();
if (thr->ts.team == NULL)
{
gomp_assert (thr->ts.level == 0,
"looking for cached team in nested region %u thr->ts.level",
thr->ts.level);
struct gomp_thread_pool *pool = gomp_get_thread_pool (thr, nthreads);
struct gomp_team *last_team = pool->last_team;
if (last_team != NULL && last_team->nthreads == nthreads)
{
gomp_assert (!pool->prev_barrier
|| pool->prev_barrier == &pool->last_team->barrier,
"prev_barrier not within cached team: %p != %p",
pool->prev_barrier, &pool->last_team->barrier);
pool->last_team = NULL;
last_team->reused = true;
return last_team;
}
}
return NULL;
}
/* Create a new team data structure. */
struct gomp_team *
gomp_new_team (unsigned nthreads)
{
struct gomp_team *team;
int i;
team = get_last_team (nthreads);
if (team == NULL)
{
void *barrier_extra;
struct gomp_barrier_extra_alloc_needs barrier_extra_alloc
= gomp_barrier_extra_alloc (nthreads);
size_t extra = sizeof (team->ordered_release[0])
+ sizeof (team->implicit_task[0]);
size_t team_extra = nthreads * extra;
size_t barrier_padding = 0;
if (barrier_extra_alloc.size)
{
/* `offsetof` in case the structure is padded and `implicit_task` is
not at the end of `sizeof (*team)`. */
size_t barrier_offset
= offsetof (struct gomp_team, implicit_task) + team_extra;
gomp_assert (barrier_extra_alloc.align
<= __alignof (struct gomp_team),
"barrier extra alignment too large: %lu > %lu",
(unsigned long) barrier_extra_alloc.align,
(unsigned long) __alignof (struct gomp_team));
barrier_padding = ((barrier_offset + barrier_extra_alloc.align - 1)
& ~(barrier_extra_alloc.align - 1))
- barrier_offset;
}
size_t alloc_extra
= team_extra + barrier_padding + barrier_extra_alloc.size;
#ifdef GOMP_USE_ALIGNED_WORK_SHARES
team = gomp_aligned_alloc (__alignof (struct gomp_team),
sizeof (*team) + alloc_extra);
#else
team = team_malloc (sizeof (*team) + alloc_extra);
#endif
#ifndef HAVE_SYNC_BUILTINS
gomp_mutex_init (&team->work_share_list_free_lock);
#endif
barrier_extra = (char *) &team->implicit_task[nthreads]
+ nthreads * sizeof (team->ordered_release[0])
+ barrier_padding;
gomp_barrier_init (&team->barrier, nthreads, barrier_extra);
gomp_mutex_init (&team->task_lock);
team->nthreads = nthreads;
team->reused = false;
}
team->work_share_chunk = 8;
#ifdef HAVE_SYNC_BUILTINS
team->single_count = 0;
#endif
team->work_shares_to_free = &team->work_shares[0];
gomp_init_work_share (&team->work_shares[0], 0, nthreads);
team->work_shares[0].next_alloc = NULL;
team->work_share_list_free = NULL;
team->work_share_list_alloc = &team->work_shares[1];
for (i = 1; i < 7; i++)
team->work_shares[i].next_free = &team->work_shares[i + 1];
team->work_shares[i].next_free = NULL;
gomp_sem_init (&team->master_release, 0);
team->ordered_release = (void *) &team->implicit_task[nthreads];
team->ordered_release[0] = &team->master_release;
priority_queue_init (&team->task_queue);
team->task_count = 0;
team->task_queued_count = 0;
team->task_running_count = 0;
team->work_share_cancelled = 0;
team->team_cancelled = 0;
team->task_detach_count = 0;
return team;
}
/* Free a team data structure. */
static void
free_team (struct gomp_team *team)
{
#ifndef HAVE_SYNC_BUILTINS
gomp_mutex_destroy (&team->work_share_list_free_lock);
#endif
gomp_barrier_destroy (&team->barrier);
gomp_mutex_destroy (&team->task_lock);
priority_queue_free (&team->task_queue);
team_free (team);
}
static void
gomp_free_pool_helper (void *thread_pool)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_thread_pool *pool
= (struct gomp_thread_pool *) thread_pool;
gomp_simple_barrier_wait_last (&pool->threads_dock);
gomp_sem_destroy (&thr->release);
thr->thread_pool = NULL;
thr->task = NULL;
#ifdef LIBGOMP_USE_PTHREADS
pthread_detach (pthread_self ());
pthread_exit (NULL);
#elif defined(__nvptx__)
asm ("exit;");
#elif defined(__AMDGCN__)
asm ("s_endpgm");
#else
#error gomp_free_pool_helper must terminate the thread
#endif
}
/* Free a thread pool and release its threads. */
void
gomp_free_thread (void *arg __attribute__((unused)))
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_thread_pool *pool = thr->thread_pool;
if (pool)
{
if (pool->threads_used > 0)
{
int i;
for (i = 1; i < pool->threads_used; i++)
{
struct gomp_thread *nthr = pool->threads[i];
nthr->fn = gomp_free_pool_helper;
nthr->data = pool;
}
/* This barrier undocks threads docked on pool->threads_dock. */
gomp_release_held_threads (pool, NULL, thr->ts.team_id);
if (!gomp_barrier_can_hold (pool->prev_barrier))
gomp_simple_barrier_wait (&pool->threads_dock);
/* And this waits till all threads have called
gomp_simple_barrier_wait_last in gomp_free_pool_helper. */
gomp_simple_barrier_wait (&pool->threads_dock);
/* Now it is safe to destroy the barrier and free the pool. */
gomp_simple_barrier_destroy (&pool->threads_dock);
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_managed_threads,
1L - pool->threads_used);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
gomp_managed_threads -= pool->threads_used - 1L;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
}
if (pool->last_team)
free_team (pool->last_team);
#ifndef __nvptx__
team_free (pool->threads);
team_free (pool);
#endif
thr->thread_pool = NULL;
}
if (thr->ts.level == 0 && __builtin_expect (thr->ts.team != NULL, 0))
gomp_team_end ();
if (thr->task != NULL)
{
struct gomp_task *task = thr->task;
gomp_end_task ();
free (task);
}
}
/* Launch a team. */
#ifdef LIBGOMP_USE_PTHREADS
static unsigned
gomp_barrier_wait_n (unsigned num_new_threads, unsigned num_exiting_threads,
unsigned num_remapped_threads, unsigned old_threads_used,
unsigned nthreads, unsigned affinity_count, bool can_hold)
{
if (can_hold)
{
unsigned tmp
= num_new_threads + num_exiting_threads + num_remapped_threads;
return tmp ? tmp + 1 : 0;
}
if (!affinity_count)
return old_threads_used > nthreads ? old_threads_used : nthreads;
return nthreads + affinity_count;
}
void
gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
unsigned flags, struct gomp_team *team,
struct gomp_taskgroup *taskgroup)
{
struct gomp_thread_start_data *start_data = NULL;
struct gomp_thread *thr, *nthr;
struct gomp_task *task;
struct gomp_task_icv *icv;
bool nested;
struct gomp_thread_pool *pool;
unsigned i, n, old_threads_used = 0;
unsigned simple_barrier_n = 0;
unsigned num_new_threads = 0;
unsigned num_remapped_threads = 0;
unsigned spawned_new_threads __attribute__ ((unused)) = 0;
unsigned num_exiting_threads = 0;
pthread_attr_t thread_attr, *attr;
unsigned long nthreads_var;
char bind, bind_var;
unsigned int s = 0, rest = 0, p = 0, k = 0;
unsigned int affinity_count = 0;
struct gomp_thread **affinity_thr = NULL;
bool force_display = false;
thr = gomp_thread ();
nested = thr->ts.level;
pool = thr->thread_pool;
task = thr->task;
icv = task ? &task->icv : &gomp_global_icv;
if (__builtin_expect (gomp_places_list != NULL, 0) && thr->place == 0)
{
gomp_init_affinity ();
if (__builtin_expect (gomp_display_affinity_var, 0) && nthreads == 1)
gomp_display_affinity_thread (gomp_thread_self (), &thr->ts,
thr->place);
}
/* Always save the previous state, even if this isn't a nested team.
In particular, we should save any work share state from an outer
orphaned work share construct. */
team->prev_ts = thr->ts;
thr->ts.team = team;
thr->ts.team_id = 0;
++thr->ts.level;
if (nthreads > 1)
++thr->ts.active_level;
thr->ts.work_share = &team->work_shares[0];
thr->ts.last_work_share = NULL;
#ifdef HAVE_SYNC_BUILTINS
thr->ts.single_count = 0;
#endif
thr->ts.static_trip = 0;
thr->task = &team->implicit_task[0];
#ifdef GOMP_NEEDS_THREAD_HANDLE
thr->handle = pthread_self ();
#endif
nthreads_var = icv->nthreads_var;
if (__builtin_expect (gomp_nthreads_var_list != NULL, 0)
&& thr->ts.level < gomp_nthreads_var_list_len)
nthreads_var = gomp_nthreads_var_list[thr->ts.level];
bind_var = icv->bind_var;
if (bind_var != omp_proc_bind_false && (flags & 7) != omp_proc_bind_false)
bind_var = flags & 7;
bind = bind_var;
if (__builtin_expect (gomp_bind_var_list != NULL, 0)
&& thr->ts.level < gomp_bind_var_list_len)
bind_var = gomp_bind_var_list[thr->ts.level];
gomp_init_task (thr->task, task, icv);
thr->task->taskgroup = taskgroup;
team->implicit_task[0].icv.nthreads_var = nthreads_var;
team->implicit_task[0].icv.bind_var = bind_var;
if (nthreads == 1)
return;
i = 1;
if (__builtin_expect (gomp_places_list != NULL, 0))
{
/* Depending on chosen proc_bind model, set subpartition
for the master thread and initialize helper variables
P and optionally S, K and/or REST used by later place
computation for each additional thread. */
p = thr->place - 1;
switch (bind)
{
case omp_proc_bind_true:
case omp_proc_bind_close:
if (nthreads > thr->ts.place_partition_len)
{
/* T > P. S threads will be placed in each place,
and the final REM threads placed one by one
into the already occupied places. */
s = nthreads / thr->ts.place_partition_len;
rest = nthreads % thr->ts.place_partition_len;
}
else
s = 1;
k = 1;
break;
case omp_proc_bind_master:
/* Each thread will be bound to master's place. */
break;
case omp_proc_bind_spread:
if (nthreads <= thr->ts.place_partition_len)
{
/* T <= P. Each subpartition will have in between s
and s+1 places (subpartitions starting at or
after rest will have s places, earlier s+1 places),
each thread will be bound to the first place in
its subpartition (except for the master thread
that can be bound to another place in its
subpartition). */
s = thr->ts.place_partition_len / nthreads;
rest = thr->ts.place_partition_len % nthreads;
rest = (s + 1) * rest + thr->ts.place_partition_off;
if (p < rest)
{
p -= (p - thr->ts.place_partition_off) % (s + 1);
thr->ts.place_partition_len = s + 1;
}
else
{
p -= (p - rest) % s;
thr->ts.place_partition_len = s;
}
thr->ts.place_partition_off = p;
}
else
{
/* T > P. Each subpartition will have just a single
place and we'll place between s and s+1
threads into each subpartition. */
s = nthreads / thr->ts.place_partition_len;
rest = nthreads % thr->ts.place_partition_len;
thr->ts.place_partition_off = p;
thr->ts.place_partition_len = 1;
k = 1;
}
break;
}
}
else
bind = omp_proc_bind_false;
/* Copy entirely rather than copy pointer so that we have an immutable
copy for the secondary threads to take from for the time-frame that
said secondary threads are executing on. Can't copy from
`team->implicit_task[0].icv` because primary thread could have adjusted
its per-task ICV by the time the secondary thread gets around to
initialising things. */
team->ti.task_icv_init = team->implicit_task[0].icv;
team->ti.task_parent_init = task;
team->ti.task_taskgroup = taskgroup;
team->ti.num_teams = thr->num_teams;
team->ti.team_num = thr->team_num;
team->ti.fn = fn;
team->ti.data = data;
team->ti.def_allocator = thr->ts.def_allocator;
/* We only allow the reuse of idle threads for non-nested PARALLEL
regions. This appears to be implied by the semantics of
threadprivate variables, but perhaps that's reading too much into
things. Certainly it does prevent any locking problems, since
only the initial program thread will modify gomp_threads. */
if (!nested)
{
gomp_assert (team->prev_ts.team_id == 0,
"starting a team from thread with id %u in previous team\n",
team->prev_ts.team_id);
old_threads_used = pool->threads_used;
if (team->reused
&& team->ppv.place_partition_off == thr->ts.place_partition_off
&& team->ppv.place_partition_len == thr->ts.place_partition_len
&& team->ppv.bind == bind)
{
gomp_assert (nthreads == old_threads_used,
"odd number of threads when re-using team: "
"nthreads = %u old_threads_used = %u\n",
nthreads, old_threads_used);
goto do_release;
}
else
{
team->ppv.place_partition_off = thr->ts.place_partition_off;
team->ppv.place_partition_len = thr->ts.place_partition_len;
team->ppv.bind = bind;
}
if (nthreads == old_threads_used)
{
n = nthreads;
num_new_threads = 0;
num_exiting_threads = 0;
simple_barrier_n
= gomp_barrier_wait_n (num_new_threads, num_exiting_threads,
num_remapped_threads, old_threads_used,
nthreads, affinity_count,
gomp_barrier_can_hold (pool->prev_barrier));
if (gomp_barrier_can_hold (pool->prev_barrier))
gomp_simple_barrier_reinit (&pool->threads_dock, simple_barrier_n);
}
else if (nthreads < old_threads_used)
{
n = nthreads;
num_new_threads = 0;
num_exiting_threads = old_threads_used - nthreads;
simple_barrier_n
= gomp_barrier_wait_n (num_new_threads, num_exiting_threads,
num_remapped_threads, old_threads_used,
nthreads, affinity_count,
gomp_barrier_can_hold (pool->prev_barrier));
if (gomp_barrier_can_hold (pool->prev_barrier))
gomp_simple_barrier_reinit (&pool->threads_dock, simple_barrier_n);
}
else if (old_threads_used == 0)
{
n = 0;
num_new_threads = nthreads - 1;
num_exiting_threads = 0;
simple_barrier_n = nthreads;
gomp_simple_barrier_init (&pool->threads_dock, nthreads);
}
else
{
n = old_threads_used;
num_new_threads = nthreads - old_threads_used;
num_exiting_threads = 0;
simple_barrier_n
= gomp_barrier_wait_n (num_new_threads, num_exiting_threads,
num_remapped_threads, old_threads_used,
nthreads, affinity_count,
gomp_barrier_can_hold (pool->prev_barrier));
/* Increase the barrier threshold to make sure all new
threads arrive before the team is released. */
gomp_simple_barrier_reinit (&pool->threads_dock, simple_barrier_n);
}
/* Not true yet, but soon will be. We're going to release all
threads from the dock, and those that aren't part of the
team will exit. */
pool->threads_used = nthreads;
/* If necessary, expand the size of the gomp_threads array. It is
expected that changes in the number of threads are rare, thus we
make no effort to expand gomp_threads_size geometrically. */
if (nthreads >= pool->threads_size)
{
pool->threads_size = nthreads + 1;
pool->threads
= gomp_realloc (pool->threads,
pool->threads_size
* sizeof (struct gomp_thread *));
/* Add current (master) thread to threads[]. */
pool->threads[0] = thr;
}
/* Release existing idle threads. */
for (; i < n; ++i)
{
unsigned int place_partition_off = thr->ts.place_partition_off;
unsigned int place_partition_len = thr->ts.place_partition_len;
unsigned int place = 0;
if (__builtin_expect (gomp_places_list != NULL, 0))
{
switch (bind)
{
case omp_proc_bind_true:
case omp_proc_bind_close:
if (k == s)
{
++p;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
k = 1;
if (i == nthreads - rest)
s = 1;
}
else
++k;
break;
case omp_proc_bind_master:
break;
case omp_proc_bind_spread:
if (k == 0)
{
/* T <= P. */
if (p < rest)
p += s + 1;
else
p += s;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
place_partition_off = p;
if (p < rest)
place_partition_len = s + 1;
else
place_partition_len = s;
}
else
{
/* T > P. */
if (k == s)
{
++p;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
k = 1;
if (i == nthreads - rest)
s = 1;
}
else
++k;
place_partition_off = p;
place_partition_len = 1;
}
break;
}
if (affinity_thr != NULL
|| (bind != omp_proc_bind_true
&& pool->threads[i]->place != p + 1)
|| pool->threads[i]->place <= place_partition_off
|| pool->threads[i]->place > (place_partition_off
+ place_partition_len))
{
unsigned int l;
force_display = true;
if (affinity_thr == NULL)
{
unsigned int j;
if (team->prev_ts.place_partition_len > 64)
affinity_thr
= gomp_malloc (team->prev_ts.place_partition_len
* sizeof (struct gomp_thread *));
else
affinity_thr
= gomp_alloca (team->prev_ts.place_partition_len
* sizeof (struct gomp_thread *));
memset (affinity_thr, '\0',
team->prev_ts.place_partition_len
* sizeof (struct gomp_thread *));
for (j = i; j < old_threads_used; j++)
{
if (pool->threads[j]->place
> team->prev_ts.place_partition_off
&& (pool->threads[j]->place
<= (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len)))
{
l = pool->threads[j]->place - 1
- team->prev_ts.place_partition_off;
pool->threads[j]->data = affinity_thr[l];
affinity_thr[l] = pool->threads[j];
}
else
pool->threads[j]->exit = true;
pool->threads[j] = NULL;
}
if (nthreads > old_threads_used)
memset (&pool->threads[old_threads_used],
'\0', ((nthreads - old_threads_used)
* sizeof (struct gomp_thread *)));
n = nthreads;
affinity_count = old_threads_used - i;
}
if (affinity_count == 0)
break;
l = p;
if (affinity_thr[l - team->prev_ts.place_partition_off]
== NULL)
{
if (bind != omp_proc_bind_true)
continue;
for (l = place_partition_off;
l < place_partition_off + place_partition_len;
l++)
if (affinity_thr[l - team->prev_ts.place_partition_off]
!= NULL)
break;
if (l == place_partition_off + place_partition_len)
continue;
}
nthr = affinity_thr[l - team->prev_ts.place_partition_off];
affinity_thr[l - team->prev_ts.place_partition_off]
= (struct gomp_thread *) nthr->data;
affinity_count--;
pool->threads[i] = nthr;
}
else
nthr = pool->threads[i];
place = p + 1;
}
else
nthr = pool->threads[i];
__atomic_store_n (&nthr->ts.team, team, MEMMODEL_RELEASE);
if (team->reused && nthr->ts.team_id != i)
num_remapped_threads++;
nthr->ts.team_id = i;
gomp_assert (nthr->ts.level == 1 && team->prev_ts.level == 0,
"level indicators surprising: nthr: %u, team->prev_ts: "
"%u\n", nthr->ts.level, team->prev_ts.level);
gomp_assert (nthr->ts.active_level == 1 && thr->ts.active_level == 1,
"active_level indicators surprising: nthr: %u, thr: "
"%u\n", nthr->ts.active_level, thr->ts.active_level);
nthr->ts.place_partition_off = place_partition_off;
nthr->ts.place_partition_len = place_partition_len;
nthr->place = place;
team->ordered_release[i] = &nthr->release;
}
if (__builtin_expect (affinity_thr != NULL, 0))
{
/* Mark each of the threads that are exiting as such. Hence in
`gomp_thread_start` they exit their main loop. */
for (unsigned l = 0; l < team->prev_ts.place_partition_len; l++)
{
struct gomp_thread *t = affinity_thr[l];
while (t)
{
struct gomp_thread *next = (struct gomp_thread *) t->data;
t->exit = true;
t = next;
}
}
/* If AFFINITY_THR is non-NULL just because we had to
permute some threads in the pool, but we've managed
to find exactly as many old threads as we'd find
without affinity, we don't need to handle this
specially anymore. */
if (nthreads <= old_threads_used
? (affinity_count == old_threads_used - nthreads)
: (i == old_threads_used))
{
if (team->prev_ts.place_partition_len > 64)
free (affinity_thr);
affinity_thr = NULL;
affinity_count = 0;
}
else
{
i = 1;
/* We are going to compute the places/subpartitions
again from the beginning. So, we need to reinitialize
vars modified by the switch (bind) above inside
of the loop, to the state they had after the initial
switch (bind). */
switch (bind)
{
case omp_proc_bind_true:
case omp_proc_bind_close:
if (nthreads > thr->ts.place_partition_len)
/* T > P. S has been changed, so needs
to be recomputed. */
s = nthreads / thr->ts.place_partition_len;
k = 1;
p = thr->place - 1;
break;
case omp_proc_bind_master:
/* No vars have been changed. */
break;
case omp_proc_bind_spread:
p = thr->ts.place_partition_off;
if (k != 0)
{
/* T > P. */
s = nthreads / team->prev_ts.place_partition_len;
k = 1;
}
break;
}
/* Increase the barrier threshold to make sure all new
threads and all the threads we're going to let die
arrive before the team is released. */
if (affinity_count)
{
/* Total number of threads running is:
nthreads + affinity_count
Equation to be satisfied is:
(nthreads + affinity_count ==
old_threads_used + num_new_threads).
So num_new_threads
== (nthreads + affinity_count - old_threads_used)
Originally
num_new_threads == nthreads > old_threads_used
? (nthreads - old_threads_used)
: 0;
Now:
nthreads + num_exiting_threads
== old_threads_used + num_new_threads
So:
num_exiting_threads
== old_threads_used + num_new_threads - nthreads
== affinity_count
*/
gomp_assert (nthreads + affinity_count > old_threads_used,
"not spawning more threads but "
"reinitialising barrier "
"nthreads = %u"
", affinity_count = %u"
", old_threads_used = %u",
nthreads, affinity_count, old_threads_used);
num_new_threads
= (nthreads + affinity_count) - old_threads_used;
num_exiting_threads = affinity_count;
}
}
}
else
for (unsigned j = n; j < old_threads_used; j++)
pool->threads[j]->exit = true;
/* Account for any affinity changes & remapped threads. */
simple_barrier_n
= gomp_barrier_wait_n (num_new_threads, num_exiting_threads,
num_remapped_threads, old_threads_used, nthreads,
affinity_count,
gomp_barrier_can_hold (pool->prev_barrier));
gomp_simple_barrier_reinit (&pool->threads_dock, simple_barrier_n);
if (i == nthreads)
{
gomp_assert (num_new_threads == 0,
"calculated need %u new threads but spawning none",
num_new_threads);
gomp_assert (gomp_barrier_can_hold (pool->prev_barrier)
? (simple_barrier_n
== ((num_exiting_threads || num_remapped_threads)
? (num_exiting_threads + num_remapped_threads
+ 1)
: 0))
: simple_barrier_n == old_threads_used,
"have calculated need simple_barrier_n == %u"
" starting 0 threads, affinity_count == %u, "
"old_threads_used == %u, calculated "
"num_exiting_threads == %u, nthreads == %u, "
"num_remapped_threads == %u",
simple_barrier_n, affinity_count, old_threads_used,
num_exiting_threads, nthreads, num_remapped_threads);
goto do_release;
}
}
if (__builtin_expect (nthreads + affinity_count > old_threads_used, 0))
{
long diff = (long) (nthreads + affinity_count) - (long) old_threads_used;
if (old_threads_used == 0)
--diff;
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_managed_threads, diff);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
gomp_managed_threads += diff;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
}
attr = &gomp_thread_attr;
if (__builtin_expect (gomp_places_list != NULL, 0))
{
size_t stacksize;
pthread_attr_init (&thread_attr);
if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
pthread_attr_setstacksize (&thread_attr, stacksize);
attr = &thread_attr;
}
if (i >= nthreads)
__builtin_unreachable ();
start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
* (nthreads - i));
/* Launch new threads. */
for (; i < nthreads; ++i)
{
int err;
start_data->place_partition_off = thr->ts.place_partition_off;
start_data->place_partition_len = thr->ts.place_partition_len;
start_data->place = 0;
if (__builtin_expect (gomp_places_list != NULL, 0))
{
switch (bind)
{
case omp_proc_bind_true:
case omp_proc_bind_close:
if (k == s)
{
++p;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
k = 1;
if (i == nthreads - rest)
s = 1;
}
else
++k;
break;
case omp_proc_bind_master:
break;
case omp_proc_bind_spread:
if (k == 0)
{
/* T <= P. */
if (p < rest)
p += s + 1;
else
p += s;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
start_data->place_partition_off = p;
if (p < rest)
start_data->place_partition_len = s + 1;
else
start_data->place_partition_len = s;
}
else
{
/* T > P. */
if (k == s)
{
++p;
if (p == (team->prev_ts.place_partition_off
+ team->prev_ts.place_partition_len))
p = team->prev_ts.place_partition_off;
k = 1;
if (i == nthreads - rest)
s = 1;
}
else
++k;
start_data->place_partition_off = p;
start_data->place_partition_len = 1;
}
break;
}
start_data->place = p + 1;
if (affinity_thr != NULL && pool->threads[i] != NULL)
continue;
gomp_init_thread_affinity (attr, p);
}
start_data->team = team;
start_data->team_id = i;
start_data->active_level = thr->ts.active_level;
start_data->thread_pool = pool;
start_data->nested = nested;
attr = gomp_adjust_thread_attr (attr, &thread_attr);
err = pthread_create (&start_data->handle, attr, gomp_thread_start,
start_data);
spawned_new_threads += 1;
start_data++;
if (err != 0)
gomp_fatal ("Thread creation failed: %s", strerror (err));
}
if (!nested)
{
gomp_assert (spawned_new_threads == num_new_threads,
"calculated num_new_threads != spawned_new_threads: "
" %u != %u",
num_new_threads, spawned_new_threads);
bool calc_check __attribute__ ((unused)) = false;
if (old_threads_used == 0)
calc_check = (num_new_threads == (simple_barrier_n - 1));
else if (gomp_barrier_can_hold (pool->prev_barrier))
calc_check
= ((num_new_threads + num_exiting_threads + num_remapped_threads)
== (simple_barrier_n - 1));
else
calc_check = (num_new_threads == (simple_barrier_n - old_threads_used));
gomp_assert (calc_check,
"simple barrier calculation incorrect for %s team: "
"simple_barrier_n: %u"
", nthreads: %u"
", num_new_threads: %u"
", num_exiting_threads: %u"
", num_remapped_threads: %u"
", old_threads_used: %u",
nested ? "nested" : "top-level", simple_barrier_n, nthreads,
num_new_threads, num_exiting_threads, num_remapped_threads,
old_threads_used);
}
if (__builtin_expect (attr == &thread_attr, 0))
pthread_attr_destroy (&thread_attr);
do_release:
if (nested)
gomp_barrier_wait (&team->barrier, thr->ts.team_id);
else
{
gomp_release_held_threads (pool, team, thr->ts.team_id);
if (num_new_threads
|| num_exiting_threads
|| num_remapped_threads
|| !gomp_barrier_can_hold (pool->prev_barrier))
gomp_simple_barrier_wait (&pool->threads_dock);
}
/* Decrease the barrier threshold to match the number of threads
that should arrive back at the end of this team. The extra
threads should be exiting. Note that we arrange for this test
to never be true for nested teams. If AFFINITY_COUNT is non-zero,
the barrier as well as gomp_managed_threads was temporarily
set to NTHREADS + AFFINITY_COUNT. For NTHREADS < OLD_THREADS_COUNT,
AFFINITY_COUNT if non-zero will be always at least
OLD_THREADS_COUNT - NTHREADS. */
if (__builtin_expect (nthreads < old_threads_used, 0)
|| __builtin_expect (affinity_count, 0))
{
long diff = (long) nthreads - (long) old_threads_used;
if (affinity_count)
diff = -affinity_count;
gomp_simple_barrier_reinit (&pool->threads_dock, nthreads);
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_managed_threads, diff);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
gomp_managed_threads += diff;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
}
else if (!nested && gomp_barrier_can_hold (pool->prev_barrier))
{
/* Need to adjust the simple barrier threshold once it's been used in
order to account for all threads that would need to be waited on in
something like `gomp_free_thread` or `gomp_pause_thread`. */
gomp_simple_barrier_reinit (&pool->threads_dock, nthreads);
}
if (__builtin_expect (gomp_display_affinity_var, 0))
{
if (nested
|| nthreads != old_threads_used
|| force_display)
{
gomp_display_affinity_thread (gomp_thread_self (), &thr->ts,
thr->place);
if (nested)
{
start_data -= nthreads - 1;
struct gomp_team_state ts;
for (i = 1; i < nthreads; ++i)
{
gomp_init_team_state (&ts, team, start_data->team_id,
start_data->active_level,
start_data->place_partition_off,
start_data->place_partition_len);
gomp_display_affinity_thread (
#ifdef LIBGOMP_USE_PTHREADS
start_data->handle,
#else
gomp_thread_self (),
#endif
&ts, start_data->place);
start_data++;
}
}
else
{
for (i = 1; i < nthreads; ++i)
{
gomp_thread_handle handle
= gomp_thread_to_pthread_t (pool->threads[i]);
gomp_display_affinity_thread (handle, &pool->threads[i]->ts,
pool->threads[i]->place);
}
}
}
}
if (__builtin_expect (affinity_thr != NULL, 0)
&& team->prev_ts.place_partition_len > 64)
free (affinity_thr);
}
#endif
/* Terminate the current team. This is only to be called by the master
thread. We assume that we must wait for the other threads. */
void
gomp_team_end (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
unsigned team_id = thr->ts.team_id;
/* This barrier handles all pending explicit threads.
As #pragma omp cancel parallel might get awaited count in
team->barrier in a inconsistent state, we need to use a different
counter here. */
gomp_team_barrier_wait_final (&team->barrier, team_id);
/* After this point all tasking has finished. Just haven't told all
secondary threads that they're good to continue. */
if (__builtin_expect (team->team_cancelled, 0))
{
struct gomp_work_share *ws = team->work_shares_to_free;
do
{
struct gomp_work_share *next_ws = gomp_ptrlock_get (&ws->next_ws);
if (next_ws == NULL)
gomp_ptrlock_set (&ws->next_ws, ws);
gomp_fini_work_share (ws);
ws = next_ws;
}
while (ws != NULL);
}
else
gomp_fini_work_share (thr->ts.work_share);
gomp_end_task ();
thr->ts = team->prev_ts;
bool finished_threads __attribute__ ((unused)) = false;
if (__builtin_expect (thr->ts.level != 0, 0))
{
gomp_team_barrier_done_final (&team->barrier, team_id);
finished_threads = true;
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_managed_threads, 1L - team->nthreads);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
gomp_managed_threads -= team->nthreads - 1L;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
/* This barrier has gomp_barrier_wait_last counterparts
and ensures the team can be safely destroyed. */
gomp_barrier_wait (&team->barrier, team_id);
}
if (__builtin_expect (team->work_shares[0].next_alloc != NULL, 0))
{
struct gomp_work_share *ws = team->work_shares[0].next_alloc;
do
{
struct gomp_work_share *next_ws = ws->next_alloc;
free (ws);
ws = next_ws;
}
while (ws != NULL);
}
gomp_sem_destroy (&team->master_release);
if (__builtin_expect (thr->ts.team != NULL
&& (thr->ts.team->nthreads != 1
|| thr->ts.level != 0),
0)
|| __builtin_expect (team->nthreads == 1, 0))
{
gomp_assert (finished_threads || team->nthreads == 1,
"freeing team while threads may be waiting on it."
" team = %p, level = %u",
team, thr->ts.level + 1);
free_team (team);
}
else
{
struct gomp_thread_pool *pool = thr->thread_pool;
if (pool->last_team)
free_team (pool->last_team);
pool->last_team = team;
gomp_assert (!finished_threads,
"have let threads go while still recording prev_barrier"
" level = %u, team = %p, nthreads = %u",
thr->ts.level + 1, team, team->nthreads);
gomp_assert (!pool->prev_barrier,
"previous barrier not released before it gets overridden"
" level = %u, team = %p, nthreads = %u",
thr->ts.level + 1, team, team->nthreads);
/* Only record `prev_barrier` when `gomp_barrier_can_hold`:
- It's only used when gomp_barrier_can_hold.
- Some targets have custom versions of `gomp_team_start` that doesn't
call `gomp_release_held_threads` and hence don't clear this value.
*/
if (gomp_barrier_can_hold (&team->barrier))
pool->prev_barrier = &team->barrier;
/* Releasing pool should only happen if we're going to then
`gomp_get_thread_pool` later. When `thr->ts.team != NULL` we don't
fetch the last team and call `gomp_get_thread_pool`. Hence we don't
want to release the pool here. */
if (thr->ts.team == NULL)
gomp_release_thread_pool (pool);
}
}
#ifdef LIBGOMP_USE_PTHREADS
/* Constructors for this file. */
static void __attribute__((constructor))
initialize_team (void)
{
#if !defined HAVE_TLS && !defined USE_EMUTLS
static struct gomp_thread initial_thread_tls_data;
pthread_key_create (&gomp_tls_key, NULL);
pthread_setspecific (gomp_tls_key, &initial_thread_tls_data);
#endif
if (pthread_key_create (&gomp_thread_destructor, gomp_free_thread) != 0)
gomp_fatal ("could not create thread pool destructor.");
}
static void __attribute__((destructor))
team_destructor (void)
{
/* Without this dlclose on libgomp could lead to subsequent
crashes. */
pthread_key_delete (gomp_thread_destructor);
}
/* Similar to gomp_free_pool_helper, but don't detach itself,
gomp_pause_host will pthread_join those threads. */
static void
gomp_pause_pool_helper (void *thread_pool)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_thread_pool *pool
= (struct gomp_thread_pool *) thread_pool;
gomp_simple_barrier_wait_last (&pool->threads_dock);
gomp_sem_destroy (&thr->release);
thr->thread_pool = NULL;
thr->task = NULL;
pthread_exit (NULL);
}
/* Free a thread pool and release its threads. Return non-zero on
failure. */
int
gomp_pause_host (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_thread_pool *pool = thr->thread_pool;
if (thr->ts.level)
return -1;
if (pool)
{
if (pool->threads_used > 0)
{
int i;
pthread_t *thrs
= gomp_alloca (sizeof (pthread_t) * pool->threads_used);
for (i = 1; i < pool->threads_used; i++)
{
struct gomp_thread *nthr = pool->threads[i];
nthr->fn = gomp_pause_pool_helper;
nthr->data = pool;
thrs[i] = gomp_thread_to_pthread_t (nthr);
}
/* This barrier undocks threads docked on pool->threads_dock. */
gomp_release_held_threads (pool, NULL, thr->ts.team_id);
if (!gomp_barrier_can_hold (pool->prev_barrier))
gomp_simple_barrier_wait (&pool->threads_dock);
/* And this waits till all threads have called
gomp_simple_barrier_wait_last in gomp_pause_pool_helper. */
gomp_simple_barrier_wait (&pool->threads_dock);
/* Now it is safe to destroy the barrier and free the pool. */
gomp_simple_barrier_destroy (&pool->threads_dock);
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_managed_threads,
1L - pool->threads_used);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
gomp_managed_threads -= pool->threads_used - 1L;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
for (i = 1; i < pool->threads_used; i++)
pthread_join (thrs[i], NULL);
}
if (pool->last_team)
free_team (pool->last_team);
#ifndef __nvptx__
team_free (pool->threads);
team_free (pool);
#endif
thr->thread_pool = NULL;
}
return 0;
}
#endif
struct gomp_task_icv *
gomp_new_icv (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_task *task = gomp_malloc (sizeof (struct gomp_task));
gomp_init_task (task, NULL, &gomp_global_icv);
thr->task = task;
#ifdef LIBGOMP_USE_PTHREADS
pthread_setspecific (gomp_thread_destructor, thr);
#endif
return &task->icv;
}