| /* Copyright (C) 2005-2021 Free Software Foundation, Inc. |
| Contributed by Richard Henderson <rth@redhat.com>. |
| |
| This file is part of the GNU Offloading and Multi Processing Library |
| (libgomp). |
| |
| Libgomp is free software; you can redistribute it and/or modify it |
| under the terms of the GNU General Public License as published by |
| the Free Software Foundation; either version 3, or (at your option) |
| any later version. |
| |
| Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| more details. |
| |
| Under Section 7 of GPL version 3, you are granted additional |
| permissions described in the GCC Runtime Library Exception, version |
| 3.1, as published by the Free Software Foundation. |
| |
| You should have received a copy of the GNU General Public License and |
| a copy of the GCC Runtime Library Exception along with this program; |
| see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| /* This file contains routines to manage the work-share queue for a team |
| of threads. */ |
| |
| #include "libgomp.h" |
| #include <stddef.h> |
| #include <stdlib.h> |
| #include <string.h> |
| |
| |
| /* Allocate a new work share structure, preferably from current team's |
| free gomp_work_share cache. */ |
| |
| static struct gomp_work_share * |
| alloc_work_share (struct gomp_team *team) |
| { |
| struct gomp_work_share *ws; |
| unsigned int i; |
| |
| /* This is called in a critical section. */ |
| if (team->work_share_list_alloc != NULL) |
| { |
| ws = team->work_share_list_alloc; |
| team->work_share_list_alloc = ws->next_free; |
| return ws; |
| } |
| |
| #ifdef HAVE_SYNC_BUILTINS |
| ws = team->work_share_list_free; |
| /* We need atomic read from work_share_list_free, |
| as free_work_share can be called concurrently. */ |
| __asm ("" : "+r" (ws)); |
| |
| if (ws && ws->next_free) |
| { |
| struct gomp_work_share *next = ws->next_free; |
| ws->next_free = NULL; |
| team->work_share_list_alloc = next->next_free; |
| return next; |
| } |
| #else |
| gomp_mutex_lock (&team->work_share_list_free_lock); |
| ws = team->work_share_list_free; |
| if (ws) |
| { |
| team->work_share_list_alloc = ws->next_free; |
| team->work_share_list_free = NULL; |
| gomp_mutex_unlock (&team->work_share_list_free_lock); |
| return ws; |
| } |
| gomp_mutex_unlock (&team->work_share_list_free_lock); |
| #endif |
| |
| team->work_share_chunk *= 2; |
| /* Allocating gomp_work_share structures aligned is just an |
| optimization, don't do it when using the fallback method. */ |
| #ifdef GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC |
| ws = gomp_aligned_alloc (__alignof (struct gomp_work_share), |
| team->work_share_chunk |
| * sizeof (struct gomp_work_share)); |
| #else |
| ws = gomp_malloc (team->work_share_chunk * sizeof (struct gomp_work_share)); |
| #endif |
| ws->next_alloc = team->work_shares[0].next_alloc; |
| team->work_shares[0].next_alloc = ws; |
| team->work_share_list_alloc = &ws[1]; |
| for (i = 1; i < team->work_share_chunk - 1; i++) |
| ws[i].next_free = &ws[i + 1]; |
| ws[i].next_free = NULL; |
| return ws; |
| } |
| |
| /* Initialize an already allocated struct gomp_work_share. |
| This shouldn't touch the next_alloc field. */ |
| |
| void |
| gomp_init_work_share (struct gomp_work_share *ws, size_t ordered, |
| unsigned nthreads) |
| { |
| gomp_mutex_init (&ws->lock); |
| if (__builtin_expect (ordered, 0)) |
| { |
| #define INLINE_ORDERED_TEAM_IDS_SIZE \ |
| (sizeof (struct gomp_work_share) \ |
| - offsetof (struct gomp_work_share, inline_ordered_team_ids)) |
| |
| if (__builtin_expect (ordered != 1, 0)) |
| { |
| size_t o = nthreads * sizeof (*ws->ordered_team_ids); |
| o += __alignof__ (long long) - 1; |
| if ((offsetof (struct gomp_work_share, inline_ordered_team_ids) |
| & (__alignof__ (long long) - 1)) == 0) |
| o &= ~(__alignof__ (long long) - 1); |
| ordered += o - 1; |
| } |
| else |
| ordered = nthreads * sizeof (*ws->ordered_team_ids); |
| if (ordered > INLINE_ORDERED_TEAM_IDS_SIZE) |
| ws->ordered_team_ids = team_malloc (ordered); |
| else |
| ws->ordered_team_ids = ws->inline_ordered_team_ids; |
| memset (ws->ordered_team_ids, '\0', ordered); |
| ws->ordered_num_used = 0; |
| ws->ordered_owner = -1; |
| ws->ordered_cur = 0; |
| } |
| else |
| ws->ordered_team_ids = ws->inline_ordered_team_ids; |
| gomp_ptrlock_init (&ws->next_ws, NULL); |
| ws->threads_completed = 0; |
| } |
| |
| /* Do any needed destruction of gomp_work_share fields before it |
| is put back into free gomp_work_share cache or freed. */ |
| |
| void |
| gomp_fini_work_share (struct gomp_work_share *ws) |
| { |
| gomp_mutex_destroy (&ws->lock); |
| if (ws->ordered_team_ids != ws->inline_ordered_team_ids) |
| team_free (ws->ordered_team_ids); |
| gomp_ptrlock_destroy (&ws->next_ws); |
| } |
| |
| /* Free a work share struct, if not orphaned, put it into current |
| team's free gomp_work_share cache. */ |
| |
| static inline void |
| free_work_share (struct gomp_team *team, struct gomp_work_share *ws) |
| { |
| gomp_fini_work_share (ws); |
| if (__builtin_expect (team == NULL, 0)) |
| free (ws); |
| else |
| { |
| struct gomp_work_share *next_ws; |
| #ifdef HAVE_SYNC_BUILTINS |
| do |
| { |
| next_ws = team->work_share_list_free; |
| ws->next_free = next_ws; |
| } |
| while (!__sync_bool_compare_and_swap (&team->work_share_list_free, |
| next_ws, ws)); |
| #else |
| gomp_mutex_lock (&team->work_share_list_free_lock); |
| next_ws = team->work_share_list_free; |
| ws->next_free = next_ws; |
| team->work_share_list_free = ws; |
| gomp_mutex_unlock (&team->work_share_list_free_lock); |
| #endif |
| } |
| } |
| |
| /* The current thread is ready to begin the next work sharing construct. |
| In all cases, thr->ts.work_share is updated to point to the new |
| structure. In all cases the work_share lock is locked. Return true |
| if this was the first thread to reach this point. */ |
| |
| bool |
| gomp_work_share_start (size_t ordered) |
| { |
| struct gomp_thread *thr = gomp_thread (); |
| struct gomp_team *team = thr->ts.team; |
| struct gomp_work_share *ws; |
| |
| /* Work sharing constructs can be orphaned. */ |
| if (team == NULL) |
| { |
| ws = gomp_malloc (sizeof (*ws)); |
| gomp_init_work_share (ws, ordered, 1); |
| thr->ts.work_share = ws; |
| return true; |
| } |
| |
| ws = thr->ts.work_share; |
| thr->ts.last_work_share = ws; |
| ws = gomp_ptrlock_get (&ws->next_ws); |
| if (ws == NULL) |
| { |
| /* This thread encountered a new ws first. */ |
| struct gomp_work_share *ws = alloc_work_share (team); |
| gomp_init_work_share (ws, ordered, team->nthreads); |
| thr->ts.work_share = ws; |
| return true; |
| } |
| else |
| { |
| thr->ts.work_share = ws; |
| return false; |
| } |
| } |
| |
| /* The current thread is done with its current work sharing construct. |
| This version does imply a barrier at the end of the work-share. */ |
| |
| void |
| gomp_work_share_end (void) |
| { |
| struct gomp_thread *thr = gomp_thread (); |
| struct gomp_team *team = thr->ts.team; |
| gomp_barrier_state_t bstate; |
| |
| /* Work sharing constructs can be orphaned. */ |
| if (team == NULL) |
| { |
| free_work_share (NULL, thr->ts.work_share); |
| thr->ts.work_share = NULL; |
| return; |
| } |
| |
| bstate = gomp_barrier_wait_start (&team->barrier); |
| |
| if (gomp_barrier_last_thread (bstate)) |
| { |
| if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) |
| { |
| team->work_shares_to_free = thr->ts.work_share; |
| free_work_share (team, thr->ts.last_work_share); |
| } |
| } |
| |
| gomp_team_barrier_wait_end (&team->barrier, bstate); |
| thr->ts.last_work_share = NULL; |
| } |
| |
| /* The current thread is done with its current work sharing construct. |
| This version implies a cancellable barrier at the end of the work-share. */ |
| |
| bool |
| gomp_work_share_end_cancel (void) |
| { |
| struct gomp_thread *thr = gomp_thread (); |
| struct gomp_team *team = thr->ts.team; |
| gomp_barrier_state_t bstate; |
| |
| /* Cancellable work sharing constructs cannot be orphaned. */ |
| bstate = gomp_barrier_wait_cancel_start (&team->barrier); |
| |
| if (gomp_barrier_last_thread (bstate)) |
| { |
| if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) |
| { |
| team->work_shares_to_free = thr->ts.work_share; |
| free_work_share (team, thr->ts.last_work_share); |
| } |
| } |
| thr->ts.last_work_share = NULL; |
| |
| return gomp_team_barrier_wait_cancel_end (&team->barrier, bstate); |
| } |
| |
| /* The current thread is done with its current work sharing construct. |
| This version does NOT imply a barrier at the end of the work-share. */ |
| |
| void |
| gomp_work_share_end_nowait (void) |
| { |
| struct gomp_thread *thr = gomp_thread (); |
| struct gomp_team *team = thr->ts.team; |
| struct gomp_work_share *ws = thr->ts.work_share; |
| unsigned completed; |
| |
| /* Work sharing constructs can be orphaned. */ |
| if (team == NULL) |
| { |
| free_work_share (NULL, ws); |
| thr->ts.work_share = NULL; |
| return; |
| } |
| |
| if (__builtin_expect (thr->ts.last_work_share == NULL, 0)) |
| return; |
| |
| #ifdef HAVE_SYNC_BUILTINS |
| completed = __sync_add_and_fetch (&ws->threads_completed, 1); |
| #else |
| gomp_mutex_lock (&ws->lock); |
| completed = ++ws->threads_completed; |
| gomp_mutex_unlock (&ws->lock); |
| #endif |
| |
| if (completed == team->nthreads) |
| { |
| team->work_shares_to_free = thr->ts.work_share; |
| free_work_share (team, thr->ts.last_work_share); |
| } |
| thr->ts.last_work_share = NULL; |
| } |