blob: c04324e5df0a173cd2b71b9250885cc5b92d7812 [file] [log] [blame]
/* Copyright (C) 2008-2021 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
Libitm is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "libitm_i.h"
#include <pthread.h>
using namespace GTM;
#if !defined(HAVE_ARCH_GTM_THREAD) || !defined(HAVE_ARCH_GTM_THREAD_DISP)
extern __thread gtm_thread_tls _gtm_thr_tls;
#endif
// Put this at the start of a cacheline so that serial_lock's writers and
// htm_fastpath fields are on the same cacheline, so that HW transactions
// only have to pay one cacheline capacity to monitor both.
gtm_rwlock GTM::gtm_thread::serial_lock
__attribute__((aligned(HW_CACHELINE_SIZE)));
gtm_thread *GTM::gtm_thread::list_of_threads = 0;
unsigned GTM::gtm_thread::number_of_threads = 0;
/* ??? Move elsewhere when we figure out library initialization. */
uint64_t GTM::gtm_spin_count_var = 1000;
#ifdef HAVE_64BIT_SYNC_BUILTINS
static atomic<_ITM_transactionId_t> global_tid;
#else
static _ITM_transactionId_t global_tid;
static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
#endif
// Provides a on-thread-exit callback used to release per-thread data.
static pthread_key_t thr_release_key;
static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
/* Allocate a transaction structure. */
void *
GTM::gtm_thread::operator new (size_t s)
{
void *tx;
assert(s == sizeof(gtm_thread));
tx = xmalloc (sizeof (gtm_thread), true);
memset (tx, 0, sizeof (gtm_thread));
return tx;
}
/* Free the given transaction. Raises an error if the transaction is still
in use. */
void
GTM::gtm_thread::operator delete(void *tx)
{
free(tx);
}
static void
thread_exit_handler(void *)
{
gtm_thread *thr = gtm_thr();
if (thr)
delete thr;
set_gtm_thr(0);
}
static void
thread_exit_init()
{
if (pthread_key_create(&thr_release_key, thread_exit_handler))
GTM_fatal("Creating thread release TLS key failed.");
}
GTM::gtm_thread::~gtm_thread()
{
if (nesting > 0)
GTM_fatal("Thread exit while a transaction is still active.");
// Deregister this transaction.
serial_lock.write_lock ();
gtm_thread **prev = &list_of_threads;
for (; *prev; prev = &(*prev)->next_thread)
{
if (*prev == this)
{
*prev = (*prev)->next_thread;
break;
}
}
number_of_threads--;
number_of_threads_changed(number_of_threads + 1, number_of_threads);
serial_lock.write_unlock ();
}
GTM::gtm_thread::gtm_thread ()
{
// This object's memory has been set to zero by operator new, so no need
// to initialize any of the other primitive-type members that do not have
// constructors.
shared_state.store(-1, memory_order_relaxed);
// Register this transaction with the list of all threads' transactions.
serial_lock.write_lock ();
next_thread = list_of_threads;
list_of_threads = this;
number_of_threads++;
number_of_threads_changed(number_of_threads - 1, number_of_threads);
serial_lock.write_unlock ();
init_cpp_exceptions ();
if (pthread_once(&thr_release_once, thread_exit_init))
GTM_fatal("Initializing thread release TLS key failed.");
// Any non-null value is sufficient to trigger destruction of this
// transaction when the current thread terminates.
if (pthread_setspecific(thr_release_key, this))
GTM_fatal("Setting thread release TLS key failed.");
}
static inline uint32_t
choose_code_path(uint32_t prop, abi_dispatch *disp)
{
if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
return a_runUninstrumentedCode;
else
return a_runInstrumentedCode;
}
#ifdef TARGET_BEGIN_TRANSACTION_ATTRIBUTE
/* This macro can be used to define target specific attributes for this
function. For example, S/390 requires floating point to be disabled in
begin_transaction. */
TARGET_BEGIN_TRANSACTION_ATTRIBUTE
#endif
uint32_t
GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
{
static const _ITM_transactionId_t tid_block_size = 1 << 16;
gtm_thread *tx;
abi_dispatch *disp;
uint32_t ret;
// ??? pr_undoLogCode is not properly defined in the ABI. Are barriers
// omitted because they are not necessary (e.g., a transaction on thread-
// local data) or because the compiler thinks that some kind of global
// synchronization might perform better?
if (unlikely(prop & pr_undoLogCode))
GTM_fatal("pr_undoLogCode not supported");
#ifdef USE_HTM_FASTPATH
// HTM fastpath. Only chosen in the absence of transaction_cancel to allow
// using an uninstrumented code path.
// The fastpath is enabled only by dispatch_htm's method group, which uses
// serial-mode methods as fallback. Serial-mode transactions cannot execute
// concurrently with HW transactions because the latter monitor the serial
// lock's writer flag and thus abort if another thread is or becomes a
// serial transaction. Therefore, if the fastpath is enabled, then a
// transaction is not executing as a HW transaction iff the serial lock is
// write-locked. Also, HW transactions monitor the fastpath control
// variable, so that they will only execute if dispatch_htm is still the
// current method group. This allows us to use htm_fastpath and the serial
// lock's writers flag to reliable determine whether the current thread runs
// a HW transaction, and thus we do not need to maintain this information in
// per-thread state.
// If an uninstrumented code path is not available, we can still run
// instrumented code from a HW transaction because the HTM fastpath kicks
// in early in both begin and commit, and the transaction is not canceled.
// HW transactions might get requests to switch to serial-irrevocable mode,
// but these can be ignored because the HTM provides all necessary
// correctness guarantees. Transactions cannot detect whether they are
// indeed in serial mode, and HW transactions should never need serial mode
// for any internal changes (e.g., they never abort visibly to the STM code
// and thus do not trigger the standard retry handling).
#ifndef HTM_CUSTOM_FASTPATH
if (likely(serial_lock.get_htm_fastpath() && (prop & pr_hasNoAbort)))
{
// Note that the snapshot of htm_fastpath that we take here could be
// outdated, and a different method group than dispatch_htm may have
// been chosen in the meantime. Therefore, take care not not touch
// anything besides the serial lock, which is independent of method
// groups.
for (uint32_t t = serial_lock.get_htm_fastpath(); t; t--)
{
uint32_t ret = htm_begin();
if (htm_begin_success(ret))
{
// We are executing a transaction now.
// Monitor the writer flag in the serial-mode lock, and abort
// if there is an active or waiting serial-mode transaction.
// Also checks that htm_fastpath is still nonzero and thus
// HW transactions are allowed to run.
// Note that this can also happen due to an enclosing
// serial-mode transaction; we handle this case below.
if (unlikely(serial_lock.htm_fastpath_disabled()))
htm_abort();
else
// We do not need to set a_saveLiveVariables because of HTM.
return (prop & pr_uninstrumentedCode) ?
a_runUninstrumentedCode : a_runInstrumentedCode;
}
// The transaction has aborted. Don't retry if it's unlikely that
// retrying the transaction will be successful.
if (!htm_abort_should_retry(ret))
break;
// Check whether the HTM fastpath has been disabled.
if (!serial_lock.get_htm_fastpath())
break;
// Wait until any concurrent serial-mode transactions have finished.
// This is an empty critical section, but won't be elided.
if (serial_lock.htm_fastpath_disabled())
{
tx = gtm_thr();
if (unlikely(tx == NULL))
{
// See below.
tx = new gtm_thread();
set_gtm_thr(tx);
}
// Check whether there is an enclosing serial-mode transaction;
// if so, we just continue as a nested transaction and don't
// try to use the HTM fastpath. This case can happen when an
// outermost relaxed transaction calls unsafe code that starts
// a transaction.
if (tx->nesting > 0)
break;
// Another thread is running a serial-mode transaction. Wait.
serial_lock.read_lock(tx);
serial_lock.read_unlock(tx);
// TODO We should probably reset the retry count t here, unless
// we have retried so often that we should go serial to avoid
// starvation.
}
}
}
#else
// If we have a custom HTM fastpath in ITM_beginTransaction, we implement
// just the retry policy here. We communicate with the custom fastpath
// through additional property bits and return codes, and either transfer
// control back to the custom fastpath or run the fallback mechanism. The
// fastpath synchronization algorithm itself is the same.
// pr_HTMRetryableAbort states that a HW transaction started by the custom
// HTM fastpath aborted, and that we thus have to decide whether to retry
// the fastpath (returning a_tryHTMFastPath) or just proceed with the
// fallback method.
if (likely(serial_lock.get_htm_fastpath() && (prop & pr_HTMRetryableAbort)))
{
tx = gtm_thr();
if (unlikely(tx == NULL))
{
// See below.
tx = new gtm_thread();
set_gtm_thr(tx);
}
// If this is the first abort, reset the retry count. We abuse
// restart_total for the retry count, which is fine because our only
// other fallback will use serial transactions, which don't use
// restart_total but will reset it when committing.
if (!(prop & pr_HTMRetriedAfterAbort))
tx->restart_total = gtm_thread::serial_lock.get_htm_fastpath();
if (--tx->restart_total > 0)
{
// Wait until any concurrent serial-mode transactions have finished.
// Essentially the same code as above.
if (!serial_lock.get_htm_fastpath())
goto stop_custom_htm_fastpath;
if (serial_lock.htm_fastpath_disabled())
{
if (tx->nesting > 0)
goto stop_custom_htm_fastpath;
serial_lock.read_lock(tx);
serial_lock.read_unlock(tx);
}
// Let ITM_beginTransaction retry the custom HTM fastpath.
return a_tryHTMFastPath;
}
}
stop_custom_htm_fastpath:
#endif
#endif
tx = gtm_thr();
if (unlikely(tx == NULL))
{
// Create the thread object. The constructor will also set up automatic
// deletion on thread termination.
tx = new gtm_thread();
set_gtm_thr(tx);
}
if (tx->nesting > 0)
{
// This is a nested transaction.
// Check prop compatibility:
// The ABI requires pr_hasNoFloatUpdate, pr_hasNoVectorUpdate,
// pr_hasNoIrrevocable, pr_aWBarriersOmitted, pr_RaRBarriersOmitted, and
// pr_hasNoSimpleReads to hold for the full dynamic scope of a
// transaction. We could check that these are set for the nested
// transaction if they are also set for the parent transaction, but the
// ABI does not require these flags to be set if they could be set,
// so the check could be too strict.
// ??? For pr_readOnly, lexical or dynamic scope is unspecified.
if (prop & pr_hasNoAbort)
{
// We can use flat nesting, so elide this transaction.
if (!(prop & pr_instrumentedCode))
{
if (!(tx->state & STATE_SERIAL) ||
!(tx->state & STATE_IRREVOCABLE))
tx->serialirr_mode();
}
// Increment nesting level after checking that we have a method that
// allows us to continue.
tx->nesting++;
return choose_code_path(prop, abi_disp());
}
// The transaction might abort, so use closed nesting if possible.
// pr_hasNoAbort has lexical scope, so the compiler should really have
// generated an instrumented code path.
assert(prop & pr_instrumentedCode);
// Create a checkpoint of the current transaction.
gtm_transaction_cp *cp = tx->parent_txns.push();
cp->save(tx);
new (&tx->alloc_actions) aa_tree<uintptr_t, gtm_alloc_action>();
// Check whether the current method actually supports closed nesting.
// If we can switch to another one, do so.
// If not, we assume that actual aborts are infrequent, and rather
// restart in _ITM_abortTransaction when we really have to.
disp = abi_disp();
if (!disp->closed_nesting())
{
// ??? Should we elide the transaction if there is no alternative
// method that supports closed nesting? If we do, we need to set
// some flag to prevent _ITM_abortTransaction from aborting the
// wrong transaction (i.e., some parent transaction).
abi_dispatch *cn_disp = disp->closed_nesting_alternative();
if (cn_disp)
{
disp = cn_disp;
set_abi_disp(disp);
}
}
}
else
{
// Outermost transaction
disp = tx->decide_begin_dispatch (prop);
set_abi_disp (disp);
}
// Initialization that is common for outermost and nested transactions.
tx->prop = prop;
tx->nesting++;
tx->jb = *jb;
// As long as we have not exhausted a previously allocated block of TIDs,
// we can avoid an atomic operation on a shared cacheline.
if (tx->local_tid & (tid_block_size - 1))
tx->id = tx->local_tid++;
else
{
#ifdef HAVE_64BIT_SYNC_BUILTINS
// We don't really care which block of TIDs we get but only that we
// acquire one atomically; therefore, relaxed memory order is
// sufficient.
tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
tx->local_tid = tx->id + 1;
#else
pthread_mutex_lock (&global_tid_lock);
global_tid += tid_block_size;
tx->id = global_tid;
tx->local_tid = tx->id + 1;
pthread_mutex_unlock (&global_tid_lock);
#endif
}
// Log the number of uncaught exceptions if we might have to roll back this
// state.
if (tx->cxa_uncaught_count_ptr != 0)
tx->cxa_uncaught_count = *tx->cxa_uncaught_count_ptr;
// Run dispatch-specific restart code. Retry until we succeed.
GTM::gtm_restart_reason rr;
while ((rr = disp->begin_or_restart()) != NO_RESTART)
{
tx->decide_retry_strategy(rr);
disp = abi_disp();
}
// Determine the code path to run. Only irrevocable transactions cannot be
// restarted, so all other transactions need to save live variables.
ret = choose_code_path(prop, disp);
if (!(tx->state & STATE_IRREVOCABLE))
ret |= a_saveLiveVariables;
return ret;
}
void
GTM::gtm_transaction_cp::save(gtm_thread* tx)
{
// Save everything that we might have to restore on restarts or aborts.
jb = tx->jb;
undolog_size = tx->undolog.size();
/* FIXME! Assignment of an aatree like alloc_actions is unsafe; if either
*this or *tx is destroyed, the other ends up pointing to a freed node. */
#pragma GCC diagnostic warning "-Wdeprecated-copy"
alloc_actions = tx->alloc_actions;
user_actions_size = tx->user_actions.size();
id = tx->id;
prop = tx->prop;
cxa_catch_count = tx->cxa_catch_count;
cxa_uncaught_count = tx->cxa_uncaught_count;
disp = abi_disp();
nesting = tx->nesting;
}
void
GTM::gtm_transaction_cp::commit(gtm_thread* tx)
{
// Restore state that is not persistent across commits. Exception handling,
// information, nesting level, and any logs do not need to be restored on
// commits of nested transactions. Allocation actions must be committed
// before committing the snapshot.
tx->jb = jb;
tx->alloc_actions = alloc_actions;
tx->id = id;
tx->prop = prop;
}
void
GTM::gtm_thread::rollback (gtm_transaction_cp *cp, bool aborting)
{
// The undo log is special in that it used for both thread-local and shared
// data. Because of the latter, we have to roll it back before any
// dispatch-specific rollback (which handles synchronization with other
// transactions).
undolog.rollback (this, cp ? cp->undolog_size : 0);
// Perform dispatch-specific rollback.
abi_disp()->rollback (cp);
// Roll back all actions that are supposed to happen around the transaction.
rollback_user_actions (cp ? cp->user_actions_size : 0);
commit_allocations (true, (cp ? &cp->alloc_actions : 0));
revert_cpp_exceptions (cp);
if (cp)
{
// We do not yet handle restarts of nested transactions. To do that, we
// would have to restore some state (jb, id, prop, nesting) not to the
// checkpoint but to the transaction that was started from this
// checkpoint (e.g., nesting = cp->nesting + 1);
assert(aborting);
// Roll back the rest of the state to the checkpoint.
jb = cp->jb;
id = cp->id;
prop = cp->prop;
if (cp->disp != abi_disp())
set_abi_disp(cp->disp);
alloc_actions = cp->alloc_actions;
nesting = cp->nesting;
}
else
{
// Roll back to the outermost transaction.
// Restore the jump buffer and transaction properties, which we will
// need for the longjmp used to restart or abort the transaction.
if (parent_txns.size() > 0)
{
jb = parent_txns[0].jb;
id = parent_txns[0].id;
prop = parent_txns[0].prop;
}
// Reset the transaction. Do not reset this->state, which is handled by
// the callers. Note that if we are not aborting, we reset the
// transaction to the point after having executed begin_transaction
// (we will return from it), so the nesting level must be one, not zero.
nesting = (aborting ? 0 : 1);
parent_txns.clear();
}
if (this->eh_in_flight)
{
_Unwind_DeleteException ((_Unwind_Exception *) this->eh_in_flight);
this->eh_in_flight = NULL;
}
}
void ITM_REGPARM
_ITM_abortTransaction (_ITM_abortReason reason)
{
gtm_thread *tx = gtm_thr();
assert (reason == userAbort || reason == (userAbort | outerAbort));
assert ((tx->prop & pr_hasNoAbort) == 0);
if (tx->state & gtm_thread::STATE_IRREVOCABLE)
abort ();
// Roll back to innermost transaction.
if (tx->parent_txns.size() > 0 && !(reason & outerAbort))
{
// If the current method does not support closed nesting but we are
// nested and must only roll back the innermost transaction, then
// restart with a method that supports closed nesting.
abi_dispatch *disp = abi_disp();
if (!disp->closed_nesting())
tx->restart(RESTART_CLOSED_NESTING);
// The innermost transaction is a closed nested transaction.
gtm_transaction_cp *cp = tx->parent_txns.pop();
uint32_t longjmp_prop = tx->prop;
gtm_jmpbuf longjmp_jb = tx->jb;
tx->rollback (cp, true);
// Jump to nested transaction (use the saved jump buffer).
GTM_longjmp (a_abortTransaction | a_restoreLiveVariables,
&longjmp_jb, longjmp_prop);
}
else
{
// There is no nested transaction or an abort of the outermost
// transaction was requested, so roll back to the outermost transaction.
tx->rollback (0, true);
// Aborting an outermost transaction finishes execution of the whole
// transaction. Therefore, reset transaction state.
if (tx->state & gtm_thread::STATE_SERIAL)
gtm_thread::serial_lock.write_unlock ();
else
gtm_thread::serial_lock.read_unlock (tx);
tx->state = 0;
GTM_longjmp (a_abortTransaction | a_restoreLiveVariables,
&tx->jb, tx->prop);
}
}
bool
GTM::gtm_thread::trycommit ()
{
nesting--;
// Skip any real commit for elided transactions.
if (nesting > 0 && (parent_txns.size() == 0 ||
nesting > parent_txns[parent_txns.size() - 1].nesting))
return true;
if (nesting > 0)
{
// Commit of a closed-nested transaction. Remove one checkpoint and add
// any effects of this transaction to the parent transaction.
gtm_transaction_cp *cp = parent_txns.pop();
commit_allocations(false, &cp->alloc_actions);
cp->commit(this);
return true;
}
// Commit of an outermost transaction.
gtm_word priv_time = 0;
if (abi_disp()->trycommit (priv_time))
{
// The transaction is now finished but we will still access some shared
// data if we have to ensure privatization safety.
bool do_read_unlock = false;
if (state & gtm_thread::STATE_SERIAL)
{
gtm_thread::serial_lock.write_unlock ();
// There are no other active transactions, so there's no need to
// enforce privatization safety.
priv_time = 0;
}
else
{
// If we have to ensure privatization safety, we must not yet
// release the read lock and become inactive because (1) we still
// have to go through the list of all transactions, which can be
// modified by serial mode threads, and (2) we interpret each
// transactions' shared_state in the context of what we believe to
// be the current method group (and serial mode transactions can
// change the method group). Therefore, if we have to ensure
// privatization safety, delay becoming inactive but set a maximum
// snapshot time (we have committed and thus have an empty snapshot,
// so it will always be most recent). Use release MO so that this
// synchronizes with other threads observing our snapshot time.
if (priv_time)
{
do_read_unlock = true;
shared_state.store((~(typeof gtm_thread::shared_state)0) - 1,
memory_order_release);
}
else
gtm_thread::serial_lock.read_unlock (this);
}
state = 0;
// We can commit the undo log after dispatch-specific commit and after
// making the transaction inactive because we only have to reset
// gtm_thread state.
undolog.commit ();
// Reset further transaction state.
cxa_catch_count = 0;
restart_total = 0;
// Ensure privatization safety, if necessary.
if (priv_time)
{
// There must be a seq_cst fence between the following loads of the
// other transactions' shared_state and the dispatch-specific stores
// that signal updates by this transaction (e.g., lock
// acquisitions). This ensures that if we read prior to other
// reader transactions setting their shared_state to 0, then those
// readers will observe our updates. We can reuse the seq_cst fence
// in serial_lock.read_unlock() if we performed that; if not, we
// issue the fence.
if (do_read_unlock)
atomic_thread_fence (memory_order_seq_cst);
// TODO Don't just spin but also block using cond vars / futexes
// here. Should probably be integrated with the serial lock code.
for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
it = it->next_thread)
{
if (it == this) continue;
// We need to load other threads' shared_state using acquire
// semantics (matching the release semantics of the respective
// updates). This is necessary to ensure that the other
// threads' memory accesses happen before our actions that
// assume privatization safety.
// TODO Are there any platform-specific optimizations (e.g.,
// merging barriers)?
while (it->shared_state.load(memory_order_acquire) < priv_time)
cpu_relax();
}
}
// After ensuring privatization safety, we are now truly inactive and
// thus can release the read lock. We will also execute potentially
// privatizing actions (e.g., calling free()). User actions are first.
if (do_read_unlock)
gtm_thread::serial_lock.read_unlock (this);
commit_user_actions ();
commit_allocations (false, 0);
return true;
}
return false;
}
void ITM_NORETURN
GTM::gtm_thread::restart (gtm_restart_reason r, bool finish_serial_upgrade)
{
// Roll back to outermost transaction. Do not reset transaction state because
// we will continue executing this transaction.
rollback ();
// If we have to restart while an upgrade of the serial lock is happening,
// we need to finish this here, after rollback (to ensure privatization
// safety despite undo writes) and before deciding about the retry strategy
// (which could switch to/from serial mode).
if (finish_serial_upgrade)
gtm_thread::serial_lock.write_upgrade_finish(this);
decide_retry_strategy (r);
// Run dispatch-specific restart code. Retry until we succeed.
abi_dispatch* disp = abi_disp();
GTM::gtm_restart_reason rr;
while ((rr = disp->begin_or_restart()) != NO_RESTART)
{
decide_retry_strategy(rr);
disp = abi_disp();
}
GTM_longjmp (choose_code_path(prop, disp) | a_restoreLiveVariables,
&jb, prop);
}
void ITM_REGPARM
_ITM_commitTransaction(void)
{
#if defined(USE_HTM_FASTPATH)
// HTM fastpath. If we are not executing a HW transaction, then we will be
// a serial-mode transaction. If we are, then there will be no other
// concurrent serial-mode transaction.
// See gtm_thread::begin_transaction.
if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
{
htm_commit();
return;
}
#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
tx->restart (RESTART_VALIDATE_COMMIT);
}
void ITM_REGPARM
_ITM_commitTransactionEH(void *exc_ptr)
{
#if defined(USE_HTM_FASTPATH)
// See _ITM_commitTransaction.
if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
{
htm_commit();
return;
}
#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
{
tx->eh_in_flight = exc_ptr;
tx->restart (RESTART_VALIDATE_COMMIT);
}
}