blob: eebdb9a29f1fc75bd0c2bf45e989bd86e21a9cc5 [file] [log] [blame]
/* Copyright (C) 2011-2021 Free Software Foundation, Inc.
Contributed by Torvald Riegel <>.
This file is part of the GNU Transactional Memory Library (libitm).
Libitm is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<>. */
#include "libitm_i.h"
#include "futex.h"
#include <limits.h>
namespace GTM HIDDEN {
// Acquire a RW lock for reading.
gtm_rwlock::read_lock (gtm_thread *tx)
for (;;)
// Fast path: first announce our intent to read, then check for
// conflicting intents to write. The fence ensures that this happens
// in exactly this order.
tx-> (0, memory_order_relaxed);
atomic_thread_fence (memory_order_seq_cst);
if (likely (writers.load (memory_order_relaxed) == 0))
// There seems to be an active, waiting, or confirmed writer, so enter
// the futex-based slow path.
// Before waiting, we clear our read intent check whether there are any
// writers that might potentially wait for readers. If so, wake them.
// We need the barrier here for the same reason that we need it in
// read_unlock().
// TODO Potentially too many wake-ups. See comments in read_unlock().
tx-> (-1, memory_order_relaxed);
atomic_thread_fence (memory_order_seq_cst);
if (writer_readers.load (memory_order_relaxed) > 0)
{ (0, memory_order_relaxed);
futex_wake(&writer_readers, 1);
// Signal that there are waiting readers and wait until there is no
// writer anymore.
// TODO Spin here on writers for a while. Consider whether we woke
// any writers before?
while (writers.load (memory_order_relaxed))
// An active writer. Wait until it has finished. To avoid lost
// wake-ups, we need to use Dekker-like synchronization.
// Note that we cannot reset readers to zero when we see that there
// are no writers anymore after the barrier because this pending
// store could then lead to lost wake-ups at other readers. (1, memory_order_relaxed);
atomic_thread_fence (memory_order_seq_cst);
if (writers.load (memory_order_relaxed))
futex_wait(&readers, 1);
// There is no writer, actually. However, we can have enabled
// a futex_wait in other readers by previously setting readers
// to 1, so we have to wake them up because there is no writer
// that will do that. We don't know whether the wake-up is
// really necessary, but we can get lost wake-up situations
// otherwise.
// No additional barrier nor a nonrelaxed load is required due
// to coherency constraints. write_unlock() checks readers to
// see if any wake-up is necessary, but it is not possible that
// a reader's store prevents a required later writer wake-up;
// If the waking reader's store (value 0) is in modification
// order after the waiting readers store (value 1), then the
// latter will have to read 0 in the futex due to coherency
// constraints and the happens-before enforced by the futex
// (paragraph 6.10 in the standard, 6.19.4 in the Batty et al
// TR); second, the writer will be forced to read in
// modification order too due to Dekker-style synchronization
// with the waiting reader (see write_unlock()).
// ??? Can we avoid the wake-up if readers is zero (like in
// write_unlock())? Anyway, this might happen too infrequently
// to improve performance significantly. (0, memory_order_relaxed);
futex_wake(&readers, INT_MAX);
// And we try again to acquire a read lock.
// Acquire a RW lock for writing. Generic version that also works for
// upgrades.
// Note that an upgrade might fail (and thus waste previous work done during
// this transaction) if there is another thread that tried to go into serial
// mode earlier (i.e., upgrades do not have higher priority than pure writers).
// However, this seems rare enough to not consider it further as we need both
// a non-upgrade writer and a writer to happen to switch to serial mode
// concurrently. If we'd want to handle this, a writer waiting for readers
// would have to coordinate with later arriving upgrades and hand over the
// lock to them, including the the reader-waiting state. We can try to support
// this if this will actually happen often enough in real workloads.
gtm_rwlock::write_lock_generic (gtm_thread *tx)
// Try to acquire the write lock. Relaxed MO is fine because of the
// additional fence below.
int w = 0;
if (unlikely (!writers.compare_exchange_strong (w, 1, memory_order_relaxed)))
// If this is an upgrade, we must not wait for other writers or
// upgrades.
if (tx != 0)
return false;
// There is already a writer. If there are no other waiting writers,
// switch to contended mode. We need seq_cst memory order to make the
// Dekker-style synchronization work.
if (w != 2)
w = (2, memory_order_relaxed);
while (w != 0)
futex_wait(&writers, 2);
w = (2, memory_order_relaxed);
// This fence is both required for the Dekker-like synchronization we do
// here and is the acquire MO required to make us synchronize-with prior
// writers.
atomic_thread_fence (memory_order_seq_cst);
// We have acquired the writer side of the R/W lock. Now wait for any
// readers that might still be active.
// TODO In the worst case, this requires one wait/wake pair for each
// active reader. Reduce this!
for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
it = it->next_thread)
if (it == tx)
// Use a loop here to check reader flags again after waiting.
while (it->shared_state.load (memory_order_relaxed)
!= ~(typeof it->shared_state)0)
// If this is an upgrade, we have to break deadlocks with
// privatization safety. This may fail on our side, in which
// case we need to cancel our attempt to upgrade. Also, we do not
// block but just spin so that we never have to be woken.
if (tx != 0)
if (!abi_disp()->snapshot_most_recent ())
write_unlock ();
return false;
// An active reader. Wait until it has finished. To avoid lost
// wake-ups, we need to use Dekker-like synchronization.
// Note that we can reset writer_readers to zero when we see after
// the barrier that the reader has finished in the meantime;
// however, this is only possible because we are the only writer.
// TODO Spin for a while on this reader flag. (1, memory_order_relaxed);
atomic_thread_fence (memory_order_seq_cst);
if (it->shared_state.load (memory_order_relaxed)
!= ~(typeof it->shared_state)0)
futex_wait(&writer_readers, 1);
else (0, memory_order_relaxed);
return true;
// Acquire a RW lock for writing.
gtm_rwlock::write_lock ()
write_lock_generic (0);
// Upgrade a RW lock that has been locked for reading to a writing lock.
// Do this without possibility of another writer incoming. Return false
// if this attempt fails (i.e. another thread also upgraded).
gtm_rwlock::write_upgrade (gtm_thread *tx)
return write_lock_generic (tx);
// Has to be called iff the previous upgrade was successful and after it is
// safe for the transaction to not be marked as a reader anymore.
gtm_rwlock::write_upgrade_finish (gtm_thread *tx)
// We are not a reader anymore. This is only safe to do after we have
// acquired the writer lock.
tx-> (-1, memory_order_release);
// Release a RW lock from reading.
gtm_rwlock::read_unlock (gtm_thread *tx)
// We only need release memory order here because of privatization safety
// (this ensures that marking the transaction as inactive happens after
// any prior data accesses by this transaction, and that neither the
// compiler nor the hardware order this store earlier).
// ??? We might be able to avoid this release here if the compiler can't
// merge the release fence with the subsequent seq_cst fence.
tx-> (-1, memory_order_release);
// If there is a writer waiting for readers, wake it up. We need the fence
// to avoid lost wake-ups. Furthermore, the privatization safety
// implementation in gtm_thread::try_commit() relies on the existence of
// this seq_cst fence.
// ??? We might not be the last active reader, so the wake-up might happen
// too early. How do we avoid this without slowing down readers too much?
// Each reader could scan the list of txns for other active readers but
// this can result in many cache misses. Use combining instead?
// TODO Sends out one wake-up for each reader in the worst case.
atomic_thread_fence (memory_order_seq_cst);
if (unlikely (writer_readers.load (memory_order_relaxed) > 0))
// No additional barrier needed here (see write_unlock()). (0, memory_order_relaxed);
futex_wake(&writer_readers, 1);
// Release a RW lock from writing.
gtm_rwlock::write_unlock ()
// Release MO so that we synchronize with subsequent writers.
if ( (0, memory_order_release) == 2)
// There might be waiting writers, so wake them. If we woke any thread,
// we assume it to indeed be a writer; waiting writers will never give
// up, so we can assume that they will take care of anything else such
// as waking readers.
if (futex_wake(&writers, 1) > 0)
// If we did not wake any waiting writers, we might indeed be the last
// writer (this can happen because write_lock_generic() exchanges 0 or 1
// to 2 and thus might go to contended mode even if no other thread
// holds the write lock currently). Therefore, we have to fall through
// to the normal reader wake-up code.
// This fence is required because we do Dekker-like synchronization here.
atomic_thread_fence (memory_order_seq_cst);
// No waiting writers, so wake up all waiting readers.
if (readers.load (memory_order_relaxed) > 0)
// No additional barrier needed here. The previous load must be in
// modification order because of the coherency constraints. Late stores
// by a reader are not a problem because readers do Dekker-style
// synchronization on writers. (0, memory_order_relaxed);
futex_wake(&readers, INT_MAX);
} // namespace GTM