libphobos/libdruntime/core/internal/atomic.d

/**
* The core.internal.atomic module comtains the low-level atomic features available in hardware.
* This module may be a routing layer for compiler intrinsics.
*
* Copyright: Copyright Manu Evans 2019.
* License:   $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
* Authors:   Sean Kelly, Alex Rønne Petersen, Manu Evans
* Source:    $(DRUNTIMESRC core/internal/_atomic.d)
*/

module core.internal.atomic;

import core.atomic : MemoryOrder, has128BitCAS;

version (DigitalMars)
{
    private
    {
        enum : int
        {
            AX, BX, CX, DX, DI, SI, R8, R9
        }

        immutable string[4][8] registerNames = [
            [ "AL", "AX", "EAX", "RAX" ],
            [ "BL", "BX", "EBX", "RBX" ],
            [ "CL", "CX", "ECX", "RCX" ],
            [ "DL", "DX", "EDX", "RDX" ],
            [ "DIL", "DI", "EDI", "RDI" ],
            [ "SIL", "SI", "ESI", "RSI" ],
            [ "R8B", "R8W", "R8D", "R8" ],
            [ "R9B", "R9W", "R9D", "R9" ],
        ];

        template RegIndex(T)
        {
            static if (T.sizeof == 1)
                enum RegIndex = 0;
            else static if (T.sizeof == 2)
                enum RegIndex = 1;
            else static if (T.sizeof == 4)
                enum RegIndex = 2;
            else static if (T.sizeof == 8)
                enum RegIndex = 3;
            else
                static assert(false, "Invalid type");
        }

        enum SizedReg(int reg, T = size_t) = registerNames[reg][RegIndex!T];
    }

    inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        static assert(order != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()");

        static if (T.sizeof == size_t.sizeof * 2)
        {
            version (D_InlineAsm_X86)
            {
                asm pure nothrow @nogc @trusted
                {
                    push EDI;
                    push EBX;
                    mov EBX, 0;
                    mov ECX, 0;
                    mov EAX, 0;
                    mov EDX, 0;
                    mov EDI, src;
                    lock; cmpxchg8b [EDI];
                    pop EBX;
                    pop EDI;
                }
            }
            else version (D_InlineAsm_X86_64)
            {
                version (Windows)
                {
                    static if (RegisterReturn!T)
                    {
                        enum SrcPtr = SizedReg!CX;
                        enum RetPtr = null;
                    }
                    else
                    {
                        enum SrcPtr = SizedReg!DX;
                        enum RetPtr = SizedReg!CX;
                    }

                    mixin (simpleFormat(q{
                        asm pure nothrow @nogc @trusted
                        {
                            naked;
                            push RBX;
                            mov R8, %0;
    ?1                        mov R9, %1;
                            mov RBX, 0;
                            mov RCX, 0;
                            mov RAX, 0;
                            mov RDX, 0;
                            lock; cmpxchg16b [R8];
    ?1                        mov [R9], RAX;
    ?1                        mov 8[R9], RDX;
                            pop RBX;
                            ret;
                        }
                    }, [SrcPtr, RetPtr]));
                }
                else
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov RBX, 0;
                        mov RCX, 0;
                        mov RAX, 0;
                        mov RDX, 0;
                        lock; cmpxchg16b [RDI];
                        pop RBX;
                        ret;
                    }
                }
            }
        }
        else static if (needsLoadBarrier!order)
        {
            version (D_InlineAsm_X86)
            {
                enum SrcReg = SizedReg!CX;
                enum ZeroReg = SizedReg!(DX, T);
                enum ResReg = SizedReg!(AX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        mov %1, 0;
                        mov %2, 0;
                        mov %0, src;
                        lock; cmpxchg [%0], %1;
                    }
                }, [SrcReg, ZeroReg, ResReg]));
            }
            else version (D_InlineAsm_X86_64)
            {
                version (Windows)
                    enum SrcReg = SizedReg!CX;
                else
                    enum SrcReg = SizedReg!DI;
                enum ZeroReg = SizedReg!(DX, T);
                enum ResReg = SizedReg!(AX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        mov %1, 0;
                        mov %2, 0;
                        lock; cmpxchg [%0], %1;
                        ret;
                    }
                }, [SrcReg, ZeroReg, ResReg]));
            }
        }
        else
            return *src;
    }

    void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore()");

        static if (T.sizeof == size_t.sizeof * 2)
        {
            version (D_InlineAsm_X86)
            {
                asm pure nothrow @nogc @trusted
                {
                    push EDI;
                    push EBX;
                    lea EDI, value;
                    mov EBX, [EDI];
                    mov ECX, 4[EDI];
                    mov EDI, dest;
                    mov EAX, [EDI];
                    mov EDX, 4[EDI];
                L1: lock; cmpxchg8b [EDI];
                    jne L1;
                    pop EBX;
                    pop EDI;
                }
            }
            else version (D_InlineAsm_X86_64)
            {
                version (Windows)
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov R8, RDX;
                        mov RAX, [RDX];
                        mov RDX, 8[RDX];
                        mov RBX, [RCX];
                        mov RCX, 8[RCX];
                    L1: lock; cmpxchg16b [R8];
                        jne L1;
                        pop RBX;
                        ret;
                    }
                }
                else
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov RBX, RDI;
                        mov RCX, RSI;
                        mov RDI, RDX;
                        mov RAX, [RDX];
                        mov RDX, 8[RDX];
                    L1: lock; cmpxchg16b [RDI];
                        jne L1;
                        pop RBX;
                        ret;
                    }
                }
            }
        }
        else static if (needsStoreBarrier!order)
            atomicExchange!(order, false)(dest, value);
        else
            *dest = value;
    }

    T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (is(T : ulong))
    {
        version (D_InlineAsm_X86)
        {
            static assert(T.sizeof <= 4, "64bit atomicFetchAdd not supported on 32bit target." );

            enum DestReg = SizedReg!DX;
            enum ValReg = SizedReg!(AX, T);

            mixin (simpleFormat(q{
                asm pure nothrow @nogc @trusted
                {
                    mov %1, value;
                    mov %0, dest;
                    lock; xadd[%0], %1;
                }
            }, [DestReg, ValReg]));
        }
        else version (D_InlineAsm_X86_64)
        {
            version (Windows)
            {
                enum DestReg = SizedReg!DX;
                enum ValReg = SizedReg!(CX, T);
            }
            else
            {
                enum DestReg = SizedReg!SI;
                enum ValReg = SizedReg!(DI, T);
            }
            enum ResReg = result ? SizedReg!(AX, T) : null;

            mixin (simpleFormat(q{
                asm pure nothrow @nogc @trusted
                {
                    naked;
                    lock; xadd[%0], %1;
    ?2                mov %2, %1;
                    ret;
                }
            }, [DestReg, ValReg, ResReg]));
        }
        else
            static assert (false, "Unsupported architecture.");
    }

    T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (is(T : ulong))
    {
        return atomicFetchAdd(dest, cast(T)-cast(IntOrLong!T)value);
    }

    T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
    if (CanCAS!T)
    {
        version (D_InlineAsm_X86)
        {
            static assert(T.sizeof <= 4, "64bit atomicExchange not supported on 32bit target." );

            enum DestReg = SizedReg!CX;
            enum ValReg = SizedReg!(AX, T);

            mixin (simpleFormat(q{
                asm pure nothrow @nogc @trusted
                {
                    mov %1, value;
                    mov %0, dest;
                    xchg [%0], %1;
                }
            }, [DestReg, ValReg]));
        }
        else version (D_InlineAsm_X86_64)
        {
            version (Windows)
            {
                enum DestReg = SizedReg!DX;
                enum ValReg = SizedReg!(CX, T);
            }
            else
            {
                enum DestReg = SizedReg!SI;
                enum ValReg = SizedReg!(DI, T);
            }
            enum ResReg = result ? SizedReg!(AX, T) : null;

            mixin (simpleFormat(q{
                asm pure nothrow @nogc @trusted
                {
                    naked;
                    xchg [%0], %1;
    ?2                mov %2, %1;
                    ret;
                }
            }, [DestReg, ValReg, ResReg]));
        }
        else
            static assert (false, "Unsupported architecture.");
    }

    alias atomicCompareExchangeWeak = atomicCompareExchangeStrong;

    bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        version (D_InlineAsm_X86)
        {
            static if (T.sizeof <= 4)
            {
                enum DestAddr = SizedReg!CX;
                enum CmpAddr = SizedReg!DI;
                enum Val = SizedReg!(DX, T);
                enum Cmp = SizedReg!(AX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        push %1;
                        mov %2, value;
                        mov %1, compare;
                        mov %3, [%1];
                        mov %0, dest;
                        lock; cmpxchg [%0], %2;
                        mov [%1], %3;
                        setz AL;
                        pop %1;
                    }
                }, [DestAddr, CmpAddr, Val, Cmp]));
            }
            else static if (T.sizeof == 8)
            {
                asm pure nothrow @nogc @trusted
                {
                    push EDI;
                    push EBX;
                    lea EDI, value;
                    mov EBX, [EDI];
                    mov ECX, 4[EDI];
                    mov EDI, compare;
                    mov EAX, [EDI];
                    mov EDX, 4[EDI];
                    mov EDI, dest;
                    lock; cmpxchg8b [EDI];
                    mov EDI, compare;
                    mov [EDI], EAX;
                    mov 4[EDI], EDX;
                    setz AL;
                    pop EBX;
                    pop EDI;
                }
            }
            else
                static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." );
        }
        else version (D_InlineAsm_X86_64)
        {
            static if (T.sizeof <= 8)
            {
                version (Windows)
                {
                    enum DestAddr = SizedReg!R8;
                    enum CmpAddr = SizedReg!DX;
                    enum Val = SizedReg!(CX, T);
                }
                else
                {
                    enum DestAddr = SizedReg!DX;
                    enum CmpAddr = SizedReg!SI;
                    enum Val = SizedReg!(DI, T);
                }
                enum Res = SizedReg!(AX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        mov %3, [%1];
                        lock; cmpxchg [%0], %2;
                        jne compare_fail;
                        mov AL, 1;
                        ret;
                    compare_fail:
                        mov [%1], %3;
                        xor AL, AL;
                        ret;
                    }
                }, [DestAddr, CmpAddr, Val, Res]));
            }
            else
            {
                version (Windows)
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov R9, RDX;
                        mov RAX, [RDX];
                        mov RDX, 8[RDX];
                        mov RBX, [RCX];
                        mov RCX, 8[RCX];
                        lock; cmpxchg16b [R8];
                        pop RBX;
                        jne compare_fail;
                        mov AL, 1;
                        ret;
                    compare_fail:
                        mov [R9], RAX;
                        mov 8[R9], RDX;
                        xor AL, AL;
                        ret;
                    }
                }
                else
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov R8, RCX;
                        mov R9, RDX;
                        mov RAX, [RDX];
                        mov RDX, 8[RDX];
                        mov RBX, RDI;
                        mov RCX, RSI;
                        lock; cmpxchg16b [R8];
                        pop RBX;
                        jne compare_fail;
                        mov AL, 1;
                        ret;
                    compare_fail:
                        mov [R9], RAX;
                        mov 8[R9], RDX;
                        xor AL, AL;
                        ret;
                    }
                }
            }
        }
        else
            static assert (false, "Unsupported architecture.");
    }

    alias atomicCompareExchangeWeakNoResult = atomicCompareExchangeStrongNoResult;

    bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        version (D_InlineAsm_X86)
        {
            static if (T.sizeof <= 4)
            {
                enum DestAddr = SizedReg!CX;
                enum Cmp = SizedReg!(AX, T);
                enum Val = SizedReg!(DX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        mov %2, value;
                        mov %1, compare;
                        mov %0, dest;
                        lock; cmpxchg [%0], %2;
                        setz AL;
                    }
                }, [DestAddr, Cmp, Val]));
            }
            else static if (T.sizeof == 8)
            {
                asm pure nothrow @nogc @trusted
                {
                    push EDI;
                    push EBX;
                    lea EDI, value;
                    mov EBX, [EDI];
                    mov ECX, 4[EDI];
                    lea EDI, compare;
                    mov EAX, [EDI];
                    mov EDX, 4[EDI];
                    mov EDI, dest;
                    lock; cmpxchg8b [EDI];
                    setz AL;
                    pop EBX;
                    pop EDI;
                }
            }
            else
                static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." );
        }
        else version (D_InlineAsm_X86_64)
        {
            static if (T.sizeof <= 8)
            {
                version (Windows)
                {
                    enum DestAddr = SizedReg!R8;
                    enum Cmp = SizedReg!(DX, T);
                    enum Val = SizedReg!(CX, T);
                }
                else
                {
                    enum DestAddr = SizedReg!DX;
                    enum Cmp = SizedReg!(SI, T);
                    enum Val = SizedReg!(DI, T);
                }
                enum AXReg = SizedReg!(AX, T);

                mixin (simpleFormat(q{
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        mov %3, %1;
                        lock; cmpxchg [%0], %2;
                        setz AL;
                        ret;
                    }
                }, [DestAddr, Cmp, Val, AXReg]));
            }
            else
            {
                version (Windows)
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov RAX, [RDX];
                        mov RDX, 8[RDX];
                        mov RBX, [RCX];
                        mov RCX, 8[RCX];
                        lock; cmpxchg16b [R8];
                        setz AL;
                        pop RBX;
                        ret;
                    }
                }
                else
                {
                    asm pure nothrow @nogc @trusted
                    {
                        naked;
                        push RBX;
                        mov RAX, RDX;
                        mov RDX, RCX;
                        mov RBX, RDI;
                        mov RCX, RSI;
                        lock; cmpxchg16b [R8];
                        setz AL;
                        pop RBX;
                        ret;
                    }
                }
            }
        }
        else
            static assert (false, "Unsupported architecture.");
    }

    void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted
    {
        // TODO: `mfence` should only be required for seq_cst operations, but this depends on
        //       the compiler's backend knowledge to not reorder code inappropriately,
        //       so we'll apply it conservatively.
        static if (order != MemoryOrder.raw)
        {
            version (D_InlineAsm_X86)
            {
                import core.cpuid;

                // TODO: review this implementation; it seems way overly complicated
                asm pure nothrow @nogc @trusted
                {
                    naked;

                    call sse2;
                    test AL, AL;
                    jne Lcpuid;

                    // Fast path: We have SSE2, so just use mfence.
                    mfence;
                    jmp Lend;

                Lcpuid:

                    // Slow path: We use cpuid to serialize. This is
                    // significantly slower than mfence, but is the
                    // only serialization facility we have available
                    // on older non-SSE2 chips.
                    push EBX;

                    mov EAX, 0;
                    cpuid;

                    pop EBX;

                Lend:

                    ret;
                }
            }
            else version (D_InlineAsm_X86_64)
            {
                asm pure nothrow @nogc @trusted
                {
                    naked;
                    mfence;
                    ret;
                }
            }
            else
                static assert (false, "Unsupported architecture.");
        }
    }

    void pause() pure nothrow @nogc @trusted
    {
        version (D_InlineAsm_X86)
        {
            asm pure nothrow @nogc @trusted
            {
                naked;
                rep; nop;
                ret;
            }
        }
        else version (D_InlineAsm_X86_64)
        {
            asm pure nothrow @nogc @trusted
            {
                naked;
    //            pause; // TODO: DMD should add this opcode to its inline asm
                rep; nop;
                ret;
            }
        }
        else
        {
            // ARM should `yield`
            // other architectures? otherwise some sort of nop...
        }
    }
}
else version (GNU)
{
    import gcc.builtins;
    import gcc.config;

    inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        static assert(order != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()");

        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == ubyte.sizeof)
            {
                ubyte value = __atomic_load_1(cast(shared)src, order);
                return *cast(typeof(return)*)&value;
            }
            else static if (T.sizeof == ushort.sizeof)
            {
                ushort value = __atomic_load_2(cast(shared)src, order);
                return *cast(typeof(return)*)&value;
            }
            else static if (T.sizeof == uint.sizeof)
            {
                uint value = __atomic_load_4(cast(shared)src, order);
                return *cast(typeof(return)*)&value;
            }
            else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
            {
                ulong value = __atomic_load_8(cast(shared)src, order);
                return *cast(typeof(return)*)&value;
            }
            else static if (GNU_Have_LibAtomic)
            {
                T value;
                __atomic_load(T.sizeof, cast(shared)src, &value, order);
                return *cast(typeof(return)*)&value;
            }
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            getAtomicMutex.lock();
            scope(exit) getAtomicMutex.unlock();
            return *cast(typeof(return)*)&src;
        }
    }

    void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore()");

        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == ubyte.sizeof)
                __atomic_store_1(cast(shared)dest, *cast(ubyte*)&value, order);
            else static if (T.sizeof == ushort.sizeof)
                __atomic_store_2(cast(shared)dest, *cast(ushort*)&value, order);
            else static if (T.sizeof == uint.sizeof)
                __atomic_store_4(cast(shared)dest, *cast(uint*)&value, order);
            else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
                __atomic_store_8(cast(shared)dest, *cast(ulong*)&value, order);
            else static if (GNU_Have_LibAtomic)
                __atomic_store(T.sizeof, cast(shared)dest, cast(void*)&value, order);
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            getAtomicMutex.lock();
            *dest = value;
            getAtomicMutex.unlock();
        }
    }

    T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (is(T : ulong))
    {
        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == ubyte.sizeof)
                return __atomic_fetch_add_1(cast(shared)dest, value, order);
            else static if (T.sizeof == ushort.sizeof)
                return __atomic_fetch_add_2(cast(shared)dest, value, order);
            else static if (T.sizeof == uint.sizeof)
                return __atomic_fetch_add_4(cast(shared)dest, value, order);
            else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
                return __atomic_fetch_add_8(cast(shared)dest, value, order);
            else static if (GNU_Have_LibAtomic)
                return __atomic_fetch_add(T.sizeof, cast(shared)dest, cast(void*)&value, order);
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            getAtomicMutex.lock();
            scope(exit) getAtomicMutex.unlock();
            T tmp = *dest;
            *dest += value;
            return tmp;
        }
    }

    T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (is(T : ulong))
    {
        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == ubyte.sizeof)
                return __atomic_fetch_sub_1(cast(shared)dest, value, order);
            else static if (T.sizeof == ushort.sizeof)
                return __atomic_fetch_sub_2(cast(shared)dest, value, order);
            else static if (T.sizeof == uint.sizeof)
                return __atomic_fetch_sub_4(cast(shared)dest, value, order);
            else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
                return __atomic_fetch_sub_8(cast(shared)dest, value, order);
            else static if (GNU_Have_LibAtomic)
                return __atomic_fetch_sub(T.sizeof, cast(shared)dest, cast(void*)&value, order);
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            getAtomicMutex.lock();
            scope(exit) getAtomicMutex.unlock();
            T tmp = *dest;
            *dest -= value;
            return tmp;
        }
    }

    T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
        if (is(T : ulong) || is(T == class) || is(T == interface) || is(T U : U*))
    {
        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == byte.sizeof)
            {
                ubyte res = __atomic_exchange_1(cast(shared)dest, *cast(ubyte*)&value, order);
                return *cast(typeof(return)*)&res;
            }
            else static if (T.sizeof == short.sizeof)
            {
                ushort res = __atomic_exchange_2(cast(shared)dest, *cast(ushort*)&value, order);
                return *cast(typeof(return)*)&res;
            }
            else static if (T.sizeof == int.sizeof)
            {
                uint res = __atomic_exchange_4(cast(shared)dest, *cast(uint*)&value, order);
                return *cast(typeof(return)*)&res;
            }
            else static if (T.sizeof == long.sizeof && GNU_Have_64Bit_Atomics)
            {
                ulong res = __atomic_exchange_8(cast(shared)dest, *cast(ulong*)&value, order);
                return *cast(typeof(return)*)&res;
            }
            else static if (GNU_Have_LibAtomic)
            {
                T res = void;
                __atomic_exchange(T.sizeof, cast(shared)dest, cast(void*)&value, &res, order);
                return res;
            }
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            getAtomicMutex.lock();
            scope(exit) getAtomicMutex.unlock();

            T res = *dest;
            *dest = value;
            return res;
        }
    }

    bool atomicCompareExchangeWeak(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        return atomicCompareExchangeImpl!(succ, fail, true)(dest, compare, value);
    }

    bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        return atomicCompareExchangeImpl!(succ, fail, false)(dest, compare, value);
    }

    bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        return atomicCompareExchangeImpl!(succ, fail, false)(dest, cast(T*)&compare, value);
    }

    bool atomicCompareExchangeWeakNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        return atomicCompareExchangeImpl!(succ, fail, true)(dest, cast(T*)&compare, value);
    }

    private bool atomicCompareExchangeImpl(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, bool weak, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
        if (CanCAS!T)
    {
        bool res = void;

        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
        {
            static if (T.sizeof == byte.sizeof)
                res = __atomic_compare_exchange_1(cast(shared)dest, compare, *cast(ubyte*)&value,
                                                  weak, succ, fail);
            else static if (T.sizeof == short.sizeof)
                res = __atomic_compare_exchange_2(cast(shared)dest, compare, *cast(ushort*)&value,
                                                  weak, succ, fail);
            else static if (T.sizeof == int.sizeof)
                res = __atomic_compare_exchange_4(cast(shared)dest, compare, *cast(uint*)&value,
                                                  weak, succ, fail);
            else static if (T.sizeof == long.sizeof && GNU_Have_64Bit_Atomics)
                res = __atomic_compare_exchange_8(cast(shared)dest, compare, *cast(ulong*)&value,
                                                  weak, succ, fail);
            else static if (GNU_Have_LibAtomic)
                res = __atomic_compare_exchange(T.sizeof, cast(shared)dest, compare, cast(void*)&value,
                                                succ, fail);
            else
                static assert(0, "Invalid template type specified.");
        }
        else
        {
            static if (T.sizeof == byte.sizeof)
                alias U = byte;
            else static if (T.sizeof == short.sizeof)
                alias U = short;
            else static if (T.sizeof == int.sizeof)
                alias U = int;
            else static if (T.sizeof == long.sizeof)
                alias U = long;
            else
                static assert(0, "Invalid template type specified.");

            getAtomicMutex.lock();
            scope(exit) getAtomicMutex.unlock();

            if (*cast(U*)dest == *cast(U*)&compare)
            {
                *dest = value;
                res = true;
            }
            else
            {
                *compare = *dest;
                res = false;
            }
        }

        return res;
    }

    void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted
    {
        static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
            __atomic_thread_fence(order);
        else
        {
            getAtomicMutex.lock();
            getAtomicMutex.unlock();
        }
    }

    void pause() pure nothrow @nogc @trusted
    {
        version (X86)
        {
            __builtin_ia32_pause();
        }
        else version (X86_64)
        {
            __builtin_ia32_pause();
        }
        else
        {
            // Other architectures? Some sort of nop or barrier.
        }
    }

    static if (!GNU_Have_Atomics && !GNU_Have_LibAtomic)
    {
        // Use system mutex for atomics, faking the purity of the functions so
        // that they can be used in pure/nothrow/@safe code.
        extern (C) private pure @trusted @nogc nothrow
        {
            static if (GNU_Thread_Model == ThreadModel.Posix)
            {
                import core.sys.posix.pthread;
                alias atomicMutexHandle = pthread_mutex_t;

                pragma(mangle, "pthread_mutex_init") int fakePureMutexInit(pthread_mutex_t*, pthread_mutexattr_t*);
                pragma(mangle, "pthread_mutex_lock") int fakePureMutexLock(pthread_mutex_t*);
                pragma(mangle, "pthread_mutex_unlock") int fakePureMutexUnlock(pthread_mutex_t*);
            }
            else static if (GNU_Thread_Model == ThreadModel.Win32)
            {
                import core.sys.windows.winbase;
                alias atomicMutexHandle = CRITICAL_SECTION;

                pragma(mangle, "InitializeCriticalSection") int fakePureMutexInit(CRITICAL_SECTION*);
                pragma(mangle, "EnterCriticalSection") void fakePureMutexLock(CRITICAL_SECTION*);
                pragma(mangle, "LeaveCriticalSection") int fakePureMutexUnlock(CRITICAL_SECTION*);
            }
            else
            {
                alias atomicMutexHandle = int;
            }
        }

        // Implements lock/unlock operations.
        private struct AtomicMutex
        {
            int lock() pure @trusted @nogc nothrow
            {
                static if (GNU_Thread_Model == ThreadModel.Posix)
                {
                    if (!_inited)
                    {
                        fakePureMutexInit(&_handle, null);
                        _inited = true;
                    }
                    return fakePureMutexLock(&_handle);
                }
                else
                {
                    static if (GNU_Thread_Model == ThreadModel.Win32)
                    {
                        if (!_inited)
                        {
                            fakePureMutexInit(&_handle);
                            _inited = true;
                        }
                        fakePureMutexLock(&_handle);
                    }
                    return 0;
                }
            }

            int unlock() pure @trusted @nogc nothrow
            {
                static if (GNU_Thread_Model == ThreadModel.Posix)
                    return fakePureMutexUnlock(&_handle);
                else
                {
                    static if (GNU_Thread_Model == ThreadModel.Win32)
                        fakePureMutexUnlock(&_handle);
                    return 0;
                }
            }

        private:
            atomicMutexHandle _handle;
            bool _inited;
        }

        // Internal static mutex reference.
        private AtomicMutex* _getAtomicMutex() @trusted @nogc nothrow
        {
            __gshared static AtomicMutex mutex;
            return &mutex;
        }

        // Pure alias for _getAtomicMutex.
        pragma(mangle, _getAtomicMutex.mangleof)
        private AtomicMutex* getAtomicMutex() pure @trusted @nogc nothrow @property;
    }
}

private:

version (Windows)
{
    enum RegisterReturn(T) = is(T : U[], U) || is(T : R delegate(A), R, A...);
}

enum CanCAS(T) = is(T : ulong) ||
                 is(T == class) ||
                 is(T == interface) ||
                 is(T : U*, U) ||
                 is(T : U[], U) ||
                 is(T : R delegate(A), R, A...) ||
                 (is(T == struct) && __traits(isPOD, T) &&
                  (T.sizeof <= size_t.sizeof*2 ||       // no more than 2 words
                   (T.sizeof == 16 && has128BitCAS)) && // or supports 128-bit CAS
                  (T.sizeof & (T.sizeof - 1)) == 0      // is power of 2
                 );

template IntOrLong(T)
{
    static if (T.sizeof > 4)
        alias IntOrLong = long;
    else
        alias IntOrLong = int;
}

// NOTE: x86 loads implicitly have acquire semantics so a memory
//       barrier is only necessary on releases.
template needsLoadBarrier( MemoryOrder ms )
{
    enum bool needsLoadBarrier = ms == MemoryOrder.seq;
}


// NOTE: x86 stores implicitly have release semantics so a memory
//       barrier is only necessary on acquires.
template needsStoreBarrier( MemoryOrder ms )
{
    enum bool needsStoreBarrier = ms == MemoryOrder.seq;
}

// this is a helper to build asm blocks
string simpleFormat(string format, scope string[] args)
{
    string result;
    outer: while (format.length)
    {
        foreach (i; 0 .. format.length)
        {
            if (format[i] == '%' || format[i] == '?')
            {
                bool isQ = format[i] == '?';
                result ~= format[0 .. i++];
                assert (i < format.length, "Invalid format string");
                if (format[i] == '%' || format[i] == '?')
                {
                    assert(!isQ, "Invalid format string");
                    result ~= format[i++];
                }
                else
                {
                    int index = 0;
                    assert (format[i] >= '0' && format[i] <= '9', "Invalid format string");
                    while (i < format.length && format[i] >= '0' && format[i] <= '9')
                        index = index * 10 + (ubyte(format[i++]) - ubyte('0'));
                    if (!isQ)
                        result ~= args[index];
                    else if (!args[index])
                    {
                        size_t j = i;
                        for (; j < format.length;)
                        {
                            if (format[j++] == '\n')
                                break;
                        }
                        i = j;
                    }
                }
                format = format[i .. $];
                continue outer;
            }
        }
        result ~= format;
        break;
    }
    return result;
}