| // GNU D Compiler emulated TLS routines. |
| // Copyright (C) 2019-2022 Free Software Foundation, Inc. |
| |
| // GCC is free software; you can redistribute it and/or modify it under |
| // the terms of the GNU General Public License as published by the Free |
| // Software Foundation; either version 3, or (at your option) any later |
| // version. |
| |
| // GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| // WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| // for more details. |
| |
| // Under Section 7 of GPL version 3, you are granted additional |
| // permissions described in the GCC Runtime Library Exception, version |
| // 3.1, as published by the Free Software Foundation. |
| |
| // You should have received a copy of the GNU General Public License and |
| // a copy of the GCC Runtime Library Exception along with this program; |
| // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| // <http://www.gnu.org/licenses/>. |
| |
| // This code is based on the libgcc emutls.c emulated TLS support. |
| |
| module gcc.emutls; |
| |
| import core.atomic, core.stdc.stdlib, core.stdc.string, core.sync.mutex; |
| import core.internal.container.array; |
| import core.internal.container.hashtab; |
| import core.internal.traits : classInstanceAlignment; |
| import gcc.builtins, gcc.gthread; |
| |
| version (GNU_EMUTLS): private: |
| |
| alias word = __builtin_machine_uint; |
| alias pointer = __builtin_pointer_uint; |
| alias TlsArray = Array!(void**); |
| |
| /* |
| * TLS control data emitted by GCC for every TLS variable. |
| */ |
| struct __emutls_object |
| { |
| word size; |
| word align_; |
| union |
| { |
| pointer offset; |
| void* ptr; |
| } |
| |
| ubyte* templ; |
| } |
| |
| // Per-thread key to obtain the per-thread TLS variable array |
| __gshared __gthread_key_t emutlsKey; |
| // Largest, currently assigned TLS variable offset |
| __gshared pointer emutlsMaxOffset = 0; |
| // Contains the size of the TLS variables (for GC) |
| __gshared Array!word emutlsSizes; |
| // Contains the TLS variable array for single-threaded apps |
| __gshared TlsArray singleArray; |
| // List of all currently alive TlsArrays (for GC) |
| __gshared HashTab!(TlsArray*, TlsArray*) emutlsArrays; |
| |
| // emutlsMutex Mutex + @nogc handling |
| enum mutexAlign = classInstanceAlignment!Mutex; |
| enum mutexClassInstanceSize = __traits(classInstanceSize, Mutex); |
| __gshared align(mutexAlign) void[mutexClassInstanceSize] _emutlsMutex; |
| |
| @property Mutex emutlsMutex() nothrow @nogc |
| { |
| return cast(Mutex) _emutlsMutex.ptr; |
| } |
| |
| /* |
| * Global (de)initialization functions |
| */ |
| extern (C) void _d_emutls_init() nothrow @nogc |
| { |
| memcpy(_emutlsMutex.ptr, typeid(Mutex).initializer.ptr, _emutlsMutex.length); |
| (cast(Mutex) _emutlsMutex.ptr).__ctor(); |
| |
| if (__gthread_key_create(&emutlsKey, &emutlsDestroyThread) != 0) |
| abort(); |
| } |
| |
| __gshared __gthread_once_t initOnce = GTHREAD_ONCE_INIT; |
| |
| /* |
| * emutls main entrypoint, called by GCC for each TLS variable access. |
| */ |
| extern (C) void* __emutls_get_address(shared __emutls_object* obj) nothrow @nogc |
| { |
| pointer offset; |
| if (__gthread_active_p()) |
| { |
| // Obtain the offset index into the TLS array (same for all-threads) |
| // for requested var. If it is unset, obtain a new offset index. |
| offset = atomicLoad!(MemoryOrder.acq, pointer)(obj.offset); |
| if (__builtin_expect(offset == 0, 0)) |
| { |
| __gthread_once(&initOnce, &_d_emutls_init); |
| emutlsMutex.lock_nothrow(); |
| |
| offset = obj.offset; |
| if (offset == 0) |
| { |
| offset = ++emutlsMaxOffset; |
| |
| emutlsSizes.ensureLength(offset); |
| // Note: it's important that we copy any data from obj and |
| // do not keep an reference to obj itself: If a library is |
| // unloaded, its tls variables are not removed from the arrays |
| // and the GC will still scan these. If we then try to reference |
| // a pointer to the data segment of an unloaded library, this |
| // will crash. |
| emutlsSizes[offset - 1] = obj.size; |
| |
| atomicStore!(MemoryOrder.rel, pointer)(obj.offset, offset); |
| } |
| emutlsMutex.unlock_nothrow(); |
| } |
| } |
| // For single-threaded systems, don't synchronize |
| else |
| { |
| if (__builtin_expect(obj.offset == 0, 0)) |
| { |
| offset = ++emutlsMaxOffset; |
| |
| emutlsSizes.ensureLength(offset); |
| emutlsSizes[offset - 1] = obj.size; |
| |
| obj.offset = offset; |
| } |
| } |
| |
| TlsArray* arr; |
| if (__gthread_active_p()) |
| arr = cast(TlsArray*) __gthread_getspecific(emutlsKey); |
| else |
| arr = &singleArray; |
| |
| // This will always be false for singleArray |
| if (__builtin_expect(arr == null, 0)) |
| { |
| arr = mallocTlsArray(offset); |
| __gthread_setspecific(emutlsKey, arr); |
| emutlsMutex.lock_nothrow(); |
| emutlsArrays[arr] = arr; |
| emutlsMutex.unlock_nothrow(); |
| } |
| // Check if we have to grow the per-thread array |
| else if (__builtin_expect(offset > arr.length, 0)) |
| { |
| (*arr).ensureLength(offset); |
| } |
| |
| // Offset 0 is used as a not-initialized marker above. In the |
| // TLS array, we start at 0. |
| auto index = offset - 1; |
| |
| // Get the per-thread pointer from the TLS array |
| void** ret = (*arr)[index]; |
| if (__builtin_expect(ret == null, 0)) |
| { |
| // Initial access, have to allocate the storage |
| ret = emutlsAlloc(obj); |
| (*arr)[index] = ret; |
| } |
| |
| return ret; |
| } |
| |
| // 1:1 copy from libgcc emutls.c |
| extern (C) void __emutls_register_common(__emutls_object* obj, word size, word align_, ubyte* templ) nothrow @nogc |
| { |
| if (obj.size < size) |
| { |
| obj.size = size; |
| obj.templ = null; |
| } |
| if (obj.align_ < align_) |
| obj.align_ = align_; |
| if (templ && size == obj.size) |
| obj.templ = templ; |
| } |
| |
| // 1:1 copy from libgcc emutls.c |
| void** emutlsAlloc(shared __emutls_object* obj) nothrow @nogc |
| { |
| void* ptr; |
| void* ret; |
| enum pointerSize = (void*).sizeof; |
| |
| /* We could use here posix_memalign if available and adjust |
| emutls_destroy accordingly. */ |
| if ((cast() obj).align_ <= pointerSize) |
| { |
| ptr = malloc((cast() obj).size + pointerSize); |
| if (ptr == null) |
| abort(); |
| (cast(void**) ptr)[0] = ptr; |
| ret = ptr + pointerSize; |
| } |
| else |
| { |
| ptr = malloc(obj.size + pointerSize + obj.align_ - 1); |
| if (ptr == null) |
| abort(); |
| ret = cast(void*)((cast(pointer)(ptr + pointerSize + obj.align_ - 1)) & ~cast( |
| pointer)(obj.align_ - 1)); |
| (cast(void**) ret)[-1] = ptr; |
| } |
| |
| if (obj.templ) |
| memcpy(ret, cast(ubyte*) obj.templ, cast() obj.size); |
| else |
| memset(ret, 0, cast() obj.size); |
| |
| return cast(void**) ret; |
| } |
| |
| /* |
| * When a thread has finished, free all allocated TLS variables and empty the |
| * array. The pointer is not free'd as it is stil referenced by the GC scan |
| * list emutlsArrays, which gets destroyed when druntime is unloaded. |
| */ |
| extern (C) void emutlsDestroyThread(void* ptr) nothrow @nogc |
| { |
| auto arr = cast(TlsArray*) ptr; |
| |
| foreach (entry; *arr) |
| { |
| if (entry) |
| free(entry[-1]); |
| } |
| |
| arr.length = 0; |
| } |
| |
| /* |
| * Allocate a new TLS array, set length according to offset. |
| */ |
| TlsArray* mallocTlsArray(pointer offset = 0) nothrow @nogc |
| { |
| static assert(TlsArray.alignof == (void*).alignof); |
| void[] data = malloc(TlsArray.sizeof)[0 .. TlsArray.sizeof]; |
| if (data.ptr == null) |
| abort(); |
| |
| static immutable TlsArray init = TlsArray.init; |
| memcpy(data.ptr, &init, data.length); |
| (cast(TlsArray*) data).length = 32; |
| return cast(TlsArray*) data.ptr; |
| } |
| |
| /* |
| * Make sure array is large enough to hold an entry for offset. |
| * Note: the array index will be offset - 1! |
| */ |
| void ensureLength(Value)(ref Array!(Value) arr, size_t offset) nothrow @nogc |
| { |
| // index is offset-1 |
| if (offset > arr.length) |
| { |
| auto newSize = arr.length * 2; |
| if (offset > newSize) |
| newSize = offset + 32; |
| arr.length = newSize; |
| } |
| } |
| |
| // Public interface |
| public: |
| void _d_emutls_scan(scope void delegate(void* pbeg, void* pend) nothrow cb) nothrow |
| { |
| void scanArray(scope TlsArray* arr) nothrow |
| { |
| foreach (index, entry; *arr) |
| { |
| auto ptr = cast(void*) entry; |
| if (ptr) |
| cb(ptr, ptr + emutlsSizes[index]); |
| } |
| } |
| |
| __gthread_once(&initOnce, &_d_emutls_init); |
| emutlsMutex.lock_nothrow(); |
| // this code is effectively nothrow |
| try |
| { |
| foreach (arr, value; emutlsArrays) |
| { |
| scanArray(arr); |
| } |
| } |
| catch (Exception) |
| { |
| } |
| emutlsMutex.unlock_nothrow(); |
| scanArray(&singleArray); |
| } |
| |
| // Call this after druntime has been unloaded |
| void _d_emutls_destroy() nothrow @nogc |
| { |
| (cast(Mutex) _emutlsMutex.ptr).__dtor(); |
| destroy(emutlsArrays); |
| } |