2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 #include <folly/ThreadLocal.h>
21 namespace folly { namespace threadlocal_detail {
23 StaticMetaBase::StaticMetaBase(ThreadEntry* (*threadEntry)(), bool strict)
24 : nextId_(1), threadEntry_(threadEntry), strict_(strict) {
25 head_.next = head_.prev = &head_;
26 int ret = pthread_key_create(&pthreadKey_, &onThreadExit);
27 checkPosixError(ret, "pthread_key_create failed");
28 PthreadKeyUnregister::registerKey(pthreadKey_);
31 void StaticMetaBase::onThreadExit(void* ptr) {
32 #ifdef FOLLY_TLD_USE_FOLLY_TLS
33 auto threadEntry = static_cast<ThreadEntry*>(ptr);
35 std::unique_ptr<ThreadEntry> threadEntry(static_cast<ThreadEntry*>(ptr));
37 DCHECK_GT(threadEntry->elementsCapacity, 0u);
38 auto& meta = *threadEntry->meta;
40 // Make sure this ThreadEntry is available if ThreadLocal A is accessed in
41 // ThreadLocal B destructor.
42 pthread_setspecific(meta.pthreadKey_, &(*threadEntry));
44 pthread_setspecific(meta.pthreadKey_, nullptr);
48 SharedMutex::ReadHolder rlock;
50 rlock = SharedMutex::ReadHolder(meta.accessAllThreadsLock_);
53 std::lock_guard<std::mutex> g(meta.lock_);
54 meta.erase(&(*threadEntry));
55 // No need to hold the lock any longer; the ThreadEntry is private to this
56 // thread now that it's been removed from meta.
58 // NOTE: User-provided deleter / object dtor itself may be using ThreadLocal
59 // with the same Tag, so dispose() calls below may (re)create some of the
60 // elements or even increase elementsCapacity, thus multiple cleanup rounds
62 for (bool shouldRun = true; shouldRun;) {
64 FOR_EACH_RANGE (i, 0, threadEntry->elementsCapacity) {
65 if (threadEntry->elements[i].dispose(TLPDestructionMode::THIS_THREAD)) {
71 free(threadEntry->elements);
72 threadEntry->elements = nullptr;
73 threadEntry->meta = nullptr;
76 uint32_t StaticMetaBase::allocate(EntryID* ent) {
79 std::lock_guard<std::mutex> g(meta.lock_);
81 id = ent->value.load();
82 if (id != kEntryIDInvalid) {
86 if (!meta.freeIds_.empty()) {
87 id = meta.freeIds_.back();
88 meta.freeIds_.pop_back();
93 uint32_t old_id = ent->value.exchange(id);
94 DCHECK_EQ(old_id, kEntryIDInvalid);
98 void StaticMetaBase::destroy(EntryID* ent) {
102 // Elements in other threads that use this id.
103 std::vector<ElementWrapper> elements;
106 SharedMutex::WriteHolder wlock;
109 * In strict mode, the logic guarantees per-thread instances are
110 * destroyed by the moment ThreadLocal<> dtor returns.
111 * In order to achieve that, we should wait until concurrent
112 * onThreadExit() calls (that might acquire ownership over per-thread
113 * instances in order to destroy them) are finished.
115 wlock = SharedMutex::WriteHolder(meta.accessAllThreadsLock_);
119 std::lock_guard<std::mutex> g(meta.lock_);
120 uint32_t id = ent->value.exchange(kEntryIDInvalid);
121 if (id == kEntryIDInvalid) {
125 for (ThreadEntry* e = meta.head_.next; e != &meta.head_; e = e->next) {
126 if (id < e->elementsCapacity && e->elements[id].ptr) {
127 elements.push_back(e->elements[id]);
130 * Writing another thread's ThreadEntry from here is fine;
131 * the only other potential reader is the owning thread --
132 * from onThreadExit (which grabs the lock, so is properly
133 * synchronized with us) or from get(), which also grabs
134 * the lock if it needs to resize the elements vector.
136 * We can't conflict with reads for a get(id), because
137 * it's illegal to call get on a thread local that's
140 e->elements[id].ptr = nullptr;
141 e->elements[id].deleter1 = nullptr;
142 e->elements[id].ownsDeleter = false;
145 meta.freeIds_.push_back(id);
148 // Delete elements outside the locks.
149 for (ElementWrapper& elem : elements) {
150 elem.dispose(TLPDestructionMode::ALL_THREADS);
152 } catch (...) { // Just in case we get a lock error or something anyway...
153 LOG(WARNING) << "Destructor discarding an exception that was thrown.";
158 * Reserve enough space in the ThreadEntry::elements for the item
161 void StaticMetaBase::reserve(EntryID* id) {
163 ThreadEntry* threadEntry = (*threadEntry_)();
164 size_t prevCapacity = threadEntry->elementsCapacity;
166 uint32_t idval = id->getOrAllocate(meta);
167 if (prevCapacity > idval) {
170 // Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
172 size_t newCapacity = static_cast<size_t>((idval + 5) * 1.7);
173 assert(newCapacity > prevCapacity);
174 ElementWrapper* reallocated = nullptr;
176 // Need to grow. Note that we can't call realloc, as elements is
177 // still linked in meta, so another thread might access invalid memory
178 // after realloc succeeds. We'll copy by hand and update our ThreadEntry
180 if (usingJEMalloc()) {
181 bool success = false;
182 size_t newByteSize = nallocx(newCapacity * sizeof(ElementWrapper), 0);
184 // Try to grow in place.
186 // Note that xallocx(MALLOCX_ZERO) will only zero newly allocated memory,
187 // even if a previous allocation allocated more than we requested.
188 // This is fine; we always use MALLOCX_ZERO with jemalloc and we
189 // always expand our allocation to the real size.
190 if (prevCapacity * sizeof(ElementWrapper) >= jemallocMinInPlaceExpandable) {
192 (xallocx(threadEntry->elements, newByteSize, 0, MALLOCX_ZERO) ==
196 // In-place growth failed.
199 ((reallocated = static_cast<ElementWrapper*>(
200 mallocx(newByteSize, MALLOCX_ZERO))) != nullptr);
204 // Expand to real size
205 assert(newByteSize / sizeof(ElementWrapper) >= newCapacity);
206 newCapacity = newByteSize / sizeof(ElementWrapper);
208 throw std::bad_alloc();
210 } else { // no jemalloc
211 // calloc() is simpler than malloc() followed by memset(), and
212 // potentially faster when dealing with a lot of memory, as it can get
213 // already-zeroed pages from the kernel.
214 reallocated = static_cast<ElementWrapper*>(
215 calloc(newCapacity, sizeof(ElementWrapper)));
217 throw std::bad_alloc();
221 // Success, update the entry
223 std::lock_guard<std::mutex> g(meta.lock_);
225 if (prevCapacity == 0) {
226 meta.push_back(threadEntry);
231 * Note: we need to hold the meta lock when copying data out of
232 * the old vector, because some other thread might be
233 * destructing a ThreadLocal and writing to the elements vector
236 if (prevCapacity != 0) {
239 threadEntry->elements,
240 sizeof(*reallocated) * prevCapacity);
242 std::swap(reallocated, threadEntry->elements);
244 threadEntry->elementsCapacity = newCapacity;
253 folly::Function<void()> prepare;
254 folly::Function<void()> parent;
255 folly::Function<void()> child;
260 static AtForkList& instance() {
261 static auto instance = new AtForkList();
265 static void prepare() noexcept {
266 instance().tasksLock.lock();
267 auto& tasks = instance().tasks;
268 for (auto task = tasks.rbegin(); task != tasks.rend(); ++task) {
273 static void parent() noexcept {
274 auto& tasks = instance().tasks;
275 for (auto& task : tasks) {
278 instance().tasksLock.unlock();
281 static void child() noexcept {
282 auto& tasks = instance().tasks;
283 for (auto& task : tasks) {
286 instance().tasksLock.unlock();
289 std::mutex tasksLock;
290 std::list<AtForkTask> tasks;
294 #if FOLLY_HAVE_PTHREAD_ATFORK
295 int ret = pthread_atfork(
296 &AtForkList::prepare, &AtForkList::parent, &AtForkList::child);
297 checkPosixError(ret, "pthread_atfork failed");
298 #elif !__ANDROID__ && !defined(_MSC_VER)
299 // pthread_atfork is not part of the Android NDK at least as of n9d. If
300 // something is trying to call native fork() directly at all with Android's
301 // process management model, this is probably the least of the problems.
303 // But otherwise, this is a problem.
304 #warning pthread_atfork unavailable
310 void StaticMetaBase::initAtFork() {
311 AtForkList::instance();
314 void StaticMetaBase::registerAtFork(
315 folly::Function<void()> prepare,
316 folly::Function<void()> parent,
317 folly::Function<void()> child) {
318 std::lock_guard<std::mutex> lg(AtForkList::instance().tasksLock);
319 AtForkList::instance().tasks.push_back(
320 {std::move(prepare), std::move(parent), std::move(child)});
323 FOLLY_STATIC_CTOR_PRIORITY_MAX
324 PthreadKeyUnregister PthreadKeyUnregister::instance_;