2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 /* override-include-guard */
19 #error "This should only be included by hazptr.h"
22 /* quality of implementation switches */
24 // NOTE: The #ifndef pattern is prone to ODR violation. Its use for
25 // quality of implementation options is temporary. Eventually these
26 // options should be added to the API in future API extensions.
29 #define HAZPTR_AMB true
33 #define HAZPTR_TC true
36 #ifndef HAZPTR_TC_SIZE
37 #define HAZPTR_TC_SIZE 10
41 #define HAZPTR_PRIV true
44 #ifndef HAZPTR_ONE_DOMAIN
45 #define HAZPTR_ONE_DOMAIN false
48 #ifndef HAZPTR_SCAN_MULT
49 #define HAZPTR_SCAN_MULT 2
52 #ifndef HAZPTR_SCAN_THRESHOLD
53 #define HAZPTR_SCAN_THRESHOLD 1000
58 #define HAZPTR_STATS false
61 #include <folly/concurrency/CacheLocality.h>
62 #include <folly/experimental/AsymmetricMemoryBarrier.h>
63 #include <folly/experimental/hazptr/debug.h>
65 #include <mutex> // for thread caching
66 #include <unordered_set> // for hash set in bulk reclamation
72 * Helper classes and functions
80 #define INC_HAZPTR_STATS(x) hazptr_stats_.x()
82 #define INC_HAZPTR_STATS(x)
99 enum hazptr_tls_state { TLS_ALIVE, TLS_UNINITIALIZED, TLS_DESTROYED };
101 /** hazptr_tc structures
102 * Thread caching of hazptr_rec-s that belong to the default domain.
105 struct hazptr_tc_entry {
108 void fill(hazptr_rec* hprec);
114 std::is_trivial<hazptr_tc_entry>::value,
115 "hazptr_tc_entry must be trivial"
116 " to avoid a branch to check initialization");
119 hazptr_tc_entry entry_[HAZPTR_TC_SIZE];
126 hazptr_tc_entry& operator[](size_t i);
128 bool put(hazptr_rec* hprec);
133 std::is_trivial<hazptr_tc>::value,
134 "hazptr_tc must be trivial to avoid a branch to check initialization");
136 hazptr_tc* hazptr_tc_tls();
137 void hazptr_tc_init();
138 void hazptr_tc_shutdown();
139 hazptr_rec* hazptr_tc_try_get();
140 bool hazptr_tc_try_put(hazptr_rec* hprec);
142 /** hazptr_priv structures
143 * Thread private lists of retired objects that belong to the default domain.
152 void push(hazptr_obj* obj);
153 void pushAllToDomain();
157 std::is_trivial<hazptr_priv>::value,
158 "hazptr_priv must be trivial to avoid a branch to check initialization");
160 void hazptr_priv_init();
161 void hazptr_priv_shutdown();
162 bool hazptr_priv_try_retire(hazptr_obj* obj);
164 /** hazptr_tls_life */
166 struct hazptr_tls_life {
171 void tls_life_odr_use();
175 extern thread_local hazptr_tls_state tls_state_;
176 extern thread_local hazptr_tc tls_tc_data_;
177 extern thread_local hazptr_priv tls_priv_data_;
178 extern thread_local hazptr_tls_life tls_life_; // last
184 inline constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept
191 template <typename T, typename D>
192 inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) {
193 DEBUG_PRINT(this << " " << &domain);
194 deleter_ = std::move(deleter);
195 reclaim_ = [](hazptr_obj* p) {
196 auto hobp = static_cast<hazptr_obj_base*>(p);
197 auto obj = static_cast<T*>(hobp);
201 (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) {
202 if (hazptr_priv_try_retire(this)) {
206 domain.objRetire(this);
214 friend class hazptr_domain;
215 friend class hazptr_holder;
216 friend struct hazptr_tc_entry;
218 FOLLY_ALIGN_TO_AVOID_FALSE_SHARING
219 std::atomic<const void*> hazptr_{nullptr};
220 hazptr_rec* next_{nullptr};
221 std::atomic<bool> active_{false};
223 void set(const void* p) noexcept;
224 const void* get() const noexcept;
225 void clear() noexcept;
226 bool isActive() noexcept;
227 bool tryAcquire() noexcept;
228 void release() noexcept;
235 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_domain& domain) {
239 (HAZPTR_ONE_DOMAIN || &domain == &default_hazptr_domain()))) {
240 auto hprec = hazptr_tc_try_get();
241 if (LIKELY(hprec != nullptr)) {
243 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
247 hazptr_ = domain_->hazptrAcquire();
248 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
249 if (hazptr_ == nullptr) { std::bad_alloc e; throw e; }
252 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(std::nullptr_t) noexcept {
255 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
258 FOLLY_ALWAYS_INLINE hazptr_holder::~hazptr_holder() {
260 if (LIKELY(hazptr_ != nullptr)) {
264 (HAZPTR_ONE_DOMAIN || domain_ == &default_hazptr_domain()))) {
265 if (LIKELY(hazptr_tc_try_put(hazptr_))) {
269 domain_->hazptrRelease(hazptr_);
273 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept {
274 domain_ = rhs.domain_;
275 hazptr_ = rhs.hazptr_;
276 rhs.domain_ = nullptr;
277 rhs.hazptr_ = nullptr;
281 hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept {
282 /* Self-move is a no-op. */
283 if (LIKELY(this != &rhs)) {
284 this->~hazptr_holder();
285 new (this) hazptr_holder(std::move(rhs));
290 template <typename T>
291 FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect(
293 const std::atomic<T*>& src) noexcept {
294 return try_protect(ptr, src, [](T* t) { return t; });
297 template <typename T, typename Func>
298 FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect(
300 const std::atomic<T*>& src,
302 DEBUG_PRINT(this << " " << ptr << " " << &src);
304 /*** Full fence ***/ hazptr_mb::light();
305 T* p = src.load(std::memory_order_acquire);
306 if (UNLIKELY(p != ptr)) {
314 template <typename T>
315 FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected(
316 const std::atomic<T*>& src) noexcept {
317 return get_protected(src, [](T* t) { return t; });
320 template <typename T, typename Func>
321 FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected(
322 const std::atomic<T*>& src,
324 T* p = src.load(std::memory_order_relaxed);
325 while (!try_protect(p, src, f)) {
327 DEBUG_PRINT(this << " " << p << " " << &src);
331 template <typename T>
332 FOLLY_ALWAYS_INLINE void hazptr_holder::reset(const T* ptr) noexcept {
333 auto p = static_cast<hazptr_obj*>(const_cast<T*>(ptr));
334 DEBUG_PRINT(this << " " << ptr << " p:" << p);
335 DCHECK(hazptr_); // UB if *this is empty
339 FOLLY_ALWAYS_INLINE void hazptr_holder::reset(std::nullptr_t) noexcept {
341 DCHECK(hazptr_); // UB if *this is empty
345 FOLLY_ALWAYS_INLINE void hazptr_holder::swap(hazptr_holder& rhs) noexcept {
347 this << " " << this->hazptr_ << " " << this->domain_ << " -- "
348 << &rhs << " " << rhs.hazptr_ << " " << rhs.domain_);
349 if (!HAZPTR_ONE_DOMAIN) {
350 std::swap(this->domain_, rhs.domain_);
352 std::swap(this->hazptr_, rhs.hazptr_);
355 FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept {
364 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array() {
365 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
367 auto ptc = hazptr_tc_tls();
368 if (LIKELY(ptc != nullptr)) {
370 auto count = tc.count();
372 size_t offset = count - M;
373 for (size_t i = 0; i < M; ++i) {
374 auto hprec = tc[offset + i].hprec_;
375 DCHECK(hprec != nullptr);
376 DEBUG_PRINT(i << " " << &h[i]);
377 new (&h[i]) hazptr_holder(nullptr);
378 h[i].hazptr_ = hprec;
380 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
388 for (size_t i = 0; i < M; ++i) {
389 new (&h[i]) hazptr_holder;
391 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
396 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array(
397 hazptr_array&& other) noexcept {
398 DEBUG_PRINT(this << " " << M << " " << &other);
399 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
400 for (size_t i = 0; i < M; ++i) {
401 new (&h[i]) hazptr_holder(std::move(other.h_[i]));
402 DEBUG_PRINT(i << " " << &h[i] << " " << &other.h_[i]);
404 empty_ = other.empty_;
409 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array(std::nullptr_t) noexcept {
410 DEBUG_PRINT(this << " " << M);
411 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
412 for (size_t i = 0; i < M; ++i) {
413 new (&h[i]) hazptr_holder(nullptr);
414 DEBUG_PRINT(i << " " << &h[i]);
420 FOLLY_ALWAYS_INLINE hazptr_array<M>::~hazptr_array() {
424 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
426 auto ptc = hazptr_tc_tls();
427 if (LIKELY(ptc != nullptr)) {
429 auto count = tc.count();
430 if ((M <= HAZPTR_TC_SIZE) && (count + M <= HAZPTR_TC_SIZE)) {
431 for (size_t i = 0; i < M; ++i) {
432 tc[count + i].hprec_ = h[i].hazptr_;
433 DEBUG_PRINT(i << " " << &h[i]);
434 new (&h[i]) hazptr_holder(nullptr);
436 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
438 tc.count_ = count + M;
444 for (size_t i = 0; i < M; ++i) {
445 h[i].~hazptr_holder();
450 FOLLY_ALWAYS_INLINE hazptr_array<M>& hazptr_array<M>::operator=(
451 hazptr_array&& other) noexcept {
452 DEBUG_PRINT(this << " " << M << " " << &other);
453 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
454 for (size_t i = 0; i < M; ++i) {
455 h[i] = std::move(other[i]);
456 DEBUG_PRINT(i << " " << &h[i] << " " << &other[i]);
458 empty_ = other.empty_;
464 FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_array<M>::operator[](
466 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
476 FOLLY_ALWAYS_INLINE hazptr_local<M>::hazptr_local() {
477 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
479 auto ptc = hazptr_tc_tls();
480 if (LIKELY(ptc != nullptr)) {
482 auto count = tc.count();
489 for (size_t i = 0; i < M; ++i) {
490 auto hprec = tc[i].hprec_;
491 DCHECK(hprec != nullptr);
492 DEBUG_PRINT(i << " " << &h[i]);
493 new (&h[i]) hazptr_holder(nullptr);
494 h[i].hazptr_ = hprec;
496 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
503 need_destruct_ = true;
504 for (size_t i = 0; i < M; ++i) {
505 new (&h[i]) hazptr_holder;
507 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
512 FOLLY_ALWAYS_INLINE hazptr_local<M>::~hazptr_local() {
513 if (LIKELY(!need_destruct_)) {
515 auto ptc = hazptr_tc_tls();
516 DCHECK(ptc != nullptr);
524 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
525 for (size_t i = 0; i < M; ++i) {
526 h[i].~hazptr_holder();
531 FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_local<M>::operator[](
533 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
538 ////////////////////////////////////////////////////////////////////////////////
540 // - Control of reclamation (when and by whom)
541 // - End-to-end lock-free implementation
543 /** Definition of default_hazptr_domain() */
545 FOLLY_ALWAYS_INLINE hazptr_domain& default_hazptr_domain() {
546 DEBUG_PRINT(&default_domain_);
547 return default_domain_;
552 FOLLY_ALWAYS_INLINE void hazptr_rec::set(const void* p) noexcept {
553 DEBUG_PRINT(this << " " << p);
554 hazptr_.store(p, std::memory_order_release);
557 inline const void* hazptr_rec::get() const noexcept {
558 auto p = hazptr_.load(std::memory_order_acquire);
559 DEBUG_PRINT(this << " " << p);
563 FOLLY_ALWAYS_INLINE void hazptr_rec::clear() noexcept {
565 hazptr_.store(nullptr, std::memory_order_release);
568 inline bool hazptr_rec::isActive() noexcept {
569 return active_.load(std::memory_order_acquire);
572 inline bool hazptr_rec::tryAcquire() noexcept {
573 bool active = isActive();
575 active_.compare_exchange_strong(
576 active, true, std::memory_order_release, std::memory_order_relaxed)) {
583 inline void hazptr_rec::release() noexcept {
585 active_.store(false, std::memory_order_release);
590 inline const void* hazptr_obj::getObjPtr() const {
597 inline hazptr_domain::~hazptr_domain() {
599 { /* reclaim all remaining retired objects */
601 auto retired = retired_.exchange(nullptr);
603 for (auto p = retired; p; p = next) {
607 retired = retired_.exchange(nullptr);
610 /* Leak the data for the default domain to avoid destruction order
611 * issues with thread caches.
613 if (this != &default_hazptr_domain()) {
614 /* free all hazptr_rec-s */
616 for (auto p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
618 DCHECK(!p->isActive());
619 mr_->deallocate(static_cast<void*>(p), sizeof(hazptr_rec));
624 inline hazptr_rec* hazptr_domain::hazptrAcquire() {
627 for (p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
629 if (p->tryAcquire()) {
633 p = static_cast<hazptr_rec*>(mr_->allocate(sizeof(hazptr_rec)));
634 DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec));
638 p->active_.store(true, std::memory_order_relaxed);
639 p->next_ = hazptrs_.load(std::memory_order_acquire);
640 while (!hazptrs_.compare_exchange_weak(
641 p->next_, p, std::memory_order_release, std::memory_order_acquire)) {
644 auto hcount = hcount_.fetch_add(1);
645 DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec) << " " << hcount);
649 inline void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept {
650 DEBUG_PRINT(this << " " << p);
655 hazptr_domain::pushRetired(hazptr_obj* head, hazptr_obj* tail, int count) {
656 /*** Full fence ***/ hazptr_mb::light();
657 tail->next_ = retired_.load(std::memory_order_acquire);
658 while (!retired_.compare_exchange_weak(
661 std::memory_order_release,
662 std::memory_order_acquire)) {
664 return rcount_.fetch_add(count) + count;
667 inline bool hazptr_domain::reachedThreshold(int rcount) {
669 rcount >= HAZPTR_SCAN_THRESHOLD &&
670 rcount >= HAZPTR_SCAN_MULT * hcount_.load(std::memory_order_acquire));
673 inline void hazptr_domain::objRetire(hazptr_obj* p) {
674 auto rcount = pushRetired(p, p, 1);
675 if (reachedThreshold(rcount)) {
680 inline void hazptr_domain::tryBulkReclaim() {
683 auto hcount = hcount_.load(std::memory_order_acquire);
684 auto rcount = rcount_.load(std::memory_order_acquire);
685 if (rcount < HAZPTR_SCAN_THRESHOLD || rcount < HAZPTR_SCAN_MULT * hcount) {
688 if (rcount_.compare_exchange_weak(
689 rcount, 0, std::memory_order_release, std::memory_order_relaxed)) {
696 inline void hazptr_domain::bulkReclaim() {
698 /*** Full fence ***/ hazptr_mb::heavy();
699 auto p = retired_.exchange(nullptr, std::memory_order_acquire);
700 auto h = hazptrs_.load(std::memory_order_acquire);
701 std::unordered_set<const void*> hs; // TODO lock-free alternative
702 for (; h; h = h->next_) {
706 hazptr_obj* retired = nullptr;
707 hazptr_obj* tail = nullptr;
709 for (; p; p = next) {
711 if (hs.count(p->getObjPtr()) == 0) {
712 DEBUG_PRINT(this << " " << p << " " << p->reclaim_);
717 if (tail == nullptr) {
724 pushRetired(retired, tail, rcount);
738 std::atomic<uint64_t> light_{0};
739 std::atomic<uint64_t> heavy_{0};
740 std::atomic<uint64_t> seq_cst_{0};
743 extern hazptr_stats hazptr_stats_;
745 inline hazptr_stats::~hazptr_stats() {
746 DEBUG_PRINT(this << " light " << light_.load());
747 DEBUG_PRINT(this << " heavy " << heavy_.load());
748 DEBUG_PRINT(this << " seq_cst " << seq_cst_.load());
751 FOLLY_ALWAYS_INLINE void hazptr_stats::light() {
753 /* atomic */ ++light_;
757 inline void hazptr_stats::heavy() {
759 /* atomic */ ++heavy_;
763 inline void hazptr_stats::seq_cst() {
765 /* atomic */ ++seq_cst_;
771 FOLLY_ALWAYS_INLINE void hazptr_mb::light() {
774 folly::asymmetricLightBarrier();
775 INC_HAZPTR_STATS(light);
777 atomic_thread_fence(std::memory_order_seq_cst);
778 INC_HAZPTR_STATS(seq_cst);
782 inline void hazptr_mb::heavy() {
785 folly::asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
786 INC_HAZPTR_STATS(heavy);
788 atomic_thread_fence(std::memory_order_seq_cst);
789 INC_HAZPTR_STATS(seq_cst);
798 * hazptr_tc structures
801 /** hazptr_tc_entry */
803 FOLLY_ALWAYS_INLINE void hazptr_tc_entry::fill(hazptr_rec* hprec) {
805 DEBUG_PRINT(this << " " << hprec);
808 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_entry::get() {
810 DEBUG_PRINT(this << " " << hprec);
814 inline void hazptr_tc_entry::evict() {
817 DEBUG_PRINT(this << " " << hprec);
822 FOLLY_ALWAYS_INLINE hazptr_tc_entry& hazptr_tc::operator[](size_t i) {
823 DCHECK(i <= HAZPTR_TC_SIZE);
827 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc::get() {
828 if (LIKELY(count_ != 0)) {
829 auto hprec = entry_[--count_].get();
830 DEBUG_PRINT(this << " " << hprec);
833 DEBUG_PRINT(this << " nullptr");
837 FOLLY_ALWAYS_INLINE bool hazptr_tc::put(hazptr_rec* hprec) {
838 if (LIKELY(count_ < HAZPTR_TC_SIZE)) {
839 entry_[count_++].fill(hprec);
840 DEBUG_PRINT(this << " " << count_ - 1);
846 FOLLY_ALWAYS_INLINE size_t hazptr_tc::count() {
850 /** hazptr_tc free functions */
852 FOLLY_ALWAYS_INLINE hazptr_tc* hazptr_tc_tls() {
853 DEBUG_PRINT(tls_state_);
854 if (LIKELY(tls_state_ == TLS_ALIVE)) {
855 DEBUG_PRINT(tls_state_);
856 return &tls_tc_data_;
857 } else if (tls_state_ == TLS_UNINITIALIZED) {
859 return &tls_tc_data_;
864 inline void hazptr_tc_init() {
866 auto& tc = tls_tc_data_;
874 inline void hazptr_tc_shutdown() {
875 auto& tc = tls_tc_data_;
877 for (size_t i = 0; i < tc.count_; ++i) {
878 tc.entry_[i].evict();
882 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_try_get() {
883 DEBUG_PRINT(TLS_UNINITIALIZED << TLS_ALIVE << TLS_DESTROYED);
884 DEBUG_PRINT(tls_state_);
885 if (LIKELY(tls_state_ == TLS_ALIVE)) {
886 DEBUG_PRINT(tls_state_);
887 return tls_tc_data_.get();
888 } else if (tls_state_ == TLS_UNINITIALIZED) {
890 return tls_tc_data_.get();
895 FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) {
896 DEBUG_PRINT(tls_state_);
897 if (LIKELY(tls_state_ == TLS_ALIVE)) {
898 DEBUG_PRINT(tls_state_);
899 return tls_tc_data_.put(hprec);
908 inline void hazptr_priv::push(hazptr_obj* obj) {
909 auto& domain = default_hazptr_domain();
910 obj->next_ = nullptr;
915 domain.objRetire(obj);
922 if (domain.reachedThreshold(rcount_)) {
927 inline void hazptr_priv::pushAllToDomain() {
928 auto& domain = default_hazptr_domain();
929 domain.pushRetired(head_, tail_, rcount_);
933 domain.tryBulkReclaim();
936 inline void hazptr_priv_init() {
937 auto& priv = tls_priv_data_;
939 priv.head_ = nullptr;
940 priv.tail_ = nullptr;
945 inline void hazptr_priv_shutdown() {
946 auto& priv = tls_priv_data_;
948 DCHECK(priv.active_);
949 priv.active_ = false;
951 priv.pushAllToDomain();
955 inline bool hazptr_priv_try_retire(hazptr_obj* obj) {
956 DEBUG_PRINT(tls_state_);
957 if (tls_state_ == TLS_ALIVE) {
958 DEBUG_PRINT(tls_state_);
959 tls_priv_data_.push(obj);
961 } else if (tls_state_ == TLS_UNINITIALIZED) {
962 DEBUG_PRINT(tls_state_);
964 tls_priv_data_.push(obj);
970 /** hazptr_tls_life */
972 inline void tls_life_odr_use() {
973 DEBUG_PRINT(tls_state_);
974 CHECK(tls_state_ == TLS_UNINITIALIZED);
975 auto volatile tlsOdrUse = &tls_life_;
976 CHECK(tlsOdrUse != nullptr);
977 DEBUG_PRINT(tlsOdrUse);
980 inline hazptr_tls_life::hazptr_tls_life() {
982 CHECK(tls_state_ == TLS_UNINITIALIZED);
985 tls_state_ = TLS_ALIVE;
988 inline hazptr_tls_life::~hazptr_tls_life() {
990 CHECK(tls_state_ == TLS_ALIVE);
991 hazptr_tc_shutdown();
992 hazptr_priv_shutdown();
993 tls_state_ = TLS_DESTROYED;
996 } // namespace hazptr