2 * Copyright 2017-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <folly/Function.h>
20 #include <folly/ThreadLocal.h>
21 #include <folly/synchronization/AsymmetricMemoryBarrier.h>
23 // This is unlike folly::ThreadCachedInt in that the full value
24 // is never rounded up globally and cached, it only supports readFull.
26 // folly/experimental/TLRefCount is similar, but does not support a
27 // waitForZero, and is not reset-able.
29 // Note that the RCU implementation is completely abstracted from the
30 // counter implementation, a rseq implementation can be dropped in
31 // if the kernel supports it.
37 template <typename Tag>
38 class ThreadCachedInts {
39 // These are only accessed under the ThreadLocal lock.
40 int64_t orphan_inc_[2]{0, 0};
41 int64_t orphan_dec_[2]{0, 0};
42 folly::detail::Futex<> waiting_;
46 ThreadCachedInts* ints_;
47 constexpr Integer(ThreadCachedInts* ints) noexcept
48 : ints_(ints), inc_{}, dec_{} {}
49 std::atomic<int64_t> inc_[2];
50 std::atomic<int64_t> dec_[2];
52 ints_->orphan_inc_[0] += inc_[0].load(std::memory_order_relaxed);
53 ints_->orphan_inc_[1] += inc_[1].load(std::memory_order_relaxed);
54 ints_->orphan_dec_[0] += dec_[0].load(std::memory_order_relaxed);
55 ints_->orphan_dec_[1] += dec_[1].load(std::memory_order_relaxed);
56 ints_->waiting_.store(0, std::memory_order_release);
57 ints_->waiting_.futexWake();
60 folly::ThreadLocalPtr<Integer, Tag> cs_;
62 // Cache the int pointer in a threadlocal.
63 static thread_local Integer* int_cache_;
66 auto ret = new Integer(this);
72 FOLLY_ALWAYS_INLINE void increment(uint8_t epoch) {
77 auto& c = int_cache_->inc_[epoch];
78 auto val = c.load(std::memory_order_relaxed);
79 c.store(val + 1, std::memory_order_relaxed);
81 folly::asymmetricLightBarrier(); // A
84 FOLLY_ALWAYS_INLINE void decrement(uint8_t epoch) {
85 folly::asymmetricLightBarrier(); // B
90 auto& c = int_cache_->dec_[epoch];
91 auto val = c.load(std::memory_order_relaxed);
92 c.store(val + 1, std::memory_order_relaxed);
94 folly::asymmetricLightBarrier(); // C
95 if (waiting_.load(std::memory_order_acquire)) {
96 waiting_.store(0, std::memory_order_release);
101 int64_t readFull(uint8_t epoch) {
104 // Matches A - ensure all threads have seen new value of version,
105 // *and* that we see current values of counters in readFull()
107 // Note that in lock_shared if a reader is currently between the
108 // version load and counter increment, they may update the wrong
109 // epoch. However, this is ok - they started concurrently *after*
110 // any callbacks that will run, and therefore it is safe to run
112 folly::asymmetricHeavyBarrier();
113 for (auto& i : cs_.accessAllThreads()) {
114 full -= i.dec_[epoch].load(std::memory_order_relaxed);
117 // Matches B - ensure that all increments are seen if decrements
118 // are seen. This is necessary because increment and decrement
119 // are allowed to happen on different threads.
120 folly::asymmetricHeavyBarrier();
122 auto accessor = cs_.accessAllThreads();
123 for (auto& i : accessor) {
124 full += i.inc_[epoch].load(std::memory_order_relaxed);
127 // orphan is read behind accessAllThreads lock
128 auto res = full + orphan_inc_[epoch] - orphan_dec_[epoch];
132 void waitForZero(uint8_t phase) {
133 // Try reading before futex sleeping.
134 if (readFull(phase) == 0) {
139 waiting_.store(1, std::memory_order_release);
140 // Matches C. Ensure either decrement sees waiting_,
141 // or we see their decrement and can safely sleep.
142 folly::asymmetricHeavyBarrier();
143 if (readFull(phase) == 0) {
146 waiting_.futexWait(1);
148 waiting_.store(0, std::memory_order_relaxed);
151 // We are guaranteed to be called while StaticMeta lock is still
152 // held because of ordering in AtForkList. We can therefore safely
153 // touch orphan_ and clear out all counts.
154 void resetAfterFork() {
156 int_cache_->dec_[0].store(0, std::memory_order_relaxed);
157 int_cache_->dec_[1].store(0, std::memory_order_relaxed);
158 int_cache_->inc_[0].store(0, std::memory_order_relaxed);
159 int_cache_->inc_[1].store(0, std::memory_order_relaxed);
168 template <typename Tag>
169 thread_local typename detail::ThreadCachedInts<Tag>::Integer*
170 detail::ThreadCachedInts<Tag>::int_cache_{nullptr};
172 } // namespace detail