#ifndef FOLLY_INDEXEDMEMPOOL_H
#define FOLLY_INDEXEDMEMPOOL_H
+#include <type_traits>
#include <stdint.h>
#include <assert.h>
#include <unistd.h>
struct IndexedMemPoolRecycler;
}
-/// Instances of IndexedMemPool dynamically allocate and then pool
-/// their element type (T), returning 4-byte integer indices that can be
-/// passed to the pool's operator[] method to access or obtain pointers
-/// to the actual elements. Once they are constructed, elements are
-/// never destroyed. These two features are useful for lock-free
-/// algorithms. The indexing behavior makes it easy to build tagged
-/// pointer-like-things, since a large number of elements can be managed
-/// using fewer bits than a full pointer. The pooling behavior makes
-/// it safe to read from T-s even after they have been recycled, since
-/// it is guaranteed that the memory won't have been returned to the OS
-/// and unmapped (the algorithm must still use a mechanism to validate
-/// that the read was correct, but it doesn't have to worry about page
-/// faults), and if the elements use internal sequence numbers it can be
-/// guaranteed that there won't be an ABA match due to the element being
-/// overwritten with a different type that has the same bit pattern.
+/// Instances of IndexedMemPool dynamically allocate and then pool their
+/// element type (T), returning 4-byte integer indices that can be passed
+/// to the pool's operator[] method to access or obtain pointers to the
+/// actual elements. The memory backing items returned from the pool
+/// will always be readable, even if items have been returned to the pool.
+/// These two features are useful for lock-free algorithms. The indexing
+/// behavior makes it easy to build tagged pointer-like-things, since
+/// a large number of elements can be managed using fewer bits than a
+/// full pointer. The access-after-free behavior makes it safe to read
+/// from T-s even after they have been recycled, since it is guaranteed
+/// that the memory won't have been returned to the OS and unmapped
+/// (the algorithm must still use a mechanism to validate that the read
+/// was correct, but it doesn't have to worry about page faults), and if
+/// the elements use internal sequence numbers it can be guaranteed that
+/// there won't be an ABA match due to the element being overwritten with
+/// a different type that has the same bit pattern.
+///
+/// IndexedMemPool has two object lifecycle strategies. The first
+/// is to construct objects when they are allocated from the pool and
+/// destroy them when they are recycled. In this mode allocIndex and
+/// allocElem have emplace-like semantics. In the second mode, objects
+/// are default-constructed the first time they are removed from the pool,
+/// and deleted when the pool itself is deleted. By default the first
+/// mode is used for non-trivial T, and the second is used for trivial T.
///
/// IMPORTANT: Space for extra elements is allocated to account for those
/// that are inaccessible because they are in other local lists, so the
template <typename T,
int NumLocalLists_ = 32,
int LocalListLimit_ = 200,
- template<typename> class Atom = std::atomic>
+ template<typename> class Atom = std::atomic,
+ bool EagerRecycleWhenTrivial = false,
+ bool EagerRecycleWhenNotTrivial = true>
struct IndexedMemPool : boost::noncopyable {
typedef T value_type;
};
+ static constexpr bool eagerRecycle() {
+ return std::is_trivial<T>::value
+ ? EagerRecycleWhenTrivial : EagerRecycleWhenNotTrivial;
+ }
+
// these are public because clients may need to reason about the number
// of bits required to hold indices from a pool, given its capacity
/// Destroys all of the contained elements
~IndexedMemPool() {
- for (size_t i = size_; i > 0; --i) {
- slots_[i].~Slot();
+ if (!eagerRecycle()) {
+ for (size_t i = size_; i > 0; --i) {
+ slots_[i].~Slot();
+ }
}
munmap(slots_, mmapLength_);
}
return capacityForMaxIndex(actualCapacity_);
}
- /// Grants ownership of (*this)[retval], or returns 0 if no elements
- /// are available
- uint32_t allocIndex() {
- return localPop(localHead());
+ /// Finds a slot with a non-zero index, emplaces a T there if we're
+ /// using the eager recycle lifecycle mode, and returns the index,
+ /// or returns 0 if no elements are available.
+ template <typename ...Args>
+ uint32_t allocIndex(Args&&... args) {
+ static_assert(sizeof...(Args) == 0 || eagerRecycle(),
+ "emplace-style allocation requires eager recycle, "
+ "which is defaulted only for non-trivial types");
+ auto idx = localPop(localHead());
+ if (idx != 0 && eagerRecycle()) {
+ T* ptr = &slot(idx).elem;
+ new (ptr) T(std::forward<Args>(args)...);
+ }
+ return idx;
}
/// If an element is available, returns a std::unique_ptr to it that will
/// recycle the element to the pool when it is reclaimed, otherwise returns
/// a null (falsy) std::unique_ptr
- UniquePtr allocElem() {
- auto idx = allocIndex();
- auto ptr = idx == 0 ? nullptr : &slot(idx).elem;
+ template <typename ...Args>
+ UniquePtr allocElem(Args&&... args) {
+ auto idx = allocIndex(std::forward<Args>(args)...);
+ T* ptr = idx == 0 ? nullptr : &slot(idx).elem;
return UniquePtr(ptr, typename UniquePtr::deleter_type(this));
}
/// Gives up ownership previously granted by alloc()
void recycleIndex(uint32_t idx) {
assert(isAllocated(idx));
+ if (eagerRecycle()) {
+ slot(idx).elem.~T();
+ }
localPush(localHead(), idx);
}
// allocation failed
return 0;
}
- // construct it
- new (&slot(idx)) T();
+ // default-construct it now if we aren't going to construct and
+ // destroy on each allocation
+ if (!eagerRecycle()) {
+ T* ptr = &slot(idx).elem;
+ new (ptr) T();
+ }
slot(idx).localNext = uint32_t(-1);
return idx;
}
#include <folly/IndexedMemPool.h>
#include <folly/test/DeterministicSchedule.h>
+#include <string>
#include <thread>
#include <unistd.h>
#include <semaphore.h>
EXPECT_EQ(pool.locateElem(nullptr), 0);
}
+struct NonTrivialStruct {
+ static __thread int count;
+
+ int elem_;
+
+ NonTrivialStruct() {
+ elem_ = 0;
+ ++count;
+ }
+
+ NonTrivialStruct(std::unique_ptr<std::string>&& arg1, int arg2) {
+ elem_ = arg1->length() + arg2;
+ ++count;
+ }
+
+ ~NonTrivialStruct() {
+ --count;
+ }
+};
+
+__thread int NonTrivialStruct::count;
+
+TEST(IndexedMemPool, eager_recycle) {
+ typedef IndexedMemPool<NonTrivialStruct> Pool;
+ Pool pool(100);
+
+ EXPECT_EQ(NonTrivialStruct::count, 0);
+
+ for (size_t i = 0; i < 10; ++i) {
+ {
+ std::unique_ptr<std::string> arg{ new std::string{ "abc" } };
+ auto ptr = pool.allocElem(std::move(arg), 100);
+ EXPECT_EQ(NonTrivialStruct::count, 1);
+ EXPECT_EQ(ptr->elem_, 103);
+ EXPECT_TRUE(!!ptr);
+ }
+ EXPECT_EQ(NonTrivialStruct::count, 0);
+ }
+}
+
+TEST(IndexedMemPool, late_recycle) {
+ {
+ typedef IndexedMemPool<NonTrivialStruct, 8, 8, std::atomic, false, false>
+ Pool;
+ Pool pool(100);
+
+ EXPECT_EQ(NonTrivialStruct::count, 0);
+
+ for (size_t i = 0; i < 10; ++i) {
+ {
+ auto ptr = pool.allocElem();
+ EXPECT_TRUE(NonTrivialStruct::count > 0);
+ EXPECT_TRUE(!!ptr);
+ ptr->elem_ = i;
+ }
+ EXPECT_TRUE(NonTrivialStruct::count > 0);
+ }
+ }
+ EXPECT_EQ(NonTrivialStruct::count, 0);
+}
+
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
gflags::ParseCommandLineFlags(&argc, &argv, true);
projects / folly.git / commitdiff