bool pop(T& val) {
DEBUG_PRINT(this);
hazptr_owner<Node> hptr;
- Node* pnode;
- while (true) {
- if ((pnode = head_.load()) == nullptr) return false;
- if (!hptr.protect(pnode, head_)) continue;
+ Node* pnode = head_.load();
+ do {
+ if (pnode == nullptr)
+ return false;
+ if (!hptr.try_protect(pnode, head_))
+ continue;
auto next = pnode->next_;
if (head_.compare_exchange_weak(pnode, next)) break;
- }
+ } while (true);
hptr.clear();
val = pnode->value_;
pnode->retire();
*/
template <typename T>
class SWMRListSet {
- class Node : public hazptr_obj_base<Node> {
+ template <typename Node>
+ struct Reclaimer {
+ void operator()(Node* p) {
+ DEBUG_PRINT(p << " " << sizeof(Node));
+ delete p;
+ }
+ };
+
+ class Node : public hazptr_obj_base<Node, Reclaimer<Node>> {
friend SWMRListSet;
T elem_;
std::atomic<Node*> next_;
};
std::atomic<Node*> head_ = {nullptr};
- hazptr_domain* domain_;
- hazptr_obj_reclaim<Node> reclaim_ = [](Node* p) { reclaim(p); };
-
- static void reclaim(Node* p) {
- DEBUG_PRINT(p << " " << sizeof(Node));
- delete p;
- };
+ hazptr_domain& domain_;
/* Used by the single writer */
- void locate_lower_bound(T v, std::atomic<Node*>*& prev) {
+ void locate_lower_bound(const T v, std::atomic<Node*>*& prev) const {
auto curr = prev->load();
while (curr) {
if (curr->elem_ >= v) break;
}
public:
- explicit SWMRListSet(hazptr_domain* domain = default_hazptr_domain())
+ explicit SWMRListSet(hazptr_domain& domain = default_hazptr_domain())
: domain_(domain) {}
~SWMRListSet() {
next = p->next_.load();
delete p;
}
- domain_->flush(&reclaim_); /* avoid destruction order fiasco */
}
- bool add(T v) {
+ bool add(const T v) {
auto prev = &head_;
locate_lower_bound(v, prev);
auto curr = prev->load();
return true;
}
- bool remove(T v) {
+ bool remove(const T v) {
auto prev = &head_;
locate_lower_bound(v, prev);
auto curr = prev->load();
if (!curr || curr->elem_ != v) return false;
prev->store(curr->next_.load());
- curr->retire(domain_, &reclaim_);
+ curr->retire(domain_);
return true;
}
/* Used by readers */
- bool contains(T val) {
+ bool contains(const T val) const {
/* Acquire two hazard pointers for hand-over-hand traversal. */
hazptr_owner<Node> hptr_prev(domain_);
hazptr_owner<Node> hptr_curr(domain_);
auto curr = prev->load();
while (true) {
if (!curr) { done = true; break; }
- if (!hptr_curr.protect(curr, *prev)) break;
+ if (!hptr_curr.try_protect(curr, *prev))
+ break;
auto next = curr->next_.load();
elem = curr->elem_;
- // Load-load order
- std::atomic_thread_fence(std::memory_order_acquire);
if (prev->load() != curr) break;
if (elem >= val) { done = true; break; }
prev = &(curr->next_);
DEBUG_PRINT(this << " " << u << " " << v);
Node* n = new Node(v);
hazptr_owner<Node> hptr;
- Node* p = p_.load();
+ Node* p;
do {
- if (!hptr.protect(p, p_)) continue;
+ p = hptr.get_protected(p_);
if (p->val_ != u) { delete n; return false; }
if (p_.compare_exchange_weak(p, n)) break;
} while (true);
constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept
: mr_(mr) {}
-template <typename T>
-void hazptr_domain::flush(const hazptr_obj_reclaim<T>* reclaim) {
- DEBUG_PRINT(this << " " << reclaim);
- flush(reinterpret_cast<const hazptr_obj_reclaim<void>*>(reclaim));
-}
-
-template <typename T>
-inline void hazptr_domain::objRetire(hazptr_obj_base<T>* p) {
- DEBUG_PRINT(this << " " << p);
- objRetire(reinterpret_cast<hazptr_obj_base<void>*>(p));
-}
-
/** hazptr_obj_base */
-template <typename T>
-inline void hazptr_obj_base<T>::retire(
- hazptr_domain* domain,
- const hazptr_obj_reclaim<T>* reclaim,
+template <typename T, typename D>
+inline void hazptr_obj_base<T, D>::retire(
+ hazptr_domain& domain,
+ D deleter,
const storage_policy /* policy */) {
- DEBUG_PRINT(this << " " << reclaim << " " << &domain);
- reclaim_ = reclaim;
- domain->objRetire<T>(this);
-}
-
-/* Definition of default_hazptr_obj_reclaim */
-
-template <typename T>
-inline hazptr_obj_reclaim<T>* default_hazptr_obj_reclaim() {
- static hazptr_obj_reclaim<T> fn = [](T* p) {
- DEBUG_PRINT("default_hazptr_obj_reclaim " << p << " " << sizeof(T));
- delete p;
+ DEBUG_PRINT(this << " " << &domain);
+ deleter_ = std::move(deleter);
+ reclaim_ = [](hazptr_obj* p) {
+ auto hobp = static_cast<hazptr_obj_base*>(p);
+ auto obj = static_cast<T*>(hobp);
+ hobp->deleter_(obj);
};
- DEBUG_PRINT(&fn);
- return &fn;
+ domain.objRetire(this);
}
/** hazptr_rec */
template <typename T>
inline hazptr_owner<T>::hazptr_owner(
- hazptr_domain* domain,
+ hazptr_domain& domain,
const cache_policy /* policy */) {
- domain_ = domain;
+ domain_ = &domain;
hazptr_ = domain_->hazptrAcquire();
DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
if (hazptr_ == nullptr) { std::bad_alloc e; throw e; }
}
template <typename T>
-hazptr_owner<T>::~hazptr_owner() noexcept {
+hazptr_owner<T>::~hazptr_owner() {
DEBUG_PRINT(this);
domain_->hazptrRelease(hazptr_);
}
template <typename T>
-inline bool hazptr_owner<T>::protect(const T* ptr, const std::atomic<T*>& src)
- const noexcept {
+inline bool hazptr_owner<T>::try_protect(
+ T*& ptr,
+ const std::atomic<T*>& src) noexcept {
DEBUG_PRINT(this << " " << ptr << " " << &src);
- hazptr_->set(ptr);
- // ORDER: store-load
- return (src.load() == ptr);
+ set(ptr);
+ T* p = src.load();
+ if (p != ptr) {
+ ptr = p;
+ clear();
+ return false;
+ }
+ return true;
}
template <typename T>
-inline void hazptr_owner<T>::set(const T* ptr) const noexcept {
+inline T* hazptr_owner<T>::get_protected(const std::atomic<T*>& src) noexcept {
+ T* p = src.load();
+ while (!try_protect(p, src)) {}
+ DEBUG_PRINT(this << " " << p << " " << &src);
+ return p;
+}
+
+template <typename T>
+inline void hazptr_owner<T>::set(const T* ptr) noexcept {
DEBUG_PRINT(this << " " << ptr);
hazptr_->set(ptr);
}
template <typename T>
-inline void hazptr_owner<T>::clear() const noexcept {
+inline void hazptr_owner<T>::clear() noexcept {
DEBUG_PRINT(this);
hazptr_->clear();
}
// - Optimized memory order
/** Definition of default_hazptr_domain() */
-inline hazptr_domain* default_hazptr_domain() {
+inline hazptr_domain& default_hazptr_domain() {
static hazptr_domain d;
- return &d;
+ return d;
}
/** hazptr_rec */
inline void hazptr_rec::clear() noexcept {
DEBUG_PRINT(this);
- // ORDER: release
hazptr_.store(nullptr);
}
inline void hazptr_rec::release() noexcept {
DEBUG_PRINT(this);
clear();
- // ORDER: release
active_.store(false);
}
+/** hazptr_obj */
+
+inline const void* hazptr_obj::getObjPtr() const {
+ return this;
+}
+
/** hazptr_domain */
inline hazptr_domain::~hazptr_domain() {
}
}
-inline void hazptr_domain::flush() {
+inline void hazptr_domain::try_reclaim() {
DEBUG_PRINT(this);
- auto rcount = rcount_.exchange(0);
- auto p = retired_.exchange(nullptr);
- hazptr_obj* next;
- for (; p; p = next) {
- next = p->next_;
- (*(p->reclaim_))(p);
- --rcount;
- }
- rcount_.fetch_add(rcount);
+ rcount_.exchange(0);
+ bulkReclaim();
}
inline hazptr_rec* hazptr_domain::hazptrAcquire() {
return p;
}
-inline void hazptr_domain::hazptrRelease(hazptr_rec* p) const noexcept {
+inline void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept {
DEBUG_PRINT(this << " " << p);
p->release();
}
inline int
hazptr_domain::pushRetired(hazptr_obj* head, hazptr_obj* tail, int count) {
tail->next_ = retired_.load();
- // ORDER: store-store order
while (!retired_.compare_exchange_weak(tail->next_, head)) {}
return rcount_.fetch_add(count);
}
inline void hazptr_domain::objRetire(hazptr_obj* p) {
auto rcount = pushRetired(p, p, 1) + 1;
if (rcount >= kScanThreshold * hcount_.load()) {
- bulkReclaim();
+ tryBulkReclaim();
}
}
-inline void hazptr_domain::bulkReclaim() {
+inline void hazptr_domain::tryBulkReclaim() {
DEBUG_PRINT(this);
- auto h = hazptrs_.load();
- auto hcount = hcount_.load();
- auto rcount = rcount_.load();
do {
+ auto hcount = hcount_.load();
+ auto rcount = rcount_.load();
if (rcount < kScanThreshold * hcount) {
return;
}
break;
}
} while (true);
- /* ORDER: store-load order between removing each object and scanning
- * the hazard pointers -- can be combined in one fence */
+ bulkReclaim();
+}
+
+inline void hazptr_domain::bulkReclaim() {
+ DEBUG_PRINT(this);
+ auto p = retired_.exchange(nullptr);
+ auto h = hazptrs_.load();
std::unordered_set<const void*> hs;
for (; h; h = h->next_) {
hs.insert(h->hazptr_.load());
}
- rcount = 0;
+ int rcount = 0;
hazptr_obj* retired = nullptr;
hazptr_obj* tail = nullptr;
- auto p = retired_.exchange(nullptr);
hazptr_obj* next;
for (; p; p = next) {
next = p->next_;
- if (hs.count(p) == 0) {
+ if (hs.count(p->getObjPtr()) == 0) {
+ DEBUG_PRINT(this << " " << p << " " << p->reclaim_);
(*(p->reclaim_))(p);
} else {
p->next_ = retired;
}
}
-inline void hazptr_domain::flush(const hazptr_obj_reclaim<void>* reclaim) {
- DEBUG_PRINT(this << " " << reclaim);
- auto rcount = rcount_.exchange(0);
- auto p = retired_.exchange(nullptr);
- hazptr_obj* retired = nullptr;
- hazptr_obj* tail = nullptr;
- hazptr_obj* next;
- for (; p; p = next) {
- next = p->next_;
- if (p->reclaim_ == reclaim) {
- (*reclaim)(p);
- } else {
- p->next_ = retired;
- retired = p;
- if (tail == nullptr) {
- tail = p;
- }
- ++rcount;
- }
- }
- if (tail) {
- pushRetired(retired, tail, rcount);
- }
-}
-
/** hazptr_user */
inline void hazptr_user::flush() {
/** hazptr_rec: Private class that contains hazard pointers. */
class hazptr_rec;
-/** hazptr_obj_base: Base template for objects protected by hazard pointers. */
-template <typename T> class hazptr_obj_base;
+/** hazptr_obj: Private class for objects protected by hazard pointers. */
+class hazptr_obj;
-/** Alias for object reclamation function template */
-template <typename T> using hazptr_obj_reclaim = std::function<void(T*)>;
+/** hazptr_obj_base: Base template for objects protected by hazard pointers. */
+template <typename T, typename Deleter>
+class hazptr_obj_base;
/** hazptr_domain: Class of hazard pointer domains. Each domain manages a set
* of hazard pointers and a set of retired objects. */
hazptr_domain& operator=(const hazptr_domain&) = delete;
hazptr_domain& operator=(hazptr_domain&&) = delete;
- /* Reclaim all retired objects with a specific reclamation
- * function currently stored by this domain */
- template <typename T> void flush(const hazptr_obj_reclaim<T>* reclaim);
- /* Reclaim all retired objects currently stored by this domain */
- void flush();
+ void try_reclaim();
private:
- template <typename> friend class hazptr_obj_base;
+ template <typename, typename>
+ friend class hazptr_obj_base;
template <typename> friend class hazptr_owner;
- using hazptr_obj = hazptr_obj_base<void>;
-
/** Constant -- May be changed to parameter in the future */
enum { kScanThreshold = 3 };
std::atomic<int> hcount_ = {0};
std::atomic<int> rcount_ = {0};
- template <typename T> void objRetire(hazptr_obj_base<T>*);
- hazptr_rec* hazptrAcquire();
- void hazptrRelease(hazptr_rec*) const noexcept;
void objRetire(hazptr_obj*);
+ hazptr_rec* hazptrAcquire();
+ void hazptrRelease(hazptr_rec*) noexcept;
int pushRetired(hazptr_obj* head, hazptr_obj* tail, int count);
+ void tryBulkReclaim();
void bulkReclaim();
- void flush(const hazptr_obj_reclaim<void>* reclaim);
};
/** Get the default hazptr_domain */
-hazptr_domain* default_hazptr_domain();
+hazptr_domain& default_hazptr_domain();
-/** Declaration of default reclamation function template */
-template <typename T> hazptr_obj_reclaim<T>* default_hazptr_obj_reclaim();
+/** Definition of hazptr_obj */
+class hazptr_obj {
+ friend class hazptr_domain;
+ template <typename, typename>
+ friend class hazptr_obj_base;
+
+ void (*reclaim_)(hazptr_obj*);
+ hazptr_obj* next_;
+ const void* getObjPtr() const;
+};
/** Definition of hazptr_obj_base */
-template <typename T> class hazptr_obj_base {
+template <typename T, typename Deleter = std::default_delete<T>>
+class hazptr_obj_base : private hazptr_obj {
public:
/* Policy for storing retired objects */
enum class storage_policy { priv, shared };
/* Retire a removed object and pass the responsibility for
* reclaiming it to the hazptr library */
void retire(
- hazptr_domain* domain = default_hazptr_domain(),
- const hazptr_obj_reclaim<T>* reclaim = default_hazptr_obj_reclaim<T>(),
+ hazptr_domain& domain = default_hazptr_domain(),
+ Deleter reclaim = {},
const storage_policy policy = storage_policy::shared);
private:
- friend class hazptr_domain;
- template <typename> friend class hazptr_owner;
-
- const hazptr_obj_reclaim<T>* reclaim_;
- hazptr_obj_base* next_;
+ Deleter deleter_;
};
/** hazptr_owner: Template for automatic acquisition and release of
* hazard pointers, and interface for hazard pointer operations. */
-template <typename T> class hazptr_owner;
-
-/* Swap ownership of hazard ponters between hazptr_owner-s. */
-/* Note: The owned hazard pointers remain unmodified during the swap
- * and continue to protect the respective objects that they were
- * protecting before the swap, if any. */
-template <typename T>
-void swap(hazptr_owner<T>&, hazptr_owner<T>&) noexcept;
-
template <typename T> class hazptr_owner {
public:
/* Policy for caching hazard pointers */
/* Constructor automatically acquires a hazard pointer. */
explicit hazptr_owner(
- hazptr_domain* domain = default_hazptr_domain(),
+ hazptr_domain& domain = default_hazptr_domain(),
const cache_policy policy = cache_policy::nocache);
/* Destructor automatically clears and releases the owned hazard pointer. */
- ~hazptr_owner() noexcept;
+ ~hazptr_owner();
/* Copy and move constructors and assignment operators are
* disallowed because:
hazptr_owner& operator=(hazptr_owner&&) = delete;
/** Hazard pointer operations */
- /* Return true if successful in protecting the object */
- bool protect(const T* ptr, const std::atomic<T*>& src) const noexcept;
+ /* Returns a protected pointer from the source */
+ T* get_protected(const std::atomic<T*>& src) noexcept;
+ /* Return true if successful in protecting ptr if src == ptr after
+ * setting the hazard pointer. Otherwise sets ptr to src. */
+ bool try_protect(T*& ptr, const std::atomic<T*>& src) noexcept;
/* Set the hazard pointer to ptr */
- void set(const T* ptr) const noexcept;
+ void set(const T* ptr) noexcept;
/* Clear the hazard pointer */
- void clear() const noexcept;
+ void clear() noexcept;
+ /* Swap ownership of hazard ponters between hazptr_owner-s. */
+ /* Note: The owned hazard pointers remain unmodified during the swap
+ * and continue to protect the respective objects that they were
+ * protecting before the swap, if any. */
void swap(hazptr_owner&) noexcept;
private:
-
hazptr_domain* domain_;
hazptr_rec* hazptr_;
};
+template <typename T>
+void swap(hazptr_owner<T>&, hazptr_owner<T>&) noexcept;
+
/** hazptr_user: Thread-specific interface for users of hazard
* pointers (i.e., threads that own hazard pointers by using
* hazptr_owner. */
#include <thread>
-using namespace folly::hazptr;
-
-static hazptr_obj_reclaim<Node1> myReclaim_ = [](Node1* p) {
- myReclaimFn(p);
-};
+DEFINE_int32(num_threads, 1, "Number of threads");
+DEFINE_int64(num_reps, 1, "Number of test reps");
+DEFINE_int64(num_ops, 10, "Number of ops or pairs of ops per rep");
-static hazptr_obj_reclaim<Node2> mineReclaim_ = [](Node2* p) {
- mineReclaimFn(p);
-};
+using namespace folly::hazptr;
TEST(Hazptr, Test1) {
DEBUG_PRINT("========== start of scope");
DEBUG_PRINT("");
- Node1* node0 = new Node1;
+ Node1* node0 = (Node1*)malloc(sizeof(Node1));
DEBUG_PRINT("=== new node0 " << node0 << " " << sizeof(*node0));
Node1* node1 = (Node1*)malloc(sizeof(Node1));
DEBUG_PRINT("=== malloc node1 " << node1 << " " << sizeof(*node1));
DEBUG_PRINT("=== hptr0");
hazptr_owner<Node1> hptr0;
DEBUG_PRINT("=== hptr1");
- hazptr_owner<Node1> hptr1(&myDomain0);
+ hazptr_owner<Node1> hptr1(myDomain0);
DEBUG_PRINT("=== hptr2");
- hazptr_owner<Node1> hptr2(&myDomain1);
+ hazptr_owner<Node1> hptr2(myDomain1);
DEBUG_PRINT("=== hptr3");
hazptr_owner<Node1> hptr3;
Node1* n2 = shared2.load();
Node1* n3 = shared3.load();
- if (hptr0.protect(n0, shared0)) {}
- if (hptr1.protect(n1, shared1)) {}
+ if (hptr0.try_protect(n0, shared0)) {}
+ if (hptr1.try_protect(n1, shared1)) {}
hptr1.clear();
hptr1.set(n2);
- if (hptr2.protect(n3, shared3)) {}
+ if (hptr2.try_protect(n3, shared3)) {}
swap(hptr1, hptr2);
hptr3.clear();
DEBUG_PRINT("=== retire n0 " << n0);
n0->retire();
DEBUG_PRINT("=== retire n1 " << n1);
-
- n1->retire(default_hazptr_domain(), &myReclaim_);
+ n1->retire(default_hazptr_domain());
DEBUG_PRINT("=== retire n2 " << n2);
- n2->retire(&myDomain0, &myReclaim_);
+ n2->retire(myDomain0);
DEBUG_PRINT("=== retire n3 " << n3);
- n3->retire(&myDomain1, &myReclaim_);
+ n3->retire(myDomain1);
DEBUG_PRINT("========== end of scope");
}
DEBUG_PRINT("=== hptr0");
hazptr_owner<Node2> hptr0;
DEBUG_PRINT("=== hptr1");
- hazptr_owner<Node2> hptr1(&mineDomain0);
+ hazptr_owner<Node2> hptr1(mineDomain0);
DEBUG_PRINT("=== hptr2");
- hazptr_owner<Node2> hptr2(&mineDomain1);
+ hazptr_owner<Node2> hptr2(mineDomain1);
DEBUG_PRINT("=== hptr3");
hazptr_owner<Node2> hptr3;
Node2* n2 = shared2.load();
Node2* n3 = shared3.load();
- if (hptr0.protect(n0, shared0)) {}
- if (hptr1.protect(n1, shared1)) {}
+ if (hptr0.try_protect(n0, shared0)) {}
+ if (hptr1.try_protect(n1, shared1)) {}
hptr1.clear();
hptr1.set(n2);
- if (hptr2.protect(n3, shared3)) {}
+ if (hptr2.try_protect(n3, shared3)) {}
swap(hptr1, hptr2);
hptr3.clear();
DEBUG_PRINT("");
DEBUG_PRINT("=== retire n0 " << n0);
- n0->retire();
+ n0->retire(default_hazptr_domain(), &mineReclaimFnDelete);
DEBUG_PRINT("=== retire n1 " << n1);
-
- n1->retire(default_hazptr_domain(), &mineReclaim_);
+ n1->retire(default_hazptr_domain(), &mineReclaimFnFree);
DEBUG_PRINT("=== retire n2 " << n2);
- n2->retire(&mineDomain0, &mineReclaim_);
+ n2->retire(mineDomain0, &mineReclaimFnFree);
DEBUG_PRINT("=== retire n3 " << n3);
- n3->retire(&mineDomain1, &mineReclaim_);
+ n3->retire(mineDomain1, &mineReclaimFnFree);
DEBUG_PRINT("========== end of scope");
}
-DEFINE_int32(num_threads, 1, "Number of threads");
-DEFINE_int64(num_reps, 1, "Number of test reps");
-DEFINE_int64(num_ops, 10, "Number of ops or pairs of ops per rep");
-
TEST(Hazptr, LIFO) {
using T = uint32_t;
DEBUG_PRINT("========== start of test scope");
CHECK_GT(FLAGS_num_threads, 0);
for (int i = 0; i < FLAGS_num_reps; ++i) {
DEBUG_PRINT("========== start of rep scope");
- SWMRListSet<T> s(&custom_domain);
+ SWMRListSet<T> s(custom_domain);
std::vector<std::thread> threads(FLAGS_num_threads);
for (int tid = 0; tid < FLAGS_num_threads; ++tid) {
threads[tid] = std::thread([&s, tid]() {
}
int main(int argc, char** argv) {
- DEBUG_PRINT("=================================================== start main");
+ DEBUG_PRINT("================================================= start main");
testing::InitGoogleTest(&argc, argv);
google::ParseCommandLineFlags(&argc, &argv, true);
auto ret = RUN_ALL_TESTS();
- default_hazptr_domain()->flush();
- DEBUG_PRINT("===================================================== end main");
+ DEBUG_PRINT("================================================= after tests");
+ default_hazptr_domain().try_reclaim();
+ DEBUG_PRINT("================================================= end main");
return ret;
}
}
};
-class Node1 : public hazptr_obj_base<Node1> {
- char a[100];
+template <typename Node1>
+struct MyReclaimerFree {
+ inline void operator()(Node1* p) {
+ DEBUG_PRINT(p << " " << sizeof(Node1));
+ free(p);
+ }
};
-inline void myReclaimFn(Node1* p) {
- DEBUG_PRINT(p << " " << sizeof(Node1));
- free(p);
-}
+class Node1 : public hazptr_obj_base<Node1, MyReclaimerFree<Node1>> {
+ char a[100];
+};
} // namespace folly {
} // namespace hazptr {
}
};
-class Node2 : public hazptr_obj_base<Node2> {
+class Node2 : public hazptr_obj_base<Node2, void (*)(Node2*)> {
char a[200];
};
-inline void mineReclaimFn(Node2* p) {
+inline void mineReclaimFnFree(Node2* p) {
DEBUG_PRINT(p << " " << sizeof(Node2));
free(p);
}
+inline void mineReclaimFnDelete(Node2* p) {
+ DEBUG_PRINT(p << " " << sizeof(Node2));
+ delete p;
+}
+
} // namespace folly {
} // namespace hazptr {
projects / folly.git / commitdiff