2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2016
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 // Dynamic Hazard Pointer memory manager implementation
33 #include <algorithm> // std::fill
34 #include <functional> // std::hash
36 #include <cds/gc/details/dhp.h>
37 #include <cds/algo/int_algo.h>
39 namespace cds { namespace gc { namespace dhp {
44 typedef retired_ptr_node * item_type;
45 typedef cds::details::Allocator<item_type, CDS_DEFAULT_ALLOCATOR> allocator_type;
47 size_t const m_nBucketCount;
48 item_type * m_Buckets;
50 item_type& bucket( retired_ptr_node& node ) const
52 return bucket( node.m_ptr.m_p );
54 item_type& bucket( guard_data::guarded_ptr p ) const
56 return m_Buckets[ std::hash<guard_data::guarded_ptr>()( p ) & (m_nBucketCount - 1) ];
60 liberate_set( size_t nBucketCount )
61 : m_nBucketCount( nBucketCount )
63 assert( nBucketCount > 0 );
64 assert( (nBucketCount & (nBucketCount - 1)) == 0 );
66 m_Buckets = allocator_type().NewArray( nBucketCount );
67 std::fill( m_Buckets, m_Buckets + nBucketCount, nullptr );
72 allocator_type().Delete( m_Buckets, m_nBucketCount );
75 void insert( retired_ptr_node& node )
77 node.m_pNext.store( nullptr, atomics::memory_order_relaxed );
79 item_type& refBucket = bucket( node );
81 item_type p = refBucket;
82 item_type prev = nullptr;
84 if ( p->m_ptr.m_p >= node.m_ptr.m_p ) {
85 node.m_pNext.store( p, atomics::memory_order_relaxed );
87 prev->m_pNext.store( &node, atomics::memory_order_relaxed );
93 p = p->m_pNext.load(atomics::memory_order_relaxed);
96 assert( prev != nullptr );
97 prev->m_pNext.store( &node, atomics::memory_order_relaxed );
116 erase_result erase( guard_data::guarded_ptr ptr )
118 item_type& refBucket = bucket( ptr );
119 item_type p = refBucket;
120 item_type pPrev = nullptr;
123 while ( p && p->m_ptr.m_p <= ptr ) {
124 if ( p->m_ptr.m_p == ptr ) {
126 pPrev->m_pNext.store( p->m_pNext.load(atomics::memory_order_relaxed ), atomics::memory_order_relaxed );
128 refBucket = p->m_pNext.load(atomics::memory_order_relaxed);
131 ret.tail->m_pNext.store( p, atomics::memory_order_relaxed );
139 p = p->m_pNext.load( atomics::memory_order_relaxed );
143 ret.tail->m_pNext.store( nullptr, atomics::memory_order_relaxed );
147 typedef std::pair<item_type, item_type> list_range;
149 list_range free_all()
151 item_type pTail = nullptr;
152 list_range ret = std::make_pair( pTail, pTail );
154 item_type const * pEndBucket = m_Buckets + m_nBucketCount;
155 for ( item_type * ppBucket = m_Buckets; ppBucket < pEndBucket; ++ppBucket ) {
156 item_type pBucket = *ppBucket;
159 pTail->m_pNextFree.store( pBucket, atomics::memory_order_relaxed );
165 item_type pNext = pTail->m_pNext.load( atomics::memory_order_relaxed );
167 pTail->m_pNext.store( nullptr, atomics::memory_order_relaxed );
170 while ( pTail->m_pNextFree.load( atomics::memory_order_relaxed )) {
171 pTail = pTail->m_pNextFree.load( atomics::memory_order_relaxed );
173 pTail->m_pNext.store( nullptr, atomics::memory_order_relaxed );
178 pTail->m_pNextFree.store( pNext, atomics::memory_order_relaxed );
188 pTail->m_pNextFree.store( nullptr, atomics::memory_order_relaxed );
195 GarbageCollector * GarbageCollector::m_pManager = nullptr;
197 void CDS_STDCALL GarbageCollector::Construct(
198 size_t nLiberateThreshold
199 , size_t nInitialThreadGuardCount
204 m_pManager = new GarbageCollector( nLiberateThreshold, nInitialThreadGuardCount, nEpochCount );
208 void CDS_STDCALL GarbageCollector::Destruct()
211 m_pManager = nullptr;
214 GarbageCollector::GarbageCollector( size_t nLiberateThreshold, size_t nInitialThreadGuardCount, size_t nEpochCount )
215 : m_nLiberateThreshold( nLiberateThreshold ? nLiberateThreshold : 1024 )
216 , m_nInitialThreadGuardCount( nInitialThreadGuardCount ? nInitialThreadGuardCount : 8 )
217 , m_RetiredAllocator( static_cast<unsigned int>( nEpochCount ? nEpochCount : 16 ))
218 , m_bStatEnabled( false )
221 GarbageCollector::~GarbageCollector()
226 void GarbageCollector::scan()
228 details::retired_ptr_buffer::privatize_result retiredList = m_RetiredBuffer.privatize();
229 if ( retiredList.first ) {
231 size_t nLiberateThreshold = m_nLiberateThreshold.load(atomics::memory_order_relaxed);
232 details::liberate_set set( beans::ceil2( retiredList.second > nLiberateThreshold ? retiredList.second : nLiberateThreshold ));
234 // Get list of retired pointers
235 size_t nRetiredCount = 0;
236 details::retired_ptr_node * pHead = retiredList.first;
238 details::retired_ptr_node * pNext = pHead->m_pNext.load( atomics::memory_order_relaxed );
239 pHead->m_pNextFree.store( nullptr, atomics::memory_order_relaxed );
240 set.insert( *pHead );
247 details::retired_ptr_node dummy;
248 dummy.m_pNext.store( nullptr, atomics::memory_order_relaxed );
249 details::retired_ptr_node * pBusyLast = &dummy;
250 size_t nBusyCount = 0;
252 for ( details::guard_data * pGuard = m_GuardPool.begin(); pGuard; pGuard = pGuard->pGlobalNext.load(atomics::memory_order_acquire) )
254 // get guarded pointer
255 details::guard_data::guarded_ptr valGuarded = pGuard->pPost.load(atomics::memory_order_acquire);
258 auto retired = set.erase( valGuarded );
259 if ( retired.head ) {
260 // Retired pointer is being guarded
261 // [retired.head, retired.tail] is the list linked by m_pNext field
263 pBusyLast->m_pNext.store( retired.head, atomics::memory_order_relaxed );
264 pBusyLast = retired.tail;
265 nBusyCount += retired.size;
270 // Place [dummy.m_pNext, pBusyLast] back to m_RetiredBuffer
272 m_RetiredBuffer.push_list( dummy.m_pNext.load(atomics::memory_order_relaxed), pBusyLast, nBusyCount );
274 // Free all retired pointers
275 details::liberate_set::list_range range = set.free_all();
277 m_RetiredAllocator.inc_epoch();
280 assert( range.second != nullptr );
281 m_RetiredAllocator.free_range( range.first, range.second );
283 else if ( nRetiredCount >= nLiberateThreshold ) {
284 // scan() cycle did not free any retired pointer - double scan() threshold
285 m_nLiberateThreshold.compare_exchange_strong( nLiberateThreshold, nLiberateThreshold * 2, atomics::memory_order_release, atomics::memory_order_relaxed );
289 }}} // namespace cds::gc::dhp