From: Max Khizhinsky Date: Sun, 19 Apr 2015 17:24:13 +0000 (+0300) Subject: Merge pull request #27 from krinkinmu/fastpath-opt X-Git-Tag: v2.1.0~249^2~5 X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=d44e28889bd823545ec071ab8028cde857d4f820;hp=d108887809018b93acfb8e6ef56ecf7bb3529ebd;p=libcds.git Merge pull request #27 from krinkinmu/fastpath-opt Optimize SplitList inc_item_count method fastpath --- diff --git a/cds/algo/atomic.h b/cds/algo/atomic.h index 1d3cd243..92bb48d9 100644 --- a/cds/algo/atomic.h +++ b/cds/algo/atomic.h @@ -187,7 +187,7 @@ namespace cds { {} /// Returns current value of the counter - counter_type value(atomics::memory_order order = atomics::memory_order_relaxed) const + counter_type value(atomics::memory_order order = atomics::memory_order_relaxed) const { return m_Counter.load( order ); } @@ -199,13 +199,13 @@ namespace cds { } /// Returns underlying atomic interface - atomic_type& getAtomic() + atomic_type& getAtomic() { return m_Counter; } /// Returns underlying atomic interface (const) - const atomic_type& getAtomic() const + const atomic_type& getAtomic() const { return m_Counter; } @@ -264,7 +264,7 @@ namespace cds { typedef size_t counter_type ; ///< Counter type public: /// Returns 0 - counter_type value(atomics::memory_order /*order*/ = atomics::memory_order_relaxed) const + counter_type value(atomics::memory_order /*order*/ = atomics::memory_order_relaxed) const { return 0; } diff --git a/cds/intrusive/details/split_list_base.h b/cds/intrusive/details/split_list_base.h index cef83734..cddfb628 100644 --- a/cds/intrusive/details/split_list_base.h +++ b/cds/intrusive/details/split_list_base.h @@ -418,7 +418,7 @@ namespace cds { namespace intrusive { { metrics m; - // Calculate m_nSegmentSize and m_nSegmentCount by nItemCount + // Calculate m_nSegmentSize and m_nSegmentCount by nItemCount m.nLoadFactor = nLoadFactor > 0 ? nLoadFactor : 1; size_t nBucketCount = (size_t)( ((float) nItemCount) / m.nLoadFactor ); diff --git a/cds/intrusive/split_list.h b/cds/intrusive/split_list.h index d67ab64d..0288e703 100644 --- a/cds/intrusive/split_list.h +++ b/cds/intrusive/split_list.h @@ -4,6 +4,7 @@ #define CDSLIB_INTRUSIVE_SPLIT_LIST_H #include +#include namespace cds { namespace intrusive { @@ -349,6 +350,7 @@ namespace cds { namespace intrusive { ordered_list_wrapper m_List; ///< Ordered list containing split-list items bucket_table m_Buckets; ///< bucket table atomics::atomic m_nBucketCountLog2; ///< log2( current bucket count ) + atomics::atomic m_nMaxItemCount; ///< number of items container can hold, before we have to resize item_counter m_ItemCounter; ///< Item counter hash m_HashFunctor; ///< Hash functor stat m_Stat; ///< Internal statistics @@ -415,7 +417,6 @@ namespace cds { namespace intrusive { // In this point, we must wait while nBucket is empty. // The compiler can decide that waiting loop can be "optimized" (stripped) // To prevent this situation, we use waiting on volatile bucket_head_ptr pointer. - // m_Stat.onBucketInitContenton(); back_off bkoff; while ( true ) { @@ -458,13 +459,28 @@ namespace cds { namespace intrusive { m_Buckets.bucket( 0, pNode ); } - void inc_item_count() + static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor ) { + return nBucketCount * nLoadFactor; + } + + void inc_item_count() + { + size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed); + if ( ++m_ItemCounter <= nMaxCount ) + return; + + const size_t nLoadFactor = m_Buckets.load_factor(); size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed); - if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() ) - { - m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed ); - } + const size_t nBucketCount = static_cast(1) << sz; + if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor )) + return; // someone already have updated m_nBucketCountLog2, so stop here + + const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor ) + : std::numeric_limits::max(); + m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed, + memory_model::memory_order_relaxed ); + m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed ); } template @@ -583,11 +599,12 @@ namespace cds { namespace intrusive { /// Initialize split-ordered list of default capacity /** The default capacity is defined in bucket table constructor. - See \p split_list::expandable_bucket_table, \p split_list::static_ducket_table + See \p split_list::expandable_bucket_table, \p split_list::static_bucket_table which selects by \p split_list::dynamic_bucket_table option. */ SplitListSet() : m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); } @@ -599,6 +616,7 @@ namespace cds { namespace intrusive { ) : m_Buckets( nItemCount, nLoadFactor ) , m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); } diff --git a/cds/intrusive/split_list_nogc.h b/cds/intrusive/split_list_nogc.h index 8166312f..5bf894ba 100644 --- a/cds/intrusive/split_list_nogc.h +++ b/cds/intrusive/split_list_nogc.h @@ -6,6 +6,8 @@ #include #include +#include + namespace cds { namespace intrusive { /// Split-ordered list (template specialization for gc::nogc) @@ -144,6 +146,7 @@ namespace cds { namespace intrusive { ordered_list_wrapper m_List; ///< Ordered list containing split-list items bucket_table m_Buckets; ///< bucket table atomics::atomic m_nBucketCountLog2; ///< log2( current bucket count ) + atomics::atomic m_nMaxItemCount; ///< number of items container can hold, before we have to resize item_counter m_ItemCounter; ///< Item counter hash m_HashFunctor; ///< Hash functor stat m_Stat; ///< Internal statistics @@ -253,13 +256,28 @@ namespace cds { namespace intrusive { m_Buckets.bucket( 0, pNode ); } - void inc_item_count() + static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor ) + { + return nBucketCount * nLoadFactor; + } + + void inc_item_count() { + size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed); + if ( ++m_ItemCounter <= nMaxCount ) + return; + + const size_t nLoadFactor = m_Buckets.load_factor(); size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed); - if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() ) - { - m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed ); - } + const size_t nBucketCount = static_cast(1) << sz; + if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor )) + return; // someone already have updated m_nBucketCountLog2, so stop here + + const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor ) + : std::numeric_limits::max(); + m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed, + memory_model::memory_order_relaxed ); + m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed ); } //@endcond @@ -273,6 +291,7 @@ namespace cds { namespace intrusive { */ SplitListSet() : m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); } @@ -284,6 +303,7 @@ namespace cds { namespace intrusive { ) : m_Buckets( nItemCount, nLoadFactor ) , m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); } diff --git a/cds/intrusive/split_list_rcu.h b/cds/intrusive/split_list_rcu.h index 8a7916b4..8deeb7ce 100644 --- a/cds/intrusive/split_list_rcu.h +++ b/cds/intrusive/split_list_rcu.h @@ -5,6 +5,7 @@ #include #include +#include namespace cds { namespace intrusive { @@ -241,6 +242,7 @@ namespace cds { namespace intrusive { ordered_list_wrapper m_List; ///< Ordered list containing split-list items bucket_table m_Buckets; ///< bucket table atomics::atomic m_nBucketCountLog2; ///< log2( current bucket count ) + atomics::atomic m_nMaxItemCount; ///< number of items container can hold, before we have to resize item_counter m_ItemCounter; ///< Item counter hash m_HashFunctor; ///< Hash functor stat m_Stat; ///< Internal stattistics accumulator @@ -350,13 +352,28 @@ namespace cds { namespace intrusive { m_Buckets.bucket( 0, pNode ); } - void inc_item_count() + static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor ) { + return nBucketCount * nLoadFactor; + } + + void inc_item_count() + { + size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed); + if ( ++m_ItemCounter <= nMaxCount ) + return; + + const size_t nLoadFactor = m_Buckets.load_factor(); size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed); - if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() ) - { - m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed ); - } + const size_t nBucketCount = static_cast(1) << sz; + if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor )) + return; // someone already have updated m_nBucketCountLog2, so stop here + + const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor ) + : std::numeric_limits::max(); + m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed, + memory_model::memory_order_relaxed ); + m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed ); } template @@ -465,6 +482,7 @@ namespace cds { namespace intrusive { */ SplitListSet() : m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); } @@ -476,6 +494,7 @@ namespace cds { namespace intrusive { ) : m_Buckets( nItemCount, nLoadFactor ) , m_nBucketCountLog2(1) + , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) ) { init(); }