Merge pull request #27 from krinkinmu/fastpath-opt

Optimize SplitList inc_item_count method fastpath

diff --git a/cds/algo/atomic.h b/cds/algo/atomic.h
index 1d3cd24382185960c53e43bb622a4f58ad8a2a07..92bb48d9cc0b1d2441a0c21be019eb8be7c8613f 100644 (file)
--- a/cds/algo/atomic.h
+++ b/cds/algo/atomic.h
@@ -187,7 +187,7 @@ namespace cds {
             {}
 
             /// Returns current value of the counter
-            counter_type    value(atomics::memory_order order = atomics::memory_order_relaxed) const
+            counter_type value(atomics::memory_order order = atomics::memory_order_relaxed) const
             {
                 return m_Counter.load( order );
             }
@@ -199,13 +199,13 @@ namespace cds {
             }
 
             /// Returns underlying atomic interface
-            atomic_type&  getAtomic()
+            atomic_type& getAtomic()
             {
                 return m_Counter;
             }
 
             /// Returns underlying atomic interface (const)
-            const atomic_type&  getAtomic() const
+            const atomic_type& getAtomic() const
             {
                 return m_Counter;
             }
@@ -264,7 +264,7 @@ namespace cds {
             typedef size_t counter_type    ;  ///< Counter type
         public:
             /// Returns 0
-            counter_type    value(atomics::memory_order /*order*/ = atomics::memory_order_relaxed) const
+            counter_type value(atomics::memory_order /*order*/ = atomics::memory_order_relaxed) const
             {
                 return 0;
             }

diff --git a/cds/intrusive/details/split_list_base.h b/cds/intrusive/details/split_list_base.h
index cef83734e697fa87d5b4094ec671eb41d288fcc9..cddfb628bfc7aecede37bb11eaa4ac247bc7cda8 100644 (file)
--- a/cds/intrusive/details/split_list_base.h
+++ b/cds/intrusive/details/split_list_base.h
@@ -418,7 +418,7 @@ namespace cds { namespace intrusive {
             {
                 metrics m;
 
-                // Calculate m_nSegmentSize and m_nSegmentCount  by nItemCount
+                // Calculate m_nSegmentSize and m_nSegmentCount by nItemCount
                 m.nLoadFactor = nLoadFactor > 0 ? nLoadFactor : 1;
 
                 size_t nBucketCount = (size_t)( ((float) nItemCount) / m.nLoadFactor );

diff --git a/cds/intrusive/split_list.h b/cds/intrusive/split_list.h
index d67ab64daa876e0e8327a689f8d65876ff03426c..0288e703f5c74b9aa1b6cd4eafdaafd67c71b1b1 100644 (file)
--- a/cds/intrusive/split_list.h
+++ b/cds/intrusive/split_list.h
@@ -4,6 +4,7 @@
 #define CDSLIB_INTRUSIVE_SPLIT_LIST_H
 
 #include <cds/intrusive/details/split_list_base.h>
+#include <limits>
 
 namespace cds { namespace intrusive {
 
@@ -349,6 +350,7 @@ namespace cds { namespace intrusive {
         ordered_list_wrapper    m_List;             ///< Ordered list containing split-list items
         bucket_table            m_Buckets;          ///< bucket table
         atomics::atomic<size_t> m_nBucketCountLog2; ///< log2( current bucket count )
+        atomics::atomic<size_t> m_nMaxItemCount;    ///< number of items container can hold, before we have to resize
         item_counter            m_ItemCounter;      ///< Item counter
         hash                    m_HashFunctor;      ///< Hash functor
         stat                    m_Stat;             ///< Internal statistics
@@ -415,7 +417,6 @@ namespace cds { namespace intrusive {
             // In this point, we must wait while nBucket is empty.
             // The compiler can decide that waiting loop can be "optimized" (stripped)
             // To prevent this situation, we use waiting on volatile bucket_head_ptr pointer.
-            //
             m_Stat.onBucketInitContenton();
             back_off bkoff;
             while ( true ) {
@@ -458,13 +459,28 @@ namespace cds { namespace intrusive {
             m_Buckets.bucket( 0, pNode );
         }
 
-        void    inc_item_count()
+        static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor )
         {
+            return nBucketCount * nLoadFactor;
+        }
+
+        void inc_item_count()
+        {
+            size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed);
+            if ( ++m_ItemCounter <= nMaxCount )
+                return;
+
+            const size_t nLoadFactor = m_Buckets.load_factor();
             size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed);
-            if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() )
-            {
-                m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed );
-            }
+            const size_t nBucketCount = static_cast<size_t>(1) << sz;
+            if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor ))
+                return; // someone already have updated m_nBucketCountLog2, so stop here
+
+            const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor )
+                                                                              : std::numeric_limits<size_t>::max();
+            m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed,
+                memory_model::memory_order_relaxed );
+            m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed );
         }
 
         template <typename Q, typename Compare, typename Func>
@@ -583,11 +599,12 @@ namespace cds { namespace intrusive {
         /// Initialize split-ordered list of default capacity
         /**
             The default capacity is defined in bucket table constructor.
-            See \p split_list::expandable_bucket_table, \p split_list::static_ducket_table
+            See \p split_list::expandable_bucket_table, \p split_list::static_bucket_table
             which selects by \p split_list::dynamic_bucket_table option.
         */
         SplitListSet()
             : m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }
@@ -599,6 +616,7 @@ namespace cds { namespace intrusive {
             )
             : m_Buckets( nItemCount, nLoadFactor )
             , m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }

diff --git a/cds/intrusive/split_list_nogc.h b/cds/intrusive/split_list_nogc.h
index 8166312f2576f17d4b5b57577af34695f243eed4..5bf894ba515628555dfe2181ff06a390aa65d4f5 100644 (file)
--- a/cds/intrusive/split_list_nogc.h
+++ b/cds/intrusive/split_list_nogc.h
@@ -6,6 +6,8 @@
 #include <cds/intrusive/details/split_list_base.h>
 #include <cds/gc/nogc.h>
 
+#include <limits>
+
 namespace cds { namespace intrusive {
 
     /// Split-ordered list (template specialization for gc::nogc)
@@ -144,6 +146,7 @@ namespace cds { namespace intrusive {
         ordered_list_wrapper    m_List;             ///< Ordered list containing split-list items
         bucket_table            m_Buckets;          ///< bucket table
         atomics::atomic<size_t> m_nBucketCountLog2; ///< log2( current bucket count )
+        atomics::atomic<size_t> m_nMaxItemCount;    ///< number of items container can hold, before we have to resize
         item_counter            m_ItemCounter;      ///< Item counter
         hash                    m_HashFunctor;      ///< Hash functor
         stat                    m_Stat;             ///< Internal statistics
@@ -253,13 +256,28 @@ namespace cds { namespace intrusive {
             m_Buckets.bucket( 0, pNode );
         }
 
-        void    inc_item_count()
+        static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor )
+        {
+            return nBucketCount * nLoadFactor;
+        }
+
+        void inc_item_count()
         {
+            size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed);
+            if ( ++m_ItemCounter <= nMaxCount )
+                return;
+
+            const size_t nLoadFactor = m_Buckets.load_factor();
             size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed);
-            if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() )
-            {
-                m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed );
-            }
+            const size_t nBucketCount = static_cast<size_t>(1) << sz;
+            if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor ))
+                return; // someone already have updated m_nBucketCountLog2, so stop here
+
+            const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor )
+                                                                              : std::numeric_limits<size_t>::max();
+            m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed,
+                memory_model::memory_order_relaxed );
+            m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed );
         }
 
         //@endcond
@@ -273,6 +291,7 @@ namespace cds { namespace intrusive {
         */
         SplitListSet()
             : m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }
@@ -284,6 +303,7 @@ namespace cds { namespace intrusive {
             )
             : m_Buckets( nItemCount, nLoadFactor )
             , m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }

diff --git a/cds/intrusive/split_list_rcu.h b/cds/intrusive/split_list_rcu.h
index 8a7916b41817e22604a169b4496e7f2a7c6da4ef..8deeb7ce75b84505d8a7b77f3988011f8f6c46a2 100644 (file)
--- a/cds/intrusive/split_list_rcu.h
+++ b/cds/intrusive/split_list_rcu.h
@@ -5,6 +5,7 @@
 
 #include <cds/intrusive/details/split_list_base.h>
 #include <cds/details/binary_functor_wrapper.h>
+#include <limits>
 
 namespace cds { namespace intrusive {
 
@@ -241,6 +242,7 @@ namespace cds { namespace intrusive {
         ordered_list_wrapper    m_List;             ///< Ordered list containing split-list items
         bucket_table            m_Buckets;          ///< bucket table
         atomics::atomic<size_t> m_nBucketCountLog2; ///< log2( current bucket count )
+        atomics::atomic<size_t> m_nMaxItemCount;    ///< number of items container can hold, before we have to resize
         item_counter            m_ItemCounter;      ///< Item counter
         hash                    m_HashFunctor;      ///< Hash functor
         stat                    m_Stat;             ///< Internal stattistics accumulator
@@ -350,13 +352,28 @@ namespace cds { namespace intrusive {
             m_Buckets.bucket( 0, pNode );
         }
 
-        void    inc_item_count()
+        static size_t max_item_count( size_t nBucketCount, size_t nLoadFactor )
         {
+            return nBucketCount * nLoadFactor;
+        }
+
+        void inc_item_count()
+        {
+            size_t nMaxCount = m_nMaxItemCount.load(memory_model::memory_order_relaxed);
+            if ( ++m_ItemCounter <= nMaxCount )
+                return;
+
+            const size_t nLoadFactor = m_Buckets.load_factor();
             size_t sz = m_nBucketCountLog2.load(memory_model::memory_order_relaxed);
-            if ( ( ++m_ItemCounter >> sz ) > m_Buckets.load_factor() && ((size_t)(1 << sz )) < m_Buckets.capacity() )
-            {
-                m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, atomics::memory_order_relaxed );
-            }
+            const size_t nBucketCount = static_cast<size_t>(1) << sz;
+            if ( nMaxCount < max_item_count( nBucketCount, nLoadFactor ))
+                return; // someone already have updated m_nBucketCountLog2, so stop here
+
+            const size_t nNewMaxCount = (nBucketCount < m_Buckets.capacity()) ? max_item_count( nBucketCount << 1, nLoadFactor )
+                                                                              : std::numeric_limits<size_t>::max();
+            m_nMaxItemCount.compare_exchange_strong( nMaxCount, nNewMaxCount, memory_model::memory_order_relaxed,
+                memory_model::memory_order_relaxed );
+            m_nBucketCountLog2.compare_exchange_strong( sz, sz + 1, memory_model::memory_order_relaxed, memory_model::memory_order_relaxed );
         }
 
         template <typename Q, typename Compare, typename Func>
@@ -465,6 +482,7 @@ namespace cds { namespace intrusive {
         */
         SplitListSet()
             : m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }
@@ -476,6 +494,7 @@ namespace cds { namespace intrusive {
             )
             : m_Buckets( nItemCount, nLoadFactor )
             , m_nBucketCountLog2(1)
+            , m_nMaxItemCount( max_item_count(2, m_Buckets.load_factor()) )
         {
             init();
         }