From: Yedidya Feldblum Date: Tue, 31 Oct 2017 00:57:54 +0000 (-0700) Subject: Move folly/experimental/AsymmetricMemoryBarrier.h X-Git-Tag: v2017.11.06.00~31 X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=056261eb833b833e8b5edb90f37207c4c622a40d;p=folly.git Move folly/experimental/AsymmetricMemoryBarrier.h Summary: [Folly] Move `folly/experimental/AsymmetricMemoryBarrier.h` to `folly/synchronization/AsymmetricMemoryBarrier.h`. Reviewed By: Orvid Differential Revision: D6180676 fbshipit-source-id: f4833318cd365181e202d5f379815e728fba168b --- diff --git a/folly/Makefile.am b/folly/Makefile.am index d535a3b6..a351f990 100644 --- a/folly/Makefile.am +++ b/folly/Makefile.am @@ -126,7 +126,6 @@ nobase_follyinclude_HEADERS = \ Expected.h \ concurrency/AtomicSharedPtr.h \ concurrency/detail/AtomicSharedPtr-detail.h \ - experimental/AsymmetricMemoryBarrier.h \ experimental/AutoTimer.h \ experimental/ThreadedRepeatingFunctionRunner.h \ experimental/Bits.h \ @@ -423,6 +422,7 @@ nobase_follyinclude_HEADERS = \ stats/MultiLevelTimeSeries.h \ stats/TimeseriesHistogram-defs.h \ stats/TimeseriesHistogram.h \ + synchronization/AsymmetricMemoryBarrier.h \ synchronization/CallOnce.h \ synchronization/detail/AtomicUtils.h \ system/MemoryMapping.h \ @@ -609,6 +609,7 @@ libfolly_la_SOURCES = \ stats/Histogram.cpp \ stats/MultiLevelTimeSeries.cpp \ stats/TimeseriesHistogram.cpp \ + synchronization/AsymmetricMemoryBarrier.cpp \ system/MemoryMapping.cpp \ system/Shell.cpp \ system/ThreadName.cpp \ @@ -617,7 +618,6 @@ libfolly_la_SOURCES = \ TimeoutQueue.cpp \ Try.cpp \ Uri.cpp \ - experimental/AsymmetricMemoryBarrier.cpp \ experimental/ThreadedRepeatingFunctionRunner.cpp \ experimental/bser/Dump.cpp \ experimental/bser/Load.cpp \ diff --git a/folly/experimental/AsymmetricMemoryBarrier.cpp b/folly/experimental/AsymmetricMemoryBarrier.cpp deleted file mode 100644 index c8ae0178..00000000 --- a/folly/experimental/AsymmetricMemoryBarrier.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "AsymmetricMemoryBarrier.h" - -#include -#include -#include -#include -#include - -namespace folly { - -namespace { - -struct DummyPageCreator { - DummyPageCreator() { - get(); - } - - static void* get() { - static auto ptr = kIsLinux ? create() : nullptr; - return ptr; - } - - private: - static void* create() { - auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - checkUnixError(reinterpret_cast(ptr), "mmap"); - - // Optimistically try to lock the page so it stays resident. Could make - // the heavy barrier faster. - auto r = mlock(ptr, 1); - if (r != 0) { - // Do nothing. - } - - return ptr; - } -}; - -// Make sure dummy page is always initialized before shutdown. -DummyPageCreator dummyPageCreator; - -void mprotectMembarrier() { - auto dummyPage = dummyPageCreator.get(); - - // This function is required to be safe to call on shutdown, - // so we must leak the mutex. - static Indestructible mprotectMutex; - std::lock_guard lg(*mprotectMutex); - - int r = 0; - - // We want to downgrade the page while it is resident. To do that, it must - // first be upgraded and forced to be resident. - r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE); - checkUnixError(r, "mprotect"); - - // Force the page to be resident. If it is already resident, almost no-op. - *static_cast(dummyPage) = 0; - - // Downgrade the page. Forces a memory barrier in every core running any - // of the process's threads. On a sane platform. - r = mprotect(dummyPage, 1, PROT_READ); - checkUnixError(r, "mprotect"); -} -} // namespace - -void asymmetricHeavyBarrier(AMBFlags flags) { - if (kIsLinux) { - static const bool useSysMembarrier = detail::sysMembarrierAvailable(); - // sys_membarrier currently does not support EXPEDITED - if (useSysMembarrier && flags != AMBFlags::EXPEDITED) { - auto r = detail::sysMembarrier(); - checkUnixError(r, "membarrier"); - } else { - mprotectMembarrier(); - } - } else { - std::atomic_thread_fence(std::memory_order_seq_cst); - } -} -} // namespace folly diff --git a/folly/experimental/AsymmetricMemoryBarrier.h b/folly/experimental/AsymmetricMemoryBarrier.h deleted file mode 100644 index c20157db..00000000 --- a/folly/experimental/AsymmetricMemoryBarrier.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include - -namespace folly { - -enum class AMBFlags { - NORMAL, - EXPEDITED, -}; - -FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() { - if (kIsLinux) { - asm_volatile_memory(); - } else { - std::atomic_thread_fence(std::memory_order_seq_cst); - } -} - -void asymmetricHeavyBarrier(AMBFlags flags = AMBFlags::NORMAL); -} // namespace folly diff --git a/folly/experimental/TLRefCount.h b/folly/experimental/TLRefCount.h index 3e099ea8..e8baf26e 100644 --- a/folly/experimental/TLRefCount.h +++ b/folly/experimental/TLRefCount.h @@ -16,7 +16,7 @@ #pragma once #include -#include +#include namespace folly { diff --git a/folly/experimental/hazptr/hazptr-impl.h b/folly/experimental/hazptr/hazptr-impl.h index 82734d6f..a9f4c20b 100644 --- a/folly/experimental/hazptr/hazptr-impl.h +++ b/folly/experimental/hazptr/hazptr-impl.h @@ -59,8 +59,8 @@ #endif #include -#include #include +#include #include // for thread caching #include // for hash set in bulk reclamation diff --git a/folly/synchronization/AsymmetricMemoryBarrier.cpp b/folly/synchronization/AsymmetricMemoryBarrier.cpp new file mode 100644 index 00000000..c8ae0178 --- /dev/null +++ b/folly/synchronization/AsymmetricMemoryBarrier.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "AsymmetricMemoryBarrier.h" + +#include +#include +#include +#include +#include + +namespace folly { + +namespace { + +struct DummyPageCreator { + DummyPageCreator() { + get(); + } + + static void* get() { + static auto ptr = kIsLinux ? create() : nullptr; + return ptr; + } + + private: + static void* create() { + auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + checkUnixError(reinterpret_cast(ptr), "mmap"); + + // Optimistically try to lock the page so it stays resident. Could make + // the heavy barrier faster. + auto r = mlock(ptr, 1); + if (r != 0) { + // Do nothing. + } + + return ptr; + } +}; + +// Make sure dummy page is always initialized before shutdown. +DummyPageCreator dummyPageCreator; + +void mprotectMembarrier() { + auto dummyPage = dummyPageCreator.get(); + + // This function is required to be safe to call on shutdown, + // so we must leak the mutex. + static Indestructible mprotectMutex; + std::lock_guard lg(*mprotectMutex); + + int r = 0; + + // We want to downgrade the page while it is resident. To do that, it must + // first be upgraded and forced to be resident. + r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE); + checkUnixError(r, "mprotect"); + + // Force the page to be resident. If it is already resident, almost no-op. + *static_cast(dummyPage) = 0; + + // Downgrade the page. Forces a memory barrier in every core running any + // of the process's threads. On a sane platform. + r = mprotect(dummyPage, 1, PROT_READ); + checkUnixError(r, "mprotect"); +} +} // namespace + +void asymmetricHeavyBarrier(AMBFlags flags) { + if (kIsLinux) { + static const bool useSysMembarrier = detail::sysMembarrierAvailable(); + // sys_membarrier currently does not support EXPEDITED + if (useSysMembarrier && flags != AMBFlags::EXPEDITED) { + auto r = detail::sysMembarrier(); + checkUnixError(r, "membarrier"); + } else { + mprotectMembarrier(); + } + } else { + std::atomic_thread_fence(std::memory_order_seq_cst); + } +} +} // namespace folly diff --git a/folly/synchronization/AsymmetricMemoryBarrier.h b/folly/synchronization/AsymmetricMemoryBarrier.h new file mode 100644 index 00000000..c20157db --- /dev/null +++ b/folly/synchronization/AsymmetricMemoryBarrier.h @@ -0,0 +1,39 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +namespace folly { + +enum class AMBFlags { + NORMAL, + EXPEDITED, +}; + +FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() { + if (kIsLinux) { + asm_volatile_memory(); + } else { + std::atomic_thread_fence(std::memory_order_seq_cst); + } +} + +void asymmetricHeavyBarrier(AMBFlags flags = AMBFlags::NORMAL); +} // namespace folly