From 3f0064f677cddc5d9c56f3beb2ef3010732e8d6f Mon Sep 17 00:00:00 2001 From: Andrii Grynenko Date: Mon, 18 Jul 2016 18:17:06 -0700 Subject: [PATCH] Use membarrier in TLRefCount Summary: membarrier guarantees that there's at most one update to thread-local counter, which collecting thread may not see. Reviewed By: djwatson Differential Revision: D3532952 fbshipit-source-id: 6106bfe87c70c5f864573a424662778e20423bbb --- folly/Makefile.am | 4 + folly/Portability.h | 9 ++ .../experimental/AsymmetricMemoryBarrier.cpp | 87 +++++++++++++++++++ folly/experimental/AsymmetricMemoryBarrier.h | 34 ++++++++ folly/experimental/TLRefCount.h | 12 ++- folly/portability/SysMembarrier.cpp | 58 +++++++++++++ folly/portability/SysMembarrier.h | 25 ++++++ 7 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 folly/experimental/AsymmetricMemoryBarrier.cpp create mode 100644 folly/experimental/AsymmetricMemoryBarrier.h create mode 100644 folly/portability/SysMembarrier.cpp create mode 100644 folly/portability/SysMembarrier.h diff --git a/folly/Makefile.am b/folly/Makefile.am index e71ec9a0..1dd5e1f4 100644 --- a/folly/Makefile.am +++ b/folly/Makefile.am @@ -89,6 +89,7 @@ nobase_follyinclude_HEADERS = \ ExceptionWrapper.h \ Executor.h \ EvictingCacheMap.h \ + experimental/AsymmetricMemoryBarrier.h \ experimental/AutoTimer.h \ experimental/Bits.h \ experimental/BitVectorCoding.h \ @@ -273,6 +274,7 @@ nobase_follyinclude_HEADERS = \ portability/String.h \ portability/Syslog.h \ portability/SysFile.h \ + portability/SysMembarrier.h \ portability/SysMman.h \ portability/SysResource.h \ portability/SysStat.h \ @@ -428,6 +430,7 @@ libfolly_la_SOURCES = \ portability/Stdlib.cpp \ portability/String.cpp \ portability/SysFile.cpp \ + portability/SysMembarrier.cpp \ portability/SysMman.cpp \ portability/SysResource.cpp \ portability/SysStat.cpp \ @@ -450,6 +453,7 @@ libfolly_la_SOURCES = \ TimeoutQueue.cpp \ Uri.cpp \ Version.cpp \ + experimental/AsymmetricMemoryBarrier.cpp \ experimental/bser/Dump.cpp \ experimental/bser/Load.cpp \ experimental/DynamicParser.cpp \ diff --git a/folly/Portability.h b/folly/Portability.h index 2d3ec417..5776a007 100644 --- a/folly/Portability.h +++ b/folly/Portability.h @@ -354,3 +354,12 @@ using namespace FOLLY_GFLAGS_NAMESPACE; // we will take the next one. #define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102))) #endif + +namespace folly { + +#ifdef __linux__ +constexpr auto kIsLinux = true; +#else +constexpr auto kIsLinux = false; +#endif +} diff --git a/folly/experimental/AsymmetricMemoryBarrier.cpp b/folly/experimental/AsymmetricMemoryBarrier.cpp new file mode 100644 index 00000000..31c34cb7 --- /dev/null +++ b/folly/experimental/AsymmetricMemoryBarrier.cpp @@ -0,0 +1,87 @@ +/* + * Copyright 2016 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "AsymmetricMemoryBarrier.h" + +#include +#include +#include +#include +#include + +namespace folly { + +namespace { + +struct DummyPageCreator { + DummyPageCreator() { + get(); + } + + static void* get() { + static auto ptr = + kIsLinux && !detail::sysMembarrierAvailable() ? create() : nullptr; + return ptr; + } + + private: + static void* create() { + auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + checkUnixError(reinterpret_cast(ptr), "mmap"); + + // Lock the memory so it can't get paged out. If it gets paged out, changing + // its protection won't accomplish anything. + auto r = mlock(ptr, 1); + checkUnixError(r, "mlock"); + + return ptr; + } +}; + +// Make sure dummy page is always initialized before shutdown +DummyPageCreator dummyPageCreator; + +void mprotectMembarrier() { + auto dummyPage = dummyPageCreator.get(); + + // This function is required to be safe to call on shutdown, + // so we must leak the mutex. + static Indestructible mprotectMutex; + std::lock_guard lg(*mprotectMutex); + + int r = 0; + r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE); + checkUnixError(r, "mprotect"); + + r = mprotect(dummyPage, 1, PROT_READ); + checkUnixError(r, "mprotect"); +} +} + +void asymmetricHeavyBarrier() { + if (kIsLinux) { + static const bool useSysMembarrier = detail::sysMembarrierAvailable(); + if (useSysMembarrier) { + auto r = detail::sysMembarrier(); + checkUnixError(r, "membarrier"); + } else { + mprotectMembarrier(); + } + } else { + std::atomic_thread_fence(std::memory_order_seq_cst); + } +} +} diff --git a/folly/experimental/AsymmetricMemoryBarrier.h b/folly/experimental/AsymmetricMemoryBarrier.h new file mode 100644 index 00000000..14573ccb --- /dev/null +++ b/folly/experimental/AsymmetricMemoryBarrier.h @@ -0,0 +1,34 @@ +/* + * Copyright 2016 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +namespace folly { + +FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() { + if (kIsLinux) { + asm_volatile_memory(); + } else { + std::atomic_thread_fence(std::memory_order_seq_cst); + } +} + +void asymmetricHeavyBarrier(); +} diff --git a/folly/experimental/TLRefCount.h b/folly/experimental/TLRefCount.h index 9667f4c1..78054edd 100644 --- a/folly/experimental/TLRefCount.h +++ b/folly/experimental/TLRefCount.h @@ -16,6 +16,7 @@ #pragma once #include +#include namespace folly { @@ -84,6 +85,8 @@ class TLRefCount { state_ = State::GLOBAL_TRANSITION; + asymmetricHeavyBarrier(); + std::weak_ptr collectGuardWeak = collectGuard_; // Make sure we can't create new LocalRefCounts @@ -147,7 +150,14 @@ class TLRefCount { return false; } - auto count = count_ += delta; + // This is equivalent to atomic fetch_add. We know that this operation + // is always performed from a single thread. asymmetricLightBarrier() + // makes things faster than atomic fetch_add on platforms with native + // support. + auto count = count_.load(std::memory_order_relaxed) + delta; + count_.store(count, std::memory_order_relaxed); + + asymmetricLightBarrier(); if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) { std::lock_guard lg(collectMutex_); diff --git a/folly/portability/SysMembarrier.cpp b/folly/portability/SysMembarrier.cpp new file mode 100644 index 00000000..923bece7 --- /dev/null +++ b/folly/portability/SysMembarrier.cpp @@ -0,0 +1,58 @@ +/* + * Copyright 2016 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SysMembarrier.h" + +#include + +#include +#include + +#if !defined(__NR_membarrier) && defined(FOLLY_X64) +#define __NR_membarrier 324 +#define MEMBARRIER_CMD_QUERY 0 +#define MEMBARRIER_CMD_SHARED 1 +#endif + +namespace folly { +namespace detail { + +bool sysMembarrierAvailable() { + if (!kIsLinux) { + return false; + } + +#ifdef __NR_membarrier + auto r = syscall(__NR_membarrier, MEMBARRIER_CMD_QUERY, /* flags = */ 0); + if (r == -1) { + return false; + } + + return r & MEMBARRIER_CMD_SHARED; +#else + return false; +#endif +} + +int sysMembarrier() { +#ifdef __NR_membarrier + return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, /* flags = */ 0); +#else + return -1; +#endif +} +} +} diff --git a/folly/portability/SysMembarrier.h b/folly/portability/SysMembarrier.h new file mode 100644 index 00000000..01284b6f --- /dev/null +++ b/folly/portability/SysMembarrier.h @@ -0,0 +1,25 @@ +/* + * Copyright 2016 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace folly { +namespace detail { + +int sysMembarrier(); +bool sysMembarrierAvailable(); +} +} -- 2.34.1